From 6e9f4a03d11367e61ed87f3113869f06736e4657 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 20 Apr 2016 13:28:27 -0400 Subject: [PATCH 01/60] Adding OBS_search function and tests --- src/pg/sql/42_observatory_exploration.sql | 33 +++++++++++++++++++ .../42_observatroy_exploration_test.out | 29 ++++++++++++++++ .../sql/42_observatory_exploration_test.sql | 3 ++ 3 files changed, 65 insertions(+) create mode 100644 src/pg/sql/42_observatory_exploration.sql create mode 100644 src/pg/test/expected/42_observatroy_exploration_test.out create mode 100644 src/pg/test/sql/42_observatory_exploration_test.sql diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql new file mode 100644 index 0000000..fd6961f --- /dev/null +++ b/src/pg/sql/42_observatory_exploration.sql @@ -0,0 +1,33 @@ +-- Functions used to search the observatory for measures +-------------------------------------------------------------------------------- +-- TODO allow the user to specify the boundary to search for measures +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search( + search_term text, + relevant_boundary text DEFAULT null +) +RETURNS TABLE(description text, name text, aggregate text, source text) as $$ +DECLARE + boundary_term text; +BEGIN + IF relevant_boundary then + boundary_term = ''; + else + boundary_term = ''; + END IF; + + RETURN QUERY + EXECUTE format($string$ + SELECT description, + name, + aggregate, + replace(split_part(id,'".', 1),'"', '') source + FROM observatory.OBS_column + where name ilike '%%%s%%' + or description ilike '%%%s%%' + %s + $string$, search_term, search_term,boundary_term); + RETURN; +END +$$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/42_observatroy_exploration_test.out b/src/pg/test/expected/42_observatroy_exploration_test.out new file mode 100644 index 0000000..b95c1f4 --- /dev/null +++ b/src/pg/test/expected/42_observatroy_exploration_test.out @@ -0,0 +1,29 @@ +\i test/sql/load_fixtures.sql +SET client_min_messages TO WARNING; +\set ECHO none +Loading obs_table.sql fixture file... +Done. +Loading obs_column.sql fixture file... +Done. +Loading obs_column_table.sql fixture file... +Done. +Loading obs_column_to_column.sql fixture file... +Done. +Loading obs_85328201013baa14e8e8a4a57a01e6f6fbc5f9b1.sql fixture file... +Done. +Loading obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb.sql fixture file... +Done. +Loading obs_ab038198aaab3f3cb055758638ee4de28ad70146.sql fixture file... +Done. +Loading obs_a92e1111ad3177676471d66bb8036e6d057f271b.sql fixture file... +Done. +Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... +Done. +Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... +Done. +obs_search_stu +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +("The total number of all people living in a geographic area.","Total Population",sum,es.ine) +("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Total Population",sum,us.census.acs) +("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Quantile:Total Population",quantile,us.census.acs) +(3 rows) diff --git a/src/pg/test/sql/42_observatory_exploration_test.sql b/src/pg/test/sql/42_observatory_exploration_test.sql new file mode 100644 index 0000000..22c811e --- /dev/null +++ b/src/pg/test/sql/42_observatory_exploration_test.sql @@ -0,0 +1,3 @@ +\i test/sql/load_fixtures.sql + +SELECT cdb_observatory.OBS_Search('total_pop'); From a29876f47f6b4e7b5f85b13ce3caef0761b73499 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 20 Apr 2016 13:42:13 -0400 Subject: [PATCH 02/60] making obs_search a bit more secure --- src/pg/sql/42_observatory_exploration.sql | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index fd6961f..b845e85 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -3,7 +3,7 @@ -- TODO allow the user to specify the boundary to search for measures -- -CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search( +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search_STU( search_term text, relevant_boundary text DEFAULT null ) @@ -24,10 +24,21 @@ BEGIN aggregate, replace(split_part(id,'".', 1),'"', '') source FROM observatory.OBS_column - where name ilike '%%%s%%' - or description ilike '%%%s%%' + where name ilike '%'|| %L || '%' + or description ilike '%'|| %L || '%' %s $string$, search_term, search_term,boundary_term); RETURN; END $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION OBS_GetAvailableBoundaries(geometry location) +RETURNS TABLE(description text, name text, id text) as $$ +BEGIN + RETURN QUERY + EXECUTE format($string$ + Select description, name, id FROM observatory.OBS_column + $string$) +END +$$ From cf93a0193b9448f23737f4dad2de5bb1485899f1 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 15:20:27 -0400 Subject: [PATCH 03/60] docs stub --- doc/methods.md | 176 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 doc/methods.md diff --git a/doc/methods.md b/doc/methods.md new file mode 100644 index 0000000..1cae40f --- /dev/null +++ b/doc/methods.md @@ -0,0 +1,176 @@ +# Measures + +Measures Services allow users to access geospatial measures for analysis workflows. Measures are used by sending an identifier or a geometry (Point or Polygon) and receiving back a measure or absolute value for that location. Every measure contained in the Data Catalog can be accessed through the CartoDB Editor. + +Below are the methods. For detailed information for accessing any measures, see the catalog here, [Catalog PDF](http://cartodb.github.io/bigmetadata/index.html) + +## OBS_GetUSCensusMeasure(point_geometry, measure_name); + +The ```OBS_GetUSCensusMeasure(point_geometry, measure_name)``` method returns a measure based on a subset of the US Census variables at a point location. The ```OBS_GetUSCensusMeasure``` method is limited to only a subset of all measures that are available in the Data Observatory, to access the full list, use the ```OBS_GetMeasure``` method below. + +#### Arguments + +Name |Description +--- | --- +geom | a WGS84 point geometry (the_geom) +measure_name | a human readable string name of a US Census variable. The glossary of measure_names is [available below]('measure_name table'). +normalize | for measures that are are **sums** (e.g. population) the default normalization is 'area' and response comes back as a rate per square kilometer. Other options are 'denominator', which will use the denominator specified in the + +#### Returns + +A JSON object containing the following properties + +Key | Description +--- | --- +value | the raw or normalized measure +name | the human readable name of the measure +description | a brief description of the measure +type | the data type (text, number, boolean) + +#### Example + +Add a Measure to an empty column based on point locations in your table + +```SQL +UPDATE tablename SET local_male_population = OBS_GetUSCensusMeasure(the_geom, 'Male Population') -> 'value' +``` + +Get a measure at a single point location + +```SQL +SELECT * FROM json_each(OBS_GetMeasure(CDB_LatLng(40.7, -73.9), 'Male Population')) +``` + + + +## OBS_GetUSCensusMeasure(polygon_geometry, measure_name); + +The ```OBS_GetUSCensusMeasure(point_geometry, measure_name)``` method returns a measure based on a subset of the US Census variables within a given polygon. The ```OBS_GetUSCensusMeasure``` method is limited to only a subset of all measures that are available in the Data Observatory, to access the full list, use the ```OBS_GetMeasure``` method below. + +#### Arguments + +Name |Description +--- | --- +geom | a WGS84 polygon geometry (the_geom) +measure_name | a human readable string name of a US Census variable. The glossary of measure_names is [available below]('measure_name table'). +normalize | for measures that are are **sums** (e.g. population) the default normalization is 'none' and response comes back as a raw value. Other options are 'denominator', which will use the denominator specified in the [Data Catalog](http://cartodb.github.io/bigmetadata/index.html) (optional) + +#### Returns + +A JSON object containing the following properties + +Key | Description +--- | --- +value | the raw or normalized measure +name | the human readable name of the measure +description | a brief description of the measure +type | the data type (text, number, boolean) + +#### Example + +Add a Measure to an empty column based on polygons in your table + +```SQL +UPDATE tablename SET local_male_population = OBS_GetUSCensusMeasure(the_geom, 'Male Population') -> 'value' +``` + +Get a measure at a single polygon + +```SQL +SELECT * FROM json_each(OBS_GetMeasure(ST_Buffer(CDB_LatLng(40.7, -73.9),0.001), 'Male Population')) +``` + + + +## OBS_GetMeasure(point_geometry, measure_id); + +The ```OBS_GetMeasure(point_geometry, measure_id)``` method returns any Data Observatory measure at a point location. + +#### Arguments + +Name |Description +--- | --- +geom | a WGS84 point geometry (the_geom) +measure_id | a measure identifier from the Data Observatory ([see available measures](http://cartodb.github.io/bigmetadata/index.html)) +normalize | for measures that are are **sums** (e.g. population) the default normalization is 'area' and response comes back as a rate per square kilometer. Other options are 'denominator', which will use the denominator specified in the [Data Catalog](http://cartodb.github.io/bigmetadata/index.html) and 'none' which will return a raw value. (optional) + +#### Returns + +A JSON object containing the following properties + +Key | Description +--- | --- +value | the raw or normalized measure +name | the human readable name of the measure +description | a brief description of the measure +type | the data type (text, number, boolean) + +#### Example + +Add a Measure to an empty column based on point locations in your table + +```SQL +UPDATE tablename SET local_male_population = OBS_GetMeasure(the_geom, '"us.census.acs".B08134006') -> 'value' +``` + +Get a measure at a single point location + +```SQL +SELECT * FROM json_each(OBS_GetMeasure(CDB_LatLng(40.7, -73.9), '"us.census.acs".B08134006')) +``` + + + + +## OBS_GetMeasure(polygon_geometry, measure_id); + +The ```OBS_GetMeasure(polygon_geometry, measure_id)``` method returns any Data Observatory measure calculated within a polygon. + +#### Arguments + +Name |Description +--- | --- +geom | a WGS84 polygon geometry (the_geom) +measure_id | a measure identifier from the Data Observatory ([see available measures](http://cartodb.github.io/bigmetadata/index.html)) +normalize | for measures that are are **sums** (e.g. population) the default normalization is 'none' and response comes back as a raw value. Other options are 'denominator', which will use the denominator specified in the [Data Catalog](http://cartodb.github.io/bigmetadata/index.html) (optional) + +#### Returns + +A JSON object containing the following properties + +Key | Description +--- | --- +value | the raw or normalized measure +name | the human readable name of the measure +description | a brief description of the measure +type | the data type (text, number, boolean) + +#### Example + +Add a Measure to an empty column based on polygons in your table + +```SQL +UPDATE tablename SET local_male_population = OBS_GetMeasure(the_geom, '"us.census.acs".B08134006') -> 'value' +``` + +Get a measure within a polygon + +```SQL +SELECT * FROM json_each(OBS_GetMeasure(ST_Buffer(CDB_LatLng(40.7, -73.9),0.001), '"us.census.acs".B08134006')) +``` + + + +--- + +# Boundaries + +# Discovery From b8d42a41dd1ed9f5cde34caa2f644a9a01ff316c Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 20 Apr 2016 15:28:45 -0400 Subject: [PATCH 04/60] adding OBS_GetAvailableBoundaries --- src/pg/sql/42_observatory_exploration.sql | 43 ++++++++++++++++--- .../sql/42_observatory_exploration_test.sql | 2 + 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index b845e85..a6063fa 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -24,8 +24,8 @@ BEGIN aggregate, replace(split_part(id,'".', 1),'"', '') source FROM observatory.OBS_column - where name ilike '%'|| %L || '%' - or description ilike '%'|| %L || '%' + where name ilike '%%' || %L || '%%' + or description ilike '%%' || %L || '%%' %s $string$, search_term, search_term,boundary_term); RETURN; @@ -33,12 +33,41 @@ END $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION OBS_GetAvailableBoundaries(geometry location) +-- Functions to return the geometry levels that a point is part of +-------------------------------------------------------------------------------- +-- TODO add test response + +CREATE OR REPLACE FUNCTION OBS_GetAvailableBoundaries( + geom geometry, + time_span text DEFAULT null) RETURNS TABLE(description text, name text, id text) as $$ +DECLARE + timespan_query TEXT DEFAULT ''; BEGIN + + IF time_span != null THEN + timespan_query = format('AND timespan = %L', time_span); + END IF; + RETURN QUERY - EXECUTE format($string$ - Select description, name, id FROM observatory.OBS_column - $string$) + EXECUTE + $string$ + select + column_id, + obs_column.description, + table_id + FROM + observatory.OBS_table, + observatory.OBS_column_table, + observatory.OBS_column + WHERE + observatory.OBS_column_table.column_id = observatory.obs_column.id + AND + observatory.OBS_column.type='Geometry' + AND + $1 && bounds::box2d + $string$ || timespan_query + USING geom + RETURN; END -$$ +$$ LANGUAGE plpgsql; diff --git a/src/pg/test/sql/42_observatory_exploration_test.sql b/src/pg/test/sql/42_observatory_exploration_test.sql index 22c811e..f9173ee 100644 --- a/src/pg/test/sql/42_observatory_exploration_test.sql +++ b/src/pg/test/sql/42_observatory_exploration_test.sql @@ -1,3 +1,5 @@ \i test/sql/load_fixtures.sql SELECT cdb_observatory.OBS_Search('total_pop'); + +SELECT * from cdb_observatory.OBS_GetAvailableBoundaries(_test_point()); From 1968b6486b1a21f9b40f8aaf8111e26ed8168734 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 15:31:05 -0400 Subject: [PATCH 05/60] not sure where to put this table... --- doc/methods.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/methods.md b/doc/methods.md index 1cae40f..a3bd7da 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -38,13 +38,20 @@ UPDATE tablename SET local_male_population = OBS_GetUSCensusMeasure(the_geom, 'M Get a measure at a single point location ```SQL -SELECT * FROM json_each(OBS_GetMeasure(CDB_LatLng(40.7, -73.9), 'Male Population')) +SELECT * FROM json_each(OBS_GetUSCensusMeasure(CDB_LatLng(40.7, -73.9), 'Male Population')) ``` +#### OBS_GetUSCensusMeasure names table + +Below is a list of human readable names accepted in the ```OBS_GetUSCensusMeasure``` method. For the more comprehensive list of columns available to the ```OBS_GetMeasure``` method, see the [Data Catalog] + +"Aggregate travel time to work", "American Indian and Alaska Native Population", "Asian Men age 45 to 54", "Asian Men age 55 to 64", "Asian Population", "Black Men age 45 to 54", "Black Men age 55 to 64", "Black or African American Population", "Children under 18 Years of Age", "Commuters by Bus", "Commuters by Carpool", "Commuters by Car, Truck, or Van", "Commuters by Public Transportation", "Commuters by Subway or Elevated", "Commuters who drove alone", "Currently married", "Divorced", "Employed Population", "Families with young children (under 6 years of age)", "Female Population", "Gini Index", "Hispanic Men age 45 to 54", "Hispanic Men age 55 to 64", "Hispanic Population", "Households", "Households with income less than $10,000", "Households with income of $100,000 To $124,999", "Households with income of $10,000 to $14,999", "Households with income of $125,000 To $149,999", "Households with income of $150,000 To $199,999", "Households with income of $15,000 to $19,999", "Households with income of $200,000 Or More", "Households with income of $20,000 To $24,999", "Households with income of $25,000 To $29,999", "Households with income of $30,000 To $34,999", "Households with income of $35,000 To $39,999", "Households with income of $40,000 To $44,999", "Households with income of $45,000 To $49,999", "Households with income of $50,000 To $59,999", "Households with income of $60,000 To $74,999", "Households with income of $75,000 To $99,999", "Housing Units", "Income In The Past 12 Months Below Poverty Level", "Male Population", "Married but separated", "Median Age", "Median Household Income in the past 12 Months", "Median Rent", "Men age 45 to 49", "Men age 45 to 64 ("middle aged")", "Men age 45 to 64 who attained between 9th and 12th grade, no diploma", "Men age 45 to 64 who attained less than a 9th grade education", "Men age 45 to 64 who completed high school or obtained GED", "Men age 45 to 64 who completed some college, no degree", "Men age 45 to 64 who obtained a bachelor's degree", "Men age 45 to 64 who obtained a graduate or professional degree", "Men age 45 to 64 who obtained an associate's degree", "Men age 50 to 54", "Men age 55 to 59", "Men age 60 to 61", "Men age 62 to 64", "Never Married", "Not a U.S. Citizen Population", "Number of workers with a commute between 10 and 14 minutes", "Number of workers with a commute between 15 and 19 minutes", "Number of workers with a commute between 20 and 24 minutes", "Number of workers with a commute between 25 and 29 minutes", "Number of workers with a commute between 30 and 34 minutes", "Number of workers with a commute between 35 and 44 minutes", "Number of workers with a commute between 45 and 59 minutes", "Number of workers with a commute of over 60 minutes", "Number of workers with less than 10 minute commute", "One-parent families, father in labor force, with young children (under 6 years of age)", "One-parent families, father, with young children (under 6 years of age)", "One-parent families with young children (under 6 years of age)", "Other Race population", "Owner-occupied Housing Units", "Owner-occupied Housing Units valued at $1,000,000 or more.", "Owner-occupied Housing Units with a Mortgage", "Per Capita Income in the past 12 Months", "Percent of Household Income Spent on Rent", "Population 15 Years and Over", "Population 25 Years and Over", "Population 3 Years and Over", "Population 5 Years and Over", "Population age 16 and over", "Population Completed Associate's Degree", "Population Completed Bachelor's Degree", "Population Completed High School", "Population completed less than one year of college, no degree", "Population Completed Master's Degree", "Population completed more than one year of college, no degree", "Population for Whom Poverty Status Determined", "Population in Armed Forces", "Population in Civilian Labor Force", "Population in Labor Force", "Population not Hispanic", "Population Not in Labor Force", "Speaks only English at Home", "Speaks Spanish at Home", "Students Enrolled as Undergraduate in College", "Students Enrolled in Grades 1 to 4", "Students Enrolled in Grades 5 to 8", "Students Enrolled in Grades 9 to 12", "Students Enrolled in School", "Total Population", "Two or more races population", "Two-parent families, both parents in labor force with young children (under 6 years of age)", "Two-parent families, father only in labor force with young children (under 6 years of age)", "Two-parent families, mother only in labor force with young children (under 6 years of age)", "Two-parent families, neither parent in labor force with young children (under 6 years of age)", "Two-parent families with young children (under 6 years of age)", "Unemployed Population", "US Census Block Group Geoids", "Vacant Housing Units", "Vacant Housing Units for Rent", "Vacant Housing Units for Sale", "Walked to Work", "White Men age 45 to 54", "White Men age 55 to 64", "White Population", "Widowed", "Worked at Home", "Workers age 16 and over who do not work from home", "Workers over the Age of 16" + + ## OBS_GetUSCensusMeasure(polygon_geometry, measure_name); The ```OBS_GetUSCensusMeasure(point_geometry, measure_name)``` method returns a measure based on a subset of the US Census variables within a given polygon. The ```OBS_GetUSCensusMeasure``` method is limited to only a subset of all measures that are available in the Data Observatory, to access the full list, use the ```OBS_GetMeasure``` method below. From 7e4243192e2ff40189c6360dc71d76a505ac5b45 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 15:41:48 -0400 Subject: [PATCH 06/60] better table... --- doc/methods.md | 145 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 6 deletions(-) diff --git a/doc/methods.md b/doc/methods.md index a3bd7da..ae5f36e 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -45,12 +45,6 @@ SELECT * FROM json_each(OBS_GetUSCensusMeasure(CDB_LatLng(40.7, -73.9), 'Male Po Should add the SQL API call here too --> -#### OBS_GetUSCensusMeasure names table - -Below is a list of human readable names accepted in the ```OBS_GetUSCensusMeasure``` method. For the more comprehensive list of columns available to the ```OBS_GetMeasure``` method, see the [Data Catalog] - -"Aggregate travel time to work", "American Indian and Alaska Native Population", "Asian Men age 45 to 54", "Asian Men age 55 to 64", "Asian Population", "Black Men age 45 to 54", "Black Men age 55 to 64", "Black or African American Population", "Children under 18 Years of Age", "Commuters by Bus", "Commuters by Carpool", "Commuters by Car, Truck, or Van", "Commuters by Public Transportation", "Commuters by Subway or Elevated", "Commuters who drove alone", "Currently married", "Divorced", "Employed Population", "Families with young children (under 6 years of age)", "Female Population", "Gini Index", "Hispanic Men age 45 to 54", "Hispanic Men age 55 to 64", "Hispanic Population", "Households", "Households with income less than $10,000", "Households with income of $100,000 To $124,999", "Households with income of $10,000 to $14,999", "Households with income of $125,000 To $149,999", "Households with income of $150,000 To $199,999", "Households with income of $15,000 to $19,999", "Households with income of $200,000 Or More", "Households with income of $20,000 To $24,999", "Households with income of $25,000 To $29,999", "Households with income of $30,000 To $34,999", "Households with income of $35,000 To $39,999", "Households with income of $40,000 To $44,999", "Households with income of $45,000 To $49,999", "Households with income of $50,000 To $59,999", "Households with income of $60,000 To $74,999", "Households with income of $75,000 To $99,999", "Housing Units", "Income In The Past 12 Months Below Poverty Level", "Male Population", "Married but separated", "Median Age", "Median Household Income in the past 12 Months", "Median Rent", "Men age 45 to 49", "Men age 45 to 64 ("middle aged")", "Men age 45 to 64 who attained between 9th and 12th grade, no diploma", "Men age 45 to 64 who attained less than a 9th grade education", "Men age 45 to 64 who completed high school or obtained GED", "Men age 45 to 64 who completed some college, no degree", "Men age 45 to 64 who obtained a bachelor's degree", "Men age 45 to 64 who obtained a graduate or professional degree", "Men age 45 to 64 who obtained an associate's degree", "Men age 50 to 54", "Men age 55 to 59", "Men age 60 to 61", "Men age 62 to 64", "Never Married", "Not a U.S. Citizen Population", "Number of workers with a commute between 10 and 14 minutes", "Number of workers with a commute between 15 and 19 minutes", "Number of workers with a commute between 20 and 24 minutes", "Number of workers with a commute between 25 and 29 minutes", "Number of workers with a commute between 30 and 34 minutes", "Number of workers with a commute between 35 and 44 minutes", "Number of workers with a commute between 45 and 59 minutes", "Number of workers with a commute of over 60 minutes", "Number of workers with less than 10 minute commute", "One-parent families, father in labor force, with young children (under 6 years of age)", "One-parent families, father, with young children (under 6 years of age)", "One-parent families with young children (under 6 years of age)", "Other Race population", "Owner-occupied Housing Units", "Owner-occupied Housing Units valued at $1,000,000 or more.", "Owner-occupied Housing Units with a Mortgage", "Per Capita Income in the past 12 Months", "Percent of Household Income Spent on Rent", "Population 15 Years and Over", "Population 25 Years and Over", "Population 3 Years and Over", "Population 5 Years and Over", "Population age 16 and over", "Population Completed Associate's Degree", "Population Completed Bachelor's Degree", "Population Completed High School", "Population completed less than one year of college, no degree", "Population Completed Master's Degree", "Population completed more than one year of college, no degree", "Population for Whom Poverty Status Determined", "Population in Armed Forces", "Population in Civilian Labor Force", "Population in Labor Force", "Population not Hispanic", "Population Not in Labor Force", "Speaks only English at Home", "Speaks Spanish at Home", "Students Enrolled as Undergraduate in College", "Students Enrolled in Grades 1 to 4", "Students Enrolled in Grades 5 to 8", "Students Enrolled in Grades 9 to 12", "Students Enrolled in School", "Total Population", "Two or more races population", "Two-parent families, both parents in labor force with young children (under 6 years of age)", "Two-parent families, father only in labor force with young children (under 6 years of age)", "Two-parent families, mother only in labor force with young children (under 6 years of age)", "Two-parent families, neither parent in labor force with young children (under 6 years of age)", "Two-parent families with young children (under 6 years of age)", "Unemployed Population", "US Census Block Group Geoids", "Vacant Housing Units", "Vacant Housing Units for Rent", "Vacant Housing Units for Sale", "Walked to Work", "White Men age 45 to 54", "White Men age 55 to 64", "White Population", "Widowed", "Worked at Home", "Workers age 16 and over who do not work from home", "Workers over the Age of 16" - ## OBS_GetUSCensusMeasure(polygon_geometry, measure_name); @@ -181,3 +175,142 @@ Should add the SQL API call here too # Boundaries # Discovery + +# Glossary + +#### OBS_GetUSCensusMeasure names table + +Below is a list of human readable names accepted in the ```OBS_GetUSCensusMeasure``` method. For the more comprehensive list of columns available to the ```OBS_GetMeasure``` method, see the [Data Catalog] + +Measure name | Measure description +Total Population | The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates. +Male Population | The number of people within each geography who are male. +Female Population | The number of people within each geography who are female. +Median Age | The median age of all people in a given geographic area. +"White Population | The number of people identifying as white, non-Hispanic in each geography." +"Black or African American Population | The number of people identifying as black or African American, non-Hispanic in each geography." +"Asian Population | The number of people identifying as Asian, non-Hispanic in each geography." +Hispanic Population | The number of people identifying as Hispanic or Latino in each geography. +American Indian and Alaska Native Population | The number of people identifying as American Indian or Alaska native in each geography. +Other Race population | The number of people identifying as another race in each geography +Two or more races population | The number of people identifying as two or more races in each geography +Population not Hispanic | The number of people not identifying as Hispanic or Latino in each geography. +Not a U.S. Citizen Population | The number of people within each geography who indicated that they are not U.S. citizens. +"Workers over the Age of 16 | The number of people in each geography who work. Workers include those employed at private for-profit companies, the self-employed, government workers and non-profit employees." +"Commuters by Car, Truck, or Van | The number of workers age 16 years and over within a geographic area who primarily traveled to work by car, truck or van. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work." +"Commuters who drove alone | The number of workers age 16 years and over within a geographic area who primarily traveled by car driving alone. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work." +"Commuters by Carpool | The number of workers age 16 years and over within a geographic area who primarily traveled to work by carpool. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work." +"Commuters by Public Transportation | The number of workers age 16 years and over within a geographic area who primarily traveled to work by public transportation. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work." +"Commuters by Bus | The number of workers age 16 years and over within a geographic area who primarily traveled to work by bus. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. This is a subset of workers who commuted by public transport." +"Commuters by Subway or Elevated | The number of workers age 16 years and over within a geographic area who primarily traveled to work by subway or elevated train. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. This is a subset of workers who commuted by public transport." +"Walked to Work | The number of workers age 16 years and over within a geographic area who primarily walked to work. This would mean that of any way of getting to work, they travelled the most distance walking." +Worked at Home | The count within a geographical area of workers over the age of 16 who worked at home. +Children under 18 Years of Age | The number of people within each geography who are under 18 years of age. +"Households | A count of the number of households in each geography. A household consists of one or more people who live in the same dwelling and also share at meals or living accommodation, and may consist of a single family or some other grouping of people. " +Population 3 Years and Over | The total number of people in each geography age 3 years and over. This denominator is mostly used to calculate rates of school enrollment. +"Students Enrolled in School | The total number of people in each geography currently enrolled at any level of school, from nursery or pre-school to advanced post-graduate education. Only includes those over the age of 3." +Students Enrolled in Grades 1 to 4 | The total number of people in each geography currently enrolled in grades 1 through 4 inclusive. This corresponds roughly to elementary school. +Students Enrolled in Grades 5 to 8 | The total number of people in each geography currently enrolled in grades 5 through 8 inclusive. This corresponds roughly to middle school. +Students Enrolled in Grades 9 to 12 | The total number of people in each geography currently enrolled in grades 9 through 12 inclusive. This corresponds roughly to high school. +"Students Enrolled as Undergraduate in College | The number of people in a geographic area who are enrolled in college at the undergraduate level. Enrollment refers to being registered or listed as a student in an educational program leading to a college degree. This may be a public school or college, a private school or college." +Population 25 Years and Over | The number of people in a geographic area who are over the age of 25. This is used mostly as a denominator of educational attainment. +"Population Completed High School | The number of people in a geographic area over the age of 25 who completed high school, and did not complete a more advanced degree." +"Population completed less than one year of college, no degree | The number of people in a geographic area over the age of 25 who attended college for less than one year and no further." +"Population completed more than one year of college, no degree | The number of people in a geographic area over the age of 25 who attended college for more than one year but did not obtain a degree" +"Population Completed Associate's Degree | The number of people in a geographic area over the age of 25 who obtained a associate's degree, and did not complete a more advanced degree." +"Population Completed Bachelor's Degree | The number of people in a geographic area over the age of 25 who obtained a bachelor's degree, and did not complete a more advanced degree." +"Population Completed Master's Degree | The number of people in a geographic area over the age of 25 who obtained a master's degree, but did not complete a more advanced degree." +Population 5 Years and Over | The number of people in a geographic area who are over the age of 5. This is primarily used as a denominator of measures of language spoken at home. +Speaks only English at Home | The number of people in a geographic area over age 5 who speak only English at home. +"Speaks Spanish at Home | The number of people in a geographic area over age 5 who speak Spanish at home, possibly in addition to other languages." +"Population for Whom Poverty Status Determined | The number of people in each geography who could be identified as either living in poverty or not. This should be used as the denominator when calculating poverty rates, as it excludes people for whom it was not possible to determine poverty." +"Income In The Past 12 Months Below Poverty Level | The number of people in a geographic area who are part of a family (which could be just them as an individual) determined to be ""in poverty"" following the Office of Management and Budget's Directive 14. (https://www.census.gov/hhes/povmeas/methodology/ombdir14.html)" +"Median Household Income in the past 12 Months | Within a geographic area, the median income received by every household on a regular basis before payments for personal income taxes, social security, union dues, medicare deductions, etc. It includes income received from wages, salary, commissions, bonuses, and tips; self-employment income from own nonfarm or farm businesses, including proprietorships and partnerships; interest, dividends, net rental income, royalty income, or income from estates and trusts; Social Security or Railroad Retirement income; Supplemental Security Income (SSI); any cash public assistance or welfare payments from the state or local welfare office; retirement, survivor, or disability benefits; and any other sources of income received regularly such as Veterans' (VA) payments, unemployment and/or worker's compensation, child support, and alimony." +Gini Index | +Per Capita Income in the past 12 Months | +"Housing Units | A count of housing units in each geography. A housing unit is a house, an apartment, a mobile home or trailer, a group of rooms, or a single room occupied as separate living quarters, or if vacant, intended for occupancy as separate living quarters." +"Vacant Housing Units | The count of vacant housing units in a geographic area. A housing unit is vacant if no one is living in it at the time of enumeration, unless its occupants are only temporarily absent. Units temporarily occupied at the time of enumeration entirely by people who have a usual residence elsewhere are also classified as vacant." +"Vacant Housing Units for Rent | The count of vacant housing units in a geographic area that are for rent. A housing unit is vacant if no one is living in it at the time of enumeration, unless its occupants are only temporarily absent. Units temporarily occupied at the time of enumeration entirely by people who have a usual residence elsewhere are also classified as vacant." +"Vacant Housing Units for Sale | The count of vacant housing units in a geographic area that are for sale. A housing unit is vacant if no one is living in it at the time of enumeration, unless its occupants are only temporarily absent. Units temporarily occupied at the time of enumeration entirely by people who have a usual residence elsewhere are also classified as vacant." +"Median Rent | The median contract rent within a geographic area. The contract rent is the monthly rent agreed to or contracted for, regardless of any furnishings, utilities, fees, meals, or services that may be included. For vacant units, it is the monthly rent asked for the rental unit at the time of interview." +"Percent of Household Income Spent on Rent | Within a geographic area, the median percentage of household income which was spent on gross rent. Gross rent is the amount of the contract rent plus the estimated average monthly cost of utilities (electricity, gas, water, sewer etc.) and fuels (oil, coal, wood, etc.) if these are paid by the renter. Household income is the sum of the income of all people 15 years and older living in the household." +Owner-occupied Housing Units | +"Owner-occupied Housing Units valued at $1,000,000 or more. | The count of owner occupied housing units in a geographic area that are valued at $1,000,000 or more. Value is the respondent's estimate of how much the property (house and lot, mobile home and lot, or condominium unit) would sell for if it were for sale." +"Owner-occupied Housing Units with a Mortgage | The count of housing units within a geographic area that are mortagaged. ""Mortgage"" refers to all forms of debt where the property is pledged as security for repayment of the debt, including deeds of trust, trust deed, contracts to purchase, land contracts, junior mortgages, and home equity loans." +Families with young children (under 6 years of age) | +Two-parent families with young children (under 6 years of age) | +"Two-parent families, both parents in labor force with young children (under 6 years of age) | " +"Two-parent families, father only in labor force with young children (under 6 years of age) | " +"Two-parent families, mother only in labor force with young children (under 6 years of age) | " +"Two-parent families, neither parent in labor force with young children (under 6 years of age) | " +One-parent families with young children (under 6 years of age) | +"One-parent families, father, with young children (under 6 years of age) | " +"Men age 45 to 64 (""middle aged"") | 0" +Men age 45 to 49 | +Men age 50 to 54 | +Men age 55 to 59 | +Men age 60 to 61 | +Men age 62 to 64 | +Black Men age 45 to 54 | +Black Men age 55 to 64 | +Hispanic Men age 45 to 54 | +Hispanic Men age 55 to 64 | +White Men age 45 to 54 | +White Men age 55 to 64 | +Asian Men age 45 to 54 | +Asian Men age 55 to 64 | +Men age 45 to 64 who attained less than a 9th grade education | +"Men age 45 to 64 who attained between 9th and 12th grade, no diploma | " +Men age 45 to 64 who completed high school or obtained GED | +"Men age 45 to 64 who completed some college, no degree | " +Men age 45 to 64 who obtained an associate's degree | +Men age 45 to 64 who obtained a bachelor's degree | +Men age 45 to 64 who obtained a graduate or professional degree | +"One-parent families, father in labor force, with young children (under 6 years of age) | " +Population 15 Years and Over | The number of people in a geographic area who are over the age of 15. This is used mostly as a denominator of marital status. +Never Married | The number of people in a geographic area who have never been married. +Currently married | The number of people in a geographic area who are currently married +Married but separated | The number of people in a geographic area who are married but separated +Widowed | The number of people in a geographic area who are widowed +Divorced | The number of people in a geographic area who are divorced +Workers age 16 and over who do not work from home | The number of workers over the age of 16 who do not work from home in a geographic area +Number of workers with less than 10 minute commute | The number of workers over the age of 16 who do not work from home and commute in less than 10 minutes in a geographic area +Number of workers with a commute between 10 and 14 minutes | The number of workers over the age of 16 who do not work from home and commute in between 10 and 14 minutes in a geographic area +Number of workers with a commute between 15 and 19 minutes | The number of workers over the age of 16 who do not work from home and commute in between 15 and 19 minutes in a geographic area +Number of workers with a commute between 20 and 24 minutes | The number of workers over the age of 16 who do not work from home and commute in between 20 and 24 minutes in a geographic area +Number of workers with a commute between 25 and 29 minutes | The number of workers over the age of 16 who do not work from home and commute in between 25 and 29 minutes in a geographic area +Number of workers with a commute between 30 and 34 minutes | The number of workers over the age of 16 who do not work from home and commute in between 30 and 34 minutes in a geographic area +Number of workers with a commute between 35 and 44 minutes | The number of workers over the age of 16 who do not work from home and commute in between 35 and 44 minutes in a geographic area +Number of workers with a commute between 45 and 59 minutes | The number of workers over the age of 16 who do not work from home and commute in between 45 and 59 minutes in a geographic area +Number of workers with a commute of over 60 minutes | The number of workers over the age of 16 who do not work from home and commute in over 60 minutes in a geographic area +Aggregate travel time to work | The total number of minutes every worker over the age of 16 who did not work from home spent spent commuting to work in one day in a geographic area +"Households with income less than $10,000 | The number of households in a geographic area whose annual income was less than $10,000." +"Households with income of $10,000 to $14,999 | The number of households in a geographic area whose annual income was between $10,000 and $14,999." +"Households with income of $15,000 to $19,999 | The number of households in a geographic area whose annual income was between $15,000 and $19,999." +"Households with income of $20,000 To $24,999 | The number of households in a geographic area whose annual income was between $20,000 and $24,999." +"Households with income of $25,000 To $29,999 | The number of households in a geographic area whose annual income was between $20,000 and $24,999." +"Households with income of $30,000 To $34,999 | The number of households in a geographic area whose annual income was between $30,000 and $34,999." +"Households with income of $35,000 To $39,999 | The number of households in a geographic area whose annual income was between $35,000 and $39,999." +"Households with income of $40,000 To $44,999 | The number of households in a geographic area whose annual income was between $40,000 and $44,999." +"Households with income of $45,000 To $49,999 | The number of households in a geographic area whose annual income was between $45,000 and $49,999." +"Households with income of $50,000 To $59,999 | The number of households in a geographic area whose annual income was between $50,000 and $59,999." +"Households with income of $60,000 To $74,999 | The number of households in a geographic area whose annual income was between $60,000 and $74,999." +"Households with income of $75,000 To $99,999 | The number of households in a geographic area whose annual income was between $75,000 and $99,999." +"Households with income of $100,000 To $124,999 | The number of households in a geographic area whose annual income was between $100,000 and $124,999." +"Households with income of $125,000 To $149,999 | The number of households in a geographic area whose annual income was between $125,000 and $149,999." +"Households with income of $150,000 To $199,999 | The number of households in a geographic area whose annual income was between $150,000 and $1999,999." +Population age 16 and over | The number of people in each geography who are age 16 or over. +"Population in Labor Force | The number of people in each geography who are either in the civilian labor force or are members of the U.S. Armed Forces (people on active duty with the United States Army, Air Force, Navy, Marine Corps, or Coast Guard)." +"Population in Civilian Labor Force | The number of civilians 16 years and over in each geography who can be classified as either ""employed"" or ""unemployed"" below." +"Employed Population | The number of civilians 16 years old and over in each geography who either (1) were ""at work,"" that is, those who did any work at all during the reference week as paid employees, worked in their own business or profession, worked on their own farm, or worked 15 hours or more as unpaid workers on a family farm or in a family business; or (2) were ""with a job but not at work,"" that is, those who did not work during the reference week but had jobs or businesses from which they were temporarily absent due to illness, bad weather, industrial dispute, vacation, or other personal reasons. Excluded from the employed are people whose only activity consisted of work around the house or unpaid volunteer work for religious, charitable, and similar organizations; also excluded are all institutionalized people and people on active duty in the United States Armed Forces." +"Unemployed Population | The number of civilians in each geography who are 16 years old and over are classified as unemployed if they (1) were neither ""at work"" nor ""with a job but not at work"" during the reference week, and (2) were actively looking for work during the last 4 weeks, and (3) were available to start a job. Also included as unemployed are civilians who did not work at all during the reference week, were waiting to be called back to a job from which they had been laid off, and were available for work except for temporary illness. Examples of job seeking activities are: + * Registering at a public or private employment office + * Meeting with prospective employers + * Investigating possibilities for starting a professional + practice or opening a business + * Placing or answering advertisements + * Writing letters of application + * Being on a union or professional register" +"Population in Armed Forces | The number of people in each geography who are members of the U.S. Armed Forces (people on active duty with the United States Army, Air Force, Navy, Marine Corps, or Coast Guard)." +"Population Not in Labor Force | The number of people in each geography who are 16 years old and over who are not classified as members of the labor force. This category consists mainly of students, homemakers, retired workers, seasonal workers interviewed in an off season who were not looking for work, institutionalized people, and people doing only incidental unpaid family work (less than 15 hours during the reference week)." +"Households with income of $200,000 Or More | The number of households in a geographic area whose annual income was more than $200,000." From 7f88e8c53a7317d1065d4f2f027c66c0066abec1 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 15:42:44 -0400 Subject: [PATCH 07/60] better table... --- doc/methods.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/methods.md b/doc/methods.md index ae5f36e..39f87ac 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -183,6 +183,7 @@ Should add the SQL API call here too Below is a list of human readable names accepted in the ```OBS_GetUSCensusMeasure``` method. For the more comprehensive list of columns available to the ```OBS_GetMeasure``` method, see the [Data Catalog] Measure name | Measure description +--- | --- Total Population | The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates. Male Population | The number of people within each geography who are male. Female Population | The number of people within each geography who are female. From 0374a64a8bbb8e239ca2bb3f81af8722e98d5b09 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 15:43:25 -0400 Subject: [PATCH 08/60] remove quotes --- doc/methods.md | 136 ++++++++++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/doc/methods.md b/doc/methods.md index 39f87ac..c61ac38 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -115,13 +115,13 @@ type | the data type (text, number, boolean) Add a Measure to an empty column based on point locations in your table ```SQL -UPDATE tablename SET local_male_population = OBS_GetMeasure(the_geom, '"us.census.acs".B08134006') -> 'value' +UPDATE tablename SET local_male_population = OBS_GetMeasure(the_geom, 'us.census.acs.B08134006') -> 'value' ``` Get a measure at a single point location ```SQL -SELECT * FROM json_each(OBS_GetMeasure(CDB_LatLng(40.7, -73.9), '"us.census.acs".B08134006')) +SELECT * FROM json_each(OBS_GetMeasure(CDB_LatLng(40.7, -73.9), 'us.census.acs.B08134006')) ``` + + # Discovery # Glossary +#### Boundary IDs + +Boundary name | Boundary ID +--------------------- | --- +US Census Block Groups | "us.census.tiger".block_group +US Census Tracts | "us.census.tiger".census_tract +US States | "us.census.tiger".state +US County | "us.census.tiger".county +US Census Public Use Microdata Areas | "us.census.tiger".puma +US Census Zip Code Tabulation Areas | "us.census.tiger".zcta5 +Unified School District | "us.census.tiger".school_district_unified +US Congressional Districts | "us.census.tiger".congressional_district +Elementary School District | "us.census.tiger".school_district_elementary +Secondary School District | "us.census.tiger".school_district_secondary +US Census Blocks | "us.census.tiger".block + #### OBS_GetUSCensusMeasure names table Below is a list of human readable names accepted in the ```OBS_GetUSCensusMeasure``` method. For the more comprehensive list of columns available to the ```OBS_GetMeasure``` method, see the [Data Catalog] From e90a0c19000b25ff48d533e80d7eedc9ca5fad7f Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:03:14 -0400 Subject: [PATCH 12/60] tautolotest --- src/pg/test/expected/41_observatory_augmentation_test.out | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 2df454c..948b4b3 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -91,14 +91,14 @@ Done. {X10} | (1 row) - obs_getmeasure + obs_getmeasure ---------------------------------------------------- {"name" : "total_pop", "value" : 9516.27915900609} (1 row) - obs_getmeasure ----------------------------------------------------- - {"name" : "total_pop", "value" : 12140.2484103572} + obs_getmeasure +---------------------------------------- + {"name" : "total_pop", "value" : 1655} (1 row) Dropping obs_table.sql fixture table... From 1cff19127fac7f6f014a19afb60da5959fac3f0d Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 16:10:45 -0400 Subject: [PATCH 13/60] stub in placeholders --- doc/methods.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/methods.md b/doc/methods.md index 035b202..e9747a5 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -206,9 +206,20 @@ UPDATE tablename SET the_geom = OBS_GetGeometry(the_geom, ' "us.census.tiger".bl Should add the SQL API call here too --> +## OBS_GetGeometryId(point_geometry, boundary_id ) + + +## OBS_GetGeometryById( Boundary Layer ID, Boundary ID ) + # Discovery +## OBS_Search(Search Term, Relevant Boundary) + + +## OBS_GetAvailableBoundaries(Relevant Geometry) + + # Glossary #### Boundary IDs From 83119d0720bcbe80f05601c86b360e189fc2628b Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:12:23 -0400 Subject: [PATCH 14/60] fix expectation typo --- ...y_exploration_test.out => 42_observatory_exploration_test.out} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/pg/test/expected/{42_observatroy_exploration_test.out => 42_observatory_exploration_test.out} (100%) diff --git a/src/pg/test/expected/42_observatroy_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out similarity index 100% rename from src/pg/test/expected/42_observatroy_exploration_test.out rename to src/pg/test/expected/42_observatory_exploration_test.out From d016c4b4bd3cfe13623bdac5850c7e09f3c036b6 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:15:39 -0400 Subject: [PATCH 15/60] remove typo in sql def --- src/pg/sql/42_observatory_exploration.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index a6063fa..e929052 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -3,7 +3,7 @@ -- TODO allow the user to specify the boundary to search for measures -- -CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search_STU( +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search( search_term text, relevant_boundary text DEFAULT null ) From 1f6347aa23d03e2e3d7530a47b6fb5007d200e87 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:18:20 -0400 Subject: [PATCH 16/60] adjusting expectations to reflect reality of the situation --- src/pg/test/expected/42_observatory_exploration_test.out | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index b95c1f4..9f84d39 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -21,9 +21,9 @@ Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... Done. Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... Done. -obs_search_stu + obs_search ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -("The total number of all people living in a geographic area.","Total Population",sum,es.ine) -("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Total Population",sum,us.census.acs) -("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Quantile:Total Population",quantile,us.census.acs) + ("The total number of all people living in a geographic area.","Total Population",sum,es.ine) + ("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Total Population",sum,us.census.acs) + ("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Quantile:Total Population",quantile,us.census.acs) (3 rows) From 73f4a682cf336cc3476835e2c261f483ae47a15c Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:22:13 -0400 Subject: [PATCH 17/60] fix whitespace and testpoint --- src/pg/test/expected/42_observatory_exploration_test.out | 2 +- src/pg/test/sql/42_observatory_exploration_test.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index 9f84d39..3191605 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -21,7 +21,7 @@ Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... Done. Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... Done. - obs_search + obs_search ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ("The total number of all people living in a geographic area.","Total Population",sum,es.ine) ("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Total Population",sum,us.census.acs) diff --git a/src/pg/test/sql/42_observatory_exploration_test.sql b/src/pg/test/sql/42_observatory_exploration_test.sql index f9173ee..4a08e87 100644 --- a/src/pg/test/sql/42_observatory_exploration_test.sql +++ b/src/pg/test/sql/42_observatory_exploration_test.sql @@ -2,4 +2,4 @@ SELECT cdb_observatory.OBS_Search('total_pop'); -SELECT * from cdb_observatory.OBS_GetAvailableBoundaries(_test_point()); +SELECT * from cdb_observatory.OBS_GetAvailableBoundaries(cdb_observatory._test_point()); From 34631d8a52de79133d7dc2f92b16d7af42562578 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 16:23:17 -0400 Subject: [PATCH 18/60] add getgeomid doc --- doc/methods.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/doc/methods.md b/doc/methods.md index e9747a5..5eb9e71 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -206,10 +206,34 @@ UPDATE tablename SET the_geom = OBS_GetGeometry(the_geom, ' "us.census.tiger".bl Should add the SQL API call here too --> -## OBS_GetGeometryId(point_geometry, boundary_id ) +## OBS_GetGeometryId(point_geometry, boundary_id) +The ```OBS_GetGeometryId(point_geometry, boundary_id)``` returns a unique geometry_id for the boundary geometry that contains a given point geometry. See the [Boundary ID glossary table below](below). The method can be combined with ```OBS_GetGeometryById(geometry_id)``` to create a point aggregation workflow. -## OBS_GetGeometryById( Boundary Layer ID, Boundary ID ) +#### Arguments + +Name |Description +--- | --- +point_geometry | a WGS84 polygon geometry (the_geom) +boundary_id | a boundary identifier from the [Boundary ID glossary table below](below) + +#### Returns + +A JSON object containing the following properties + +Key | Description +--- | --- +geometry_id | a string identifier of a geometry in the Boundaries + +#### Example + +Write the geometry_id that contains the point geometry for every row as a new column in your table + +```SQL +UPDATE tablename SET new_column_name = OBS_GetGeometryId(the_geom, ' "us.census.tiger".block_group') +``` + +## OBS_GetGeometryById(geometry_id) # Discovery From 594c6b5d8068da2d62ef68b0427ba88843294264 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:23:57 -0400 Subject: [PATCH 19/60] test_point => TestPoint --- src/pg/test/sql/42_observatory_exploration_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/test/sql/42_observatory_exploration_test.sql b/src/pg/test/sql/42_observatory_exploration_test.sql index 4a08e87..479caf3 100644 --- a/src/pg/test/sql/42_observatory_exploration_test.sql +++ b/src/pg/test/sql/42_observatory_exploration_test.sql @@ -2,4 +2,4 @@ SELECT cdb_observatory.OBS_Search('total_pop'); -SELECT * from cdb_observatory.OBS_GetAvailableBoundaries(cdb_observatory._test_point()); +SELECT * from cdb_observatory.OBS_GetAvailableBoundaries(cdb_observatory._TestPoint()); From 0ab1cba06af9c7ff449a2f1a007939a16efbe1af Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 16:28:06 -0400 Subject: [PATCH 20/60] add getgeombyid docs --- doc/methods.md | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/doc/methods.md b/doc/methods.md index 5eb9e71..4c517ed 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -181,16 +181,14 @@ The ```OBS_GetGeometry(point_geometry, boundary_id)``` method returns a boundary #### Arguments -Name |Description +Name | Description --- | --- point_geometry | a WGS84 polygon geometry (the_geom) boundary_id | a boundary identifier from the [Boundary ID glossary table below](below) #### Returns -A JSON object containing the following properties - -Key | Description +Value | Description --- | --- geom | WKB geometry @@ -219,9 +217,7 @@ boundary_id | a boundary identifier from the [Boundary ID glossary table below]( #### Returns -A JSON object containing the following properties - -Key | Description +Value | Description --- | --- geometry_id | a string identifier of a geometry in the Boundaries @@ -235,6 +231,29 @@ UPDATE tablename SET new_column_name = OBS_GetGeometryId(the_geom, ' "us.census. ## OBS_GetGeometryById(geometry_id) +The ```OBS_GetGeometryById(geometry_id)``` returns the boundary geometry for a unique geometry_id. A geometry_id can be found using the ```OBS_GetGeometryId(point_geometry, boundary_id)``` method described above. + +#### Arguments + +Name |Description +--- | --- +geometry_id | a string identifier for a Boundary geometry + +#### Returns + +A JSON object containing the following properties + +Key | Description +--- | --- +geom | a WGS84 polygon geometry + +#### Example + +Use a table of geometry_id to select the unique boundaries. Useful with the ```Table from query``` option in CartoDB. + +```SQL +SELECT OBS_GetGeometryById(geometry_id) the_geom, geometry_id FROM tablename GROUP BY geometry_id +``` # Discovery From 9f44caaa62edb6cee3ab215103d07f5fb9d73db7 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 16:37:11 -0400 Subject: [PATCH 21/60] added obssearch doc --- doc/methods.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/methods.md b/doc/methods.md index 4c517ed..1b188e8 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -257,7 +257,31 @@ SELECT OBS_GetGeometryById(geometry_id) the_geom, geometry_id FROM tablename GRO # Discovery -## OBS_Search(Search Term, Relevant Boundary) +## OBS_Search(search_term) + + +#### Arguments + +Name | Description +--- | --- +search_term | a string to search for available Measures +boundary_id | a string identifier for a Boundary geometry (optional) + +#### Returns + +Key | Description +--- | --- +measure_id | the unique id of the measue for use with the ```OBS_GetMeasure``` method +name | the human readable name of the measure +description | a brief description of the measure +aggregate_type | **sum** are raw count values, **median** are statistical medians, **average** are statistical averages, **undefined** other (e.g. an index value) +sources | where the data came from (e.g. US Census Bureau) + +#### Example + +```SQL +SELECT * FROM OBS_Search('inequality') +``` ## OBS_GetAvailableBoundaries(Relevant Geometry) From 346f94438329fe51b0c2b9b56ad0b2ab6a9b7d78 Mon Sep 17 00:00:00 2001 From: andrewxhill Date: Wed, 20 Apr 2016 16:45:19 -0400 Subject: [PATCH 22/60] added getavailboundaries --- doc/methods.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/methods.md b/doc/methods.md index 1b188e8..8d52e44 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -259,6 +259,7 @@ SELECT OBS_GetGeometryById(geometry_id) the_geom, geometry_id FROM tablename GRO ## OBS_Search(search_term) +Use arbitrary text to search all available Measures #### Arguments @@ -283,9 +284,29 @@ sources | where the data came from (e.g. US Census Bureau) SELECT * FROM OBS_Search('inequality') ``` +## OBS_GetAvailableBoundaries(point_geometry) -## OBS_GetAvailableBoundaries(Relevant Geometry) +Returns available boundary_ids at a given point geometry. +#### Arguments + +Name | Description +--- | --- +point_geometry | a WGS84 point geometry (e.g. the_geom) + +#### Returns + +Key | Description +--- | --- +boundary_id | a boundary identifier from the [Boundary ID glossary table below](below) +description | a brief description of the boundary dataset +timespan | the timespan attached the boundary. this does not mean that the boundary is invalid outside of the timespan, but is the explicit timespan published with the geometry. + +#### Example + +```SQL +SELECT * FROM OBS_GetAvailableBoundaries(CDB_LatLng(40.7, -73.9)) +``` # Glossary From 6696d4fcf5ae4351394313d503f94ab1fc2e3359 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:45:24 -0400 Subject: [PATCH 23/60] adding id to return for obs_search, fixing OBS_GetAvailableBoundaries --- src/pg/sql/42_observatory_exploration.sql | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index e929052..0460237 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -7,7 +7,7 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search( search_term text, relevant_boundary text DEFAULT null ) -RETURNS TABLE(description text, name text, aggregate text, source text) as $$ +RETURNS TABLE(id text, description text, name text, aggregate text, source text) as $$ DECLARE boundary_term text; BEGIN @@ -19,7 +19,7 @@ BEGIN RETURN QUERY EXECUTE format($string$ - SELECT description, + SELECT id, description, name, aggregate, replace(split_part(id,'".', 1),'"', '') source @@ -40,7 +40,7 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION OBS_GetAvailableBoundaries( geom geometry, time_span text DEFAULT null) -RETURNS TABLE(description text, name text, id text) as $$ +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) as $$ DECLARE timespan_query TEXT DEFAULT ''; BEGIN @@ -53,15 +53,17 @@ BEGIN EXECUTE $string$ select - column_id, + column_id boundary_id, obs_column.description, - table_id + timespan time_span, + tablename FROM observatory.OBS_table, observatory.OBS_column_table, observatory.OBS_column WHERE observatory.OBS_column_table.column_id = observatory.obs_column.id + observatory.OBS_column_table.table_id = observatory.obs_table.id AND observatory.OBS_column.type='Geometry' AND From a6655003aaeb884ff6cec1efb59a240c55969469 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:47:40 -0400 Subject: [PATCH 24/60] fix duplicate var name issue --- src/pg/sql/42_observatory_exploration.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index 0460237..ea87b21 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -53,9 +53,9 @@ BEGIN EXECUTE $string$ select - column_id boundary_id, + column_id, obs_column.description, - timespan time_span, + timespan, tablename FROM observatory.OBS_table, From 851419a927769d04d91d71c28ff99dce913b6963 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:49:41 -0400 Subject: [PATCH 25/60] fix duplicate var name issue --- src/pg/sql/42_observatory_exploration.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index ea87b21..8a72777 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -39,7 +39,7 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION OBS_GetAvailableBoundaries( geom geometry, - time_span text DEFAULT null) + timespan text DEFAULT null) RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) as $$ DECLARE timespan_query TEXT DEFAULT ''; From 39e9ff32e56a32c3176ea41630841dd56a240ae0 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 16:57:07 -0400 Subject: [PATCH 26/60] missing "and" --- src/pg/sql/42_observatory_exploration.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index 8a72777..b40a0ae 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -62,7 +62,7 @@ BEGIN observatory.OBS_column_table, observatory.OBS_column WHERE - observatory.OBS_column_table.column_id = observatory.obs_column.id + observatory.OBS_column_table.column_id = observatory.obs_column.id AND observatory.OBS_column_table.table_id = observatory.obs_table.id AND observatory.OBS_column.type='Geometry' From 30c1ed08c0e879110431daeafbc41a8ac6a176ed Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 17:05:29 -0400 Subject: [PATCH 27/60] adjusting expectations --- .../42_observatory_exploration_test.out | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index 3191605..6dcf77f 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -21,9 +21,47 @@ Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... Done. Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... Done. - obs_search ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - ("The total number of all people living in a geographic area.","Total Population",sum,es.ine) - ("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Total Population",sum,us.census.acs) - ("The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Quantile:Total Population",quantile,us.census.acs) + obs_search +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + ("""es.ine"".total_pop","The total number of all people living in a geographic area.","Total Population",sum,es.ine) + ("""us.census.acs"".B01001001","The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Total Population",sum,us.census.acs) + ("""us.census.acs"".B01001001_quantile","The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Quantile:Total Population",quantile,us.census.acs) (3 rows) + + boundary_id | description | time_span | tablename +--------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+---------------------------------------------- + "us.census.tiger".block_group | Block groups (BGs) are statistical divisions of census tracts, are generally defined to contain between 600 and 3,000 people, and are used to present data and control block numbering. A block group consists of clusters of blocks within the same census tract that have the same first digit of their four-digit census block number. For example, blocks 3001, 3002, 3003, ..., 3999 in census tract 1210.02 belong to BG 3 in that census tract. Most BGs were delineated by local participants in the Census Bureau\u2019s Participant Statistical Areas Program. The Census Bureau delineated BGs only where a local or tribal government declined to participate, and a regional organization or State Data Center was not available to participate.\r +| 2013 | obs_85328201013baa14e8e8a4a57a01e6f6fbc5f9b1 + | \r +| | + | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | + "us.census.tiger".census_tract | Census tracts are identified by an up to four-digit integer number and may have an optional two-digit suffix; for example 1457.02 or 23. The census tract codes consist of six digits with an implied decimal between the fourth and fifth digit corresponding to the basic census tract number but with leading zeroes and trailing zeroes for census tracts without a suffix. The tract number examples above would have codes of 145702 and 002300, respectively.\r +| 2013 | obs_a92e1111ad3177676471d66bb8036e6d057f271b + | \r +| | + | Some ranges of census tract numbers in the 2010 Census are used to identify distinctive types of census tracts. The code range in the 9400s is used for those census tracts with a majority of population, housing, or land area associated with an American Indian area and matches the numbering used in Census 2000. The code range in the 9800s is new for 2010 and is used to specifically identify special land-use census tracts; that is, census tracts defined to encompass a large area with little or no residential population with special characteristics, such as large parks or employment areas. The range of census tracts in the 9900s represents census tracts delineated specifically to cover large bodies of water. This is different from Census 2000 when water-only census tracts were assigned codes of all zeroes (000000); 000000 is no longer used as a census tract code for the 2010 Census.\r +| | + | \r +| | + | The Census Bureau uses suffixes to help identify census tract changes for comparison purposes. Census tract suffixes may range from .01 to .98. As part of local review of existing census tracts before each census, some census tracts may have grown enough in population size to qualify as more than one census tract. When a census tract is split, the split parts usually retain the basic number but receive different suffixes. For example, if census tract 14 is split, the new tract numbers would be 14.01 and 14.02. In a few counties, local participants request major changes to, and renumbering of, the census tracts; however, this is generally discouraged. Changes to individual census tract boundaries usually do not result in census tract numbering changes.\r +| | + | \r +| | + | The Census Bureau introduced the concept of tribal census tracts for the first time for Census 2000. Tribal census tracts for that census consisted of the standard county-based census tracts tabulated within American Indian areas, thus allowing for the tracts to ignore state and county boundaries for tabulation. The Census Bureau assigned the 9400 range of numbers to identify specific tribal census tracts; however, not all tribal census tracts used this numbering scheme. For the 2010 Census, tribal census tracts no longer are tied to or numbered in the same way as the county-based census tracts (see \u201cTribal Census Tract\u201d). | | + "us.census.tiger".state | States and Equivalent Entities are the primary governmental divisions of the United States. In addition to the 50 states, the Census Bureau treats the District of Columbia, Puerto Rico, American Samoa, the Commonwealth of the Northern Mariana Islands, Guam, and the U.S. Virgin Islands as the statistical equivalents of states for the purpose of data presentation. | 2013 | obs_f3f0912fe24bc0c976e837b5a116d0c803cc01ce + "us.census.tiger".puma | PUMAs are geographic areas for which the Census Bureau provides selected extracts of raw data from a small sample of census records that are screened to protect confidentiality. These extracts are referred to as public use microdata sample (PUMS) files.\r +| 2013 | obs_0008b162b516c295d7204c9ba043ab5dbc67c59c + | \r +| | + | For the 2010 Census, each state, the District of Columbia, Puerto Rico, and some Island Area participants delineated PUMAs for use in presenting PUMS data based on a 5 percent sample of decennial census or American Community Survey data. These areas are required to contain at least 100,000 people. This is different from Census 2000 when two types of PUMAs were defined: a 5 percent PUMA as for 2010 and an additional super-PUMA designed to provide a 1 percent sample. The PUMAs are identified by a five-digit census code unique within state. | | + "us.census.tiger".zcta5 | ZCTAs are approximate area representations of U.S. Postal Service (USPS) five-digit ZIP Code service areas that the Census Bureau creates using whole blocks to present statistical data from censuses and surveys. The Census Bureau defines ZCTAs by allocating each block that contains addresses to a single ZCTA, usually to the ZCTA that reflects the most frequently occurring ZIP Code for the addresses within that tabulation block. Blocks that do not contain addresses but are completely surrounded by a single ZCTA (enclaves) are assigned to the surrounding ZCTA; those surrounded by multiple ZCTAs will be added to a single ZCTA based on limited buffering performed between multiple ZCTAs. The Census Bureau identifies five-digit ZCTAs using a five-character numeric code that represents the most frequently occurring USPS ZIP Code within that ZCTA, and this code may contain leading zeros.\r +| 2013 | obs_d483723c5cc76c107d9e0af279d1e7056df3c2be + | \r +| | + | There are significant changes to the 2010 ZCTA delineation from that used in 2000. Coverage was extended to include the Island Areas for 2010 so that the United States, Puerto Rico, and the Island Areas have ZCTAs. Unlike 2000, when areas that could not be assigned to a ZCTA were given a generic code ending in \u201cXX\u201d (land area) or \u201cHH\u201d (water area), for 2010 there is no universal coverage by ZCTAs, and only legitimate five-digit areas are defined. The 2010 ZCTAs will better represent the actual Zip Code service areas because the Census Bureau initiated a process before creation of 2010 blocks to add block boundaries that split polygons with large numbers of addresses using different Zip Codes.\r +| | + | \r +| | + | Data users should not use ZCTAs to identify the official USPS ZIP Code for mail delivery. The USPS makes periodic changes to ZIP Codes to support more efficient mail delivery. The ZCTAs process used primarily residential addresses and was biased towards Zip Codes used for city-style mail delivery, thus there may be Zip Codes that are primarily nonresidential or boxes only that may not have a corresponding ZCTA. | | + "us.census.tiger".county | The primary legal divisions of most states are termed counties. In Louisiana, these divisions are known as parishes. In Alaska, which has no counties, the equivalent entities are the organized boroughs, city and boroughs, municipalities, and census areas; the latter of which are delineated cooperatively for statistical purposes by the state of Alaska and the Census Bureau. In four states (Maryland, Missouri, Nevada, and Virginia), there are one or more incorporated places that are independent of any county organization and thus constitute primary divisions of their states. These incorporated places are known as independent cities and are treated as equivalent entities for purposes of data presentation. The District of Columbia and Guam have no primary divisions, and each area is considered an equivalent entity for purposes of data presentation. All of the counties in Connecticut and Rhode Island and nine counties in Massachusetts were dissolved as functioning governmental entities; however, the Census Bureau continues to present data for these historical entities in order to provide comparable geographic units at the county level of the geographic hierarchy for these states and represents them as nonfunctioning legal entities in data products. The Census Bureau treats the following entities as equivalents of counties for purposes of data presentation: municipios in Puerto Rico, districts and islands in American Samoa, municipalities in the Commonwealth of the Northern Mariana Islands, and islands in the U.S. Virgin Islands. Each county or statistically equivalent entity is assigned a three-character numeric Federal Information Processing Series (FIPS) code based on alphabetical sequence that is unique within state and an eight-digit National Standard feature identifier. | 2013 | obs_b0ef6dd68d5faddbf231fd7f02916b3d00ec43c4 + "us.census.tiger".state | States and Equivalent Entities are the primary governmental divisions of the United States. In addition to the 50 states, the Census Bureau treats the District of Columbia, Puerto Rico, American Samoa, the Commonwealth of the Northern Mariana Islands, Guam, and the U.S. Virgin Islands as the statistical equivalents of states for the purpose of data presentation. | 2013 | obs_a20f5260b618a2fe2eb95fc1e23febe0db7db096 + "us.census.tiger".county | The primary legal divisions of most states are termed counties. In Louisiana, these divisions are known as parishes. In Alaska, which has no counties, the equivalent entities are the organized boroughs, city and boroughs, municipalities, and census areas; the latter of which are delineated cooperatively for statistical purposes by the state of Alaska and the Census Bureau. In four states (Maryland, Missouri, Nevada, and Virginia), there are one or more incorporated places that are independent of any county organization and thus constitute primary divisions of their states. These incorporated places are known as independent cities and are treated as equivalent entities for purposes of data presentation. The District of Columbia and Guam have no primary divisions, and each area is considered an equivalent entity for purposes of data presentation. All of the counties in Connecticut and Rhode Island and nine counties in Massachusetts were dissolved as functioning governmental entities; however, the Census Bureau continues to present data for these historical entities in order to provide comparable geographic units at the county level of the geographic hierarchy for these states and represents them as nonfunctioning legal entities in data products. The Census Bureau treats the following entities as equivalents of counties for purposes of data presentation: municipios in Puerto Rico, districts and islands in American Samoa, municipalities in the Commonwealth of the Northern Mariana Islands, and islands in the U.S. Virgin Islands. Each county or statistically equivalent entity is assigned a three-character numeric Federal Information Processing Series (FIPS) code based on alphabetical sequence that is unique within state and an eight-digit National Standard feature identifier. | 2013 | obs_23da37d4e66e9de2f525572967f8618bde99a8c0 + "us.census.tiger".census_tract | Census tracts are identified by an up to four-digit integer number and may have an optional two-digit suffix; for example 1457.02 or 23. The census tract codes consist of six digits with an implied decimal between the fourth and fifth digit corresponding to the basic census tract number but with leading zeroes and trailing zeroes for census tracts without a suffix. The tract number examples above would have codes of 145702 and 002300, respectively.\r +| 2013 | obs_d125aeef87aaa23287a40b454519ece22ee25acf + | \r +| | + | Some ranges of census tract numbers in the 2010 Census are used to identify distinctive types of census tracts. The code range in the 9400s is used for those census tracts with a majority of population, housing, or land area associated with an American Indian area and matches the numbering used in Census 2000. The code range in the 9800s is new for 2010 and is used to specifically identify special land-use census tracts; that is, census tracts defined to encompass a large area with little or no residential population with special characteristics, such as large parks or employment areas. The range of census tracts in the 9900s represents census tracts delineated specifically to cover large bodies of water. This is different from Census 2000 when water-only census tracts were assigned codes of all zeroes (000000); 000000 is no longer used as a census tract code for the 2010 Census.\r +| | + | \r +| | + | The Census Bureau uses suffixes to help identify census tract changes for comparison purposes. Census tract suffixes may range from .01 to .98. As part of local review of existing census tracts before each census, some census tracts may have grown enough in population size to qualify as more than one census tract. When a census tract is split, the split parts usually retain the basic number but receive different suffixes. For example, if census tract 14 is split, the new tract numbers would be 14.01 and 14.02. In a few counties, local participants request major changes to, and renumbering of, the census tracts; however, this is generally discouraged. Changes to individual census tract boundaries usually do not result in census tract numbering changes.\r +| | + | \r +| | + | The Census Bureau introduced the concept of tribal census tracts for the first time for Census 2000. Tribal census tracts for that census consisted of the standard county-based census tracts tabulated within American Indian areas, thus allowing for the tracts to ignore state and county boundaries for tabulation. The Census Bureau assigned the 9400 range of numbers to identify specific tribal census tracts; however, not all tribal census tracts used this numbering scheme. For the 2010 Census, tribal census tracts no longer are tied to or numbered in the same way as the county-based census tracts (see \u201cTribal Census Tract\u201d). | | + "us.census.tiger".block_group | Block groups (BGs) are statistical divisions of census tracts, are generally defined to contain between 600 and 3,000 people, and are used to present data and control block numbering. A block group consists of clusters of blocks within the same census tract that have the same first digit of their four-digit census block number. For example, blocks 3001, 3002, 3003, ..., 3999 in census tract 1210.02 belong to BG 3 in that census tract. Most BGs were delineated by local participants in the Census Bureau\u2019s Participant Statistical Areas Program. The Census Bureau delineated BGs only where a local or tribal government declined to participate, and a regional organization or State Data Center was not available to participate.\r +| 2013 | obs_d610cb3225f282693b8d4dcd98d2c2e2078354c6 + | \r +| | + | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | +(10 rows) + + From 1702a392b387995c5eca902bfcffc3813f31c6dc Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 17:08:56 -0400 Subject: [PATCH 28/60] add trailing whitespace --- .../42_observatory_exploration_test.out | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index 6dcf77f..35065fe 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -31,37 +31,37 @@ Done. boundary_id | description | time_span | tablename --------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+---------------------------------------------- "us.census.tiger".block_group | Block groups (BGs) are statistical divisions of census tracts, are generally defined to contain between 600 and 3,000 people, and are used to present data and control block numbering. A block group consists of clusters of blocks within the same census tract that have the same first digit of their four-digit census block number. For example, blocks 3001, 3002, 3003, ..., 3999 in census tract 1210.02 belong to BG 3 in that census tract. Most BGs were delineated by local participants in the Census Bureau\u2019s Participant Statistical Areas Program. The Census Bureau delineated BGs only where a local or tribal government declined to participate, and a regional organization or State Data Center was not available to participate.\r +| 2013 | obs_85328201013baa14e8e8a4a57a01e6f6fbc5f9b1 - | \r +| | - | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | + | \r +| | + | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | "us.census.tiger".census_tract | Census tracts are identified by an up to four-digit integer number and may have an optional two-digit suffix; for example 1457.02 or 23. The census tract codes consist of six digits with an implied decimal between the fourth and fifth digit corresponding to the basic census tract number but with leading zeroes and trailing zeroes for census tracts without a suffix. The tract number examples above would have codes of 145702 and 002300, respectively.\r +| 2013 | obs_a92e1111ad3177676471d66bb8036e6d057f271b - | \r +| | - | Some ranges of census tract numbers in the 2010 Census are used to identify distinctive types of census tracts. The code range in the 9400s is used for those census tracts with a majority of population, housing, or land area associated with an American Indian area and matches the numbering used in Census 2000. The code range in the 9800s is new for 2010 and is used to specifically identify special land-use census tracts; that is, census tracts defined to encompass a large area with little or no residential population with special characteristics, such as large parks or employment areas. The range of census tracts in the 9900s represents census tracts delineated specifically to cover large bodies of water. This is different from Census 2000 when water-only census tracts were assigned codes of all zeroes (000000); 000000 is no longer used as a census tract code for the 2010 Census.\r +| | - | \r +| | - | The Census Bureau uses suffixes to help identify census tract changes for comparison purposes. Census tract suffixes may range from .01 to .98. As part of local review of existing census tracts before each census, some census tracts may have grown enough in population size to qualify as more than one census tract. When a census tract is split, the split parts usually retain the basic number but receive different suffixes. For example, if census tract 14 is split, the new tract numbers would be 14.01 and 14.02. In a few counties, local participants request major changes to, and renumbering of, the census tracts; however, this is generally discouraged. Changes to individual census tract boundaries usually do not result in census tract numbering changes.\r +| | - | \r +| | - | The Census Bureau introduced the concept of tribal census tracts for the first time for Census 2000. Tribal census tracts for that census consisted of the standard county-based census tracts tabulated within American Indian areas, thus allowing for the tracts to ignore state and county boundaries for tabulation. The Census Bureau assigned the 9400 range of numbers to identify specific tribal census tracts; however, not all tribal census tracts used this numbering scheme. For the 2010 Census, tribal census tracts no longer are tied to or numbered in the same way as the county-based census tracts (see \u201cTribal Census Tract\u201d). | | + | \r +| | + | Some ranges of census tract numbers in the 2010 Census are used to identify distinctive types of census tracts. The code range in the 9400s is used for those census tracts with a majority of population, housing, or land area associated with an American Indian area and matches the numbering used in Census 2000. The code range in the 9800s is new for 2010 and is used to specifically identify special land-use census tracts; that is, census tracts defined to encompass a large area with little or no residential population with special characteristics, such as large parks or employment areas. The range of census tracts in the 9900s represents census tracts delineated specifically to cover large bodies of water. This is different from Census 2000 when water-only census tracts were assigned codes of all zeroes (000000); 000000 is no longer used as a census tract code for the 2010 Census.\r +| | + | \r +| | + | The Census Bureau uses suffixes to help identify census tract changes for comparison purposes. Census tract suffixes may range from .01 to .98. As part of local review of existing census tracts before each census, some census tracts may have grown enough in population size to qualify as more than one census tract. When a census tract is split, the split parts usually retain the basic number but receive different suffixes. For example, if census tract 14 is split, the new tract numbers would be 14.01 and 14.02. In a few counties, local participants request major changes to, and renumbering of, the census tracts; however, this is generally discouraged. Changes to individual census tract boundaries usually do not result in census tract numbering changes.\r +| | + | \r +| | + | The Census Bureau introduced the concept of tribal census tracts for the first time for Census 2000. Tribal census tracts for that census consisted of the standard county-based census tracts tabulated within American Indian areas, thus allowing for the tracts to ignore state and county boundaries for tabulation. The Census Bureau assigned the 9400 range of numbers to identify specific tribal census tracts; however, not all tribal census tracts used this numbering scheme. For the 2010 Census, tribal census tracts no longer are tied to or numbered in the same way as the county-based census tracts (see \u201cTribal Census Tract\u201d). | | "us.census.tiger".state | States and Equivalent Entities are the primary governmental divisions of the United States. In addition to the 50 states, the Census Bureau treats the District of Columbia, Puerto Rico, American Samoa, the Commonwealth of the Northern Mariana Islands, Guam, and the U.S. Virgin Islands as the statistical equivalents of states for the purpose of data presentation. | 2013 | obs_f3f0912fe24bc0c976e837b5a116d0c803cc01ce "us.census.tiger".puma | PUMAs are geographic areas for which the Census Bureau provides selected extracts of raw data from a small sample of census records that are screened to protect confidentiality. These extracts are referred to as public use microdata sample (PUMS) files.\r +| 2013 | obs_0008b162b516c295d7204c9ba043ab5dbc67c59c - | \r +| | - | For the 2010 Census, each state, the District of Columbia, Puerto Rico, and some Island Area participants delineated PUMAs for use in presenting PUMS data based on a 5 percent sample of decennial census or American Community Survey data. These areas are required to contain at least 100,000 people. This is different from Census 2000 when two types of PUMAs were defined: a 5 percent PUMA as for 2010 and an additional super-PUMA designed to provide a 1 percent sample. The PUMAs are identified by a five-digit census code unique within state. | | + | \r +| | + | For the 2010 Census, each state, the District of Columbia, Puerto Rico, and some Island Area participants delineated PUMAs for use in presenting PUMS data based on a 5 percent sample of decennial census or American Community Survey data. These areas are required to contain at least 100,000 people. This is different from Census 2000 when two types of PUMAs were defined: a 5 percent PUMA as for 2010 and an additional super-PUMA designed to provide a 1 percent sample. The PUMAs are identified by a five-digit census code unique within state. | | "us.census.tiger".zcta5 | ZCTAs are approximate area representations of U.S. Postal Service (USPS) five-digit ZIP Code service areas that the Census Bureau creates using whole blocks to present statistical data from censuses and surveys. The Census Bureau defines ZCTAs by allocating each block that contains addresses to a single ZCTA, usually to the ZCTA that reflects the most frequently occurring ZIP Code for the addresses within that tabulation block. Blocks that do not contain addresses but are completely surrounded by a single ZCTA (enclaves) are assigned to the surrounding ZCTA; those surrounded by multiple ZCTAs will be added to a single ZCTA based on limited buffering performed between multiple ZCTAs. The Census Bureau identifies five-digit ZCTAs using a five-character numeric code that represents the most frequently occurring USPS ZIP Code within that ZCTA, and this code may contain leading zeros.\r +| 2013 | obs_d483723c5cc76c107d9e0af279d1e7056df3c2be - | \r +| | - | There are significant changes to the 2010 ZCTA delineation from that used in 2000. Coverage was extended to include the Island Areas for 2010 so that the United States, Puerto Rico, and the Island Areas have ZCTAs. Unlike 2000, when areas that could not be assigned to a ZCTA were given a generic code ending in \u201cXX\u201d (land area) or \u201cHH\u201d (water area), for 2010 there is no universal coverage by ZCTAs, and only legitimate five-digit areas are defined. The 2010 ZCTAs will better represent the actual Zip Code service areas because the Census Bureau initiated a process before creation of 2010 blocks to add block boundaries that split polygons with large numbers of addresses using different Zip Codes.\r +| | - | \r +| | - | Data users should not use ZCTAs to identify the official USPS ZIP Code for mail delivery. The USPS makes periodic changes to ZIP Codes to support more efficient mail delivery. The ZCTAs process used primarily residential addresses and was biased towards Zip Codes used for city-style mail delivery, thus there may be Zip Codes that are primarily nonresidential or boxes only that may not have a corresponding ZCTA. | | + | \r +| | + | There are significant changes to the 2010 ZCTA delineation from that used in 2000. Coverage was extended to include the Island Areas for 2010 so that the United States, Puerto Rico, and the Island Areas have ZCTAs. Unlike 2000, when areas that could not be assigned to a ZCTA were given a generic code ending in \u201cXX\u201d (land area) or \u201cHH\u201d (water area), for 2010 there is no universal coverage by ZCTAs, and only legitimate five-digit areas are defined. The 2010 ZCTAs will better represent the actual Zip Code service areas because the Census Bureau initiated a process before creation of 2010 blocks to add block boundaries that split polygons with large numbers of addresses using different Zip Codes.\r +| | + | \r +| | + | Data users should not use ZCTAs to identify the official USPS ZIP Code for mail delivery. The USPS makes periodic changes to ZIP Codes to support more efficient mail delivery. The ZCTAs process used primarily residential addresses and was biased towards Zip Codes used for city-style mail delivery, thus there may be Zip Codes that are primarily nonresidential or boxes only that may not have a corresponding ZCTA. | | "us.census.tiger".county | The primary legal divisions of most states are termed counties. In Louisiana, these divisions are known as parishes. In Alaska, which has no counties, the equivalent entities are the organized boroughs, city and boroughs, municipalities, and census areas; the latter of which are delineated cooperatively for statistical purposes by the state of Alaska and the Census Bureau. In four states (Maryland, Missouri, Nevada, and Virginia), there are one or more incorporated places that are independent of any county organization and thus constitute primary divisions of their states. These incorporated places are known as independent cities and are treated as equivalent entities for purposes of data presentation. The District of Columbia and Guam have no primary divisions, and each area is considered an equivalent entity for purposes of data presentation. All of the counties in Connecticut and Rhode Island and nine counties in Massachusetts were dissolved as functioning governmental entities; however, the Census Bureau continues to present data for these historical entities in order to provide comparable geographic units at the county level of the geographic hierarchy for these states and represents them as nonfunctioning legal entities in data products. The Census Bureau treats the following entities as equivalents of counties for purposes of data presentation: municipios in Puerto Rico, districts and islands in American Samoa, municipalities in the Commonwealth of the Northern Mariana Islands, and islands in the U.S. Virgin Islands. Each county or statistically equivalent entity is assigned a three-character numeric Federal Information Processing Series (FIPS) code based on alphabetical sequence that is unique within state and an eight-digit National Standard feature identifier. | 2013 | obs_b0ef6dd68d5faddbf231fd7f02916b3d00ec43c4 "us.census.tiger".state | States and Equivalent Entities are the primary governmental divisions of the United States. In addition to the 50 states, the Census Bureau treats the District of Columbia, Puerto Rico, American Samoa, the Commonwealth of the Northern Mariana Islands, Guam, and the U.S. Virgin Islands as the statistical equivalents of states for the purpose of data presentation. | 2013 | obs_a20f5260b618a2fe2eb95fc1e23febe0db7db096 "us.census.tiger".county | The primary legal divisions of most states are termed counties. In Louisiana, these divisions are known as parishes. In Alaska, which has no counties, the equivalent entities are the organized boroughs, city and boroughs, municipalities, and census areas; the latter of which are delineated cooperatively for statistical purposes by the state of Alaska and the Census Bureau. In four states (Maryland, Missouri, Nevada, and Virginia), there are one or more incorporated places that are independent of any county organization and thus constitute primary divisions of their states. These incorporated places are known as independent cities and are treated as equivalent entities for purposes of data presentation. The District of Columbia and Guam have no primary divisions, and each area is considered an equivalent entity for purposes of data presentation. All of the counties in Connecticut and Rhode Island and nine counties in Massachusetts were dissolved as functioning governmental entities; however, the Census Bureau continues to present data for these historical entities in order to provide comparable geographic units at the county level of the geographic hierarchy for these states and represents them as nonfunctioning legal entities in data products. The Census Bureau treats the following entities as equivalents of counties for purposes of data presentation: municipios in Puerto Rico, districts and islands in American Samoa, municipalities in the Commonwealth of the Northern Mariana Islands, and islands in the U.S. Virgin Islands. Each county or statistically equivalent entity is assigned a three-character numeric Federal Information Processing Series (FIPS) code based on alphabetical sequence that is unique within state and an eight-digit National Standard feature identifier. | 2013 | obs_23da37d4e66e9de2f525572967f8618bde99a8c0 "us.census.tiger".census_tract | Census tracts are identified by an up to four-digit integer number and may have an optional two-digit suffix; for example 1457.02 or 23. The census tract codes consist of six digits with an implied decimal between the fourth and fifth digit corresponding to the basic census tract number but with leading zeroes and trailing zeroes for census tracts without a suffix. The tract number examples above would have codes of 145702 and 002300, respectively.\r +| 2013 | obs_d125aeef87aaa23287a40b454519ece22ee25acf - | \r +| | - | Some ranges of census tract numbers in the 2010 Census are used to identify distinctive types of census tracts. The code range in the 9400s is used for those census tracts with a majority of population, housing, or land area associated with an American Indian area and matches the numbering used in Census 2000. The code range in the 9800s is new for 2010 and is used to specifically identify special land-use census tracts; that is, census tracts defined to encompass a large area with little or no residential population with special characteristics, such as large parks or employment areas. The range of census tracts in the 9900s represents census tracts delineated specifically to cover large bodies of water. This is different from Census 2000 when water-only census tracts were assigned codes of all zeroes (000000); 000000 is no longer used as a census tract code for the 2010 Census.\r +| | - | \r +| | - | The Census Bureau uses suffixes to help identify census tract changes for comparison purposes. Census tract suffixes may range from .01 to .98. As part of local review of existing census tracts before each census, some census tracts may have grown enough in population size to qualify as more than one census tract. When a census tract is split, the split parts usually retain the basic number but receive different suffixes. For example, if census tract 14 is split, the new tract numbers would be 14.01 and 14.02. In a few counties, local participants request major changes to, and renumbering of, the census tracts; however, this is generally discouraged. Changes to individual census tract boundaries usually do not result in census tract numbering changes.\r +| | - | \r +| | - | The Census Bureau introduced the concept of tribal census tracts for the first time for Census 2000. Tribal census tracts for that census consisted of the standard county-based census tracts tabulated within American Indian areas, thus allowing for the tracts to ignore state and county boundaries for tabulation. The Census Bureau assigned the 9400 range of numbers to identify specific tribal census tracts; however, not all tribal census tracts used this numbering scheme. For the 2010 Census, tribal census tracts no longer are tied to or numbered in the same way as the county-based census tracts (see \u201cTribal Census Tract\u201d). | | + | \r +| | + | Some ranges of census tract numbers in the 2010 Census are used to identify distinctive types of census tracts. The code range in the 9400s is used for those census tracts with a majority of population, housing, or land area associated with an American Indian area and matches the numbering used in Census 2000. The code range in the 9800s is new for 2010 and is used to specifically identify special land-use census tracts; that is, census tracts defined to encompass a large area with little or no residential population with special characteristics, such as large parks or employment areas. The range of census tracts in the 9900s represents census tracts delineated specifically to cover large bodies of water. This is different from Census 2000 when water-only census tracts were assigned codes of all zeroes (000000); 000000 is no longer used as a census tract code for the 2010 Census.\r +| | + | \r +| | + | The Census Bureau uses suffixes to help identify census tract changes for comparison purposes. Census tract suffixes may range from .01 to .98. As part of local review of existing census tracts before each census, some census tracts may have grown enough in population size to qualify as more than one census tract. When a census tract is split, the split parts usually retain the basic number but receive different suffixes. For example, if census tract 14 is split, the new tract numbers would be 14.01 and 14.02. In a few counties, local participants request major changes to, and renumbering of, the census tracts; however, this is generally discouraged. Changes to individual census tract boundaries usually do not result in census tract numbering changes.\r +| | + | \r +| | + | The Census Bureau introduced the concept of tribal census tracts for the first time for Census 2000. Tribal census tracts for that census consisted of the standard county-based census tracts tabulated within American Indian areas, thus allowing for the tracts to ignore state and county boundaries for tabulation. The Census Bureau assigned the 9400 range of numbers to identify specific tribal census tracts; however, not all tribal census tracts used this numbering scheme. For the 2010 Census, tribal census tracts no longer are tied to or numbered in the same way as the county-based census tracts (see \u201cTribal Census Tract\u201d). | | "us.census.tiger".block_group | Block groups (BGs) are statistical divisions of census tracts, are generally defined to contain between 600 and 3,000 people, and are used to present data and control block numbering. A block group consists of clusters of blocks within the same census tract that have the same first digit of their four-digit census block number. For example, blocks 3001, 3002, 3003, ..., 3999 in census tract 1210.02 belong to BG 3 in that census tract. Most BGs were delineated by local participants in the Census Bureau\u2019s Participant Statistical Areas Program. The Census Bureau delineated BGs only where a local or tribal government declined to participate, and a regional organization or State Data Center was not available to participate.\r +| 2013 | obs_d610cb3225f282693b8d4dcd98d2c2e2078354c6 - | \r +| | - | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | + | \r +| | + | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | (10 rows) From 75f4e6d412965ccb3f3993e6583f2f2383e063e1 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 20 Apr 2016 17:11:12 -0400 Subject: [PATCH 29/60] add trailing whitespace --- src/pg/test/expected/42_observatory_exploration_test.out | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index 35065fe..9db640a 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -28,7 +28,7 @@ Done. ("""us.census.acs"".B01001001_quantile","The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","Quantile:Total Population",quantile,us.census.acs) (3 rows) - boundary_id | description | time_span | tablename + boundary_id | description | time_span | tablename --------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------+---------------------------------------------- "us.census.tiger".block_group | Block groups (BGs) are statistical divisions of census tracts, are generally defined to contain between 600 and 3,000 people, and are used to present data and control block numbering. A block group consists of clusters of blocks within the same census tract that have the same first digit of their four-digit census block number. For example, blocks 3001, 3002, 3003, ..., 3999 in census tract 1210.02 belong to BG 3 in that census tract. Most BGs were delineated by local participants in the Census Bureau\u2019s Participant Statistical Areas Program. The Census Bureau delineated BGs only where a local or tribal government declined to participate, and a regional organization or State Data Center was not available to participate.\r +| 2013 | obs_85328201013baa14e8e8a4a57a01e6f6fbc5f9b1 | \r +| | @@ -64,4 +64,3 @@ Done. | A BG usually covers a contiguous area. Each census tract contains at least one BG, and BGs are uniquely numbered within the census tract. Within the standard census geographic hierarchy, BGs never cross state, county, or census tract boundaries but may cross the boundaries of any other geographic entity. Tribal census tracts and tribal BGs are separate and unique geographic areas defined within federally recognized American Indian reservations and can cross state and county boundaries (see \u201cTribal Census Tract\u201d and \u201cTribal Block Group\u201d). The tribal census tracts and tribal block groups may be completely different from the census tracts and block groups defined by state and county. | | (10 rows) - From c10da16d55e74d6baede7990814438d10e7e2a68 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 20 Apr 2016 17:24:07 -0400 Subject: [PATCH 30/60] adding _OBS_GetRelatedColumn function --- src/pg/sql/40_observatory_utility.sql | 22 ++++++++++ .../expected/40_observatory_utility_test.out | 41 +++++++++++++------ .../test/sql/40_observatory_utility_test.sql | 10 +++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index d1b1f3b..22948a7 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -151,3 +151,25 @@ BEGIN END; $$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_observatory._GetRelatedColumn(columns_ids text[], reltype text ) +RETURNS TEXT[] +AS $$ +DECLARE + result TEXT[]; +BEGIN + EXECUTE ' + With ids as ( + select row_number() over() as no, id from (select unnest($1) as id) t + ) + select array_agg(target_id order by no) + FROM ids + LEFT JOIN observatory.obs_column_to_column + on source_id = id + where reltype = $2 or reltype is null + ' + INTO result + using columns_ids, reltype; + return result; +END; +$$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index a0dca1f..9b54718 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -29,7 +29,7 @@ SELECT CDB_LatLng(40.7128,-74.0059), '"us.census.tiger".census_tract' ); - _obs_geomtable + _obs_geomtable ---------------------------------------------- obs_a92e1111ad3177676471d66bb8036e6d057f271b (1 row) @@ -41,9 +41,9 @@ SELECT CDB_LatLng(40.7128,-74.0059), '"us.census.tiger".nonexistant_id' -- not in catalog ); - _obs_geomtable + _obs_geomtable ---------------- - + (1 row) -- future test: give back nulls when geometry doesn't intersect @@ -65,11 +65,11 @@ SELECT '2009 - 2013' ))).* ORDER BY colname, tablename ASC; - colname | tablename | aggregate + colname | tablename | aggregate -----------+----------------------------------------------+----------- - geoid | obs_11ee8b82c877c073438bc935a91d3dfccef875d1 | - geoid | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | - geoid | obs_d34555209878e8c4b37cf0b2b3d072ff129ec470 | + geoid | obs_11ee8b82c877c073438bc935a91d3dfccef875d1 | + geoid | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | + geoid | obs_d34555209878e8c4b37cf0b2b3d072ff129ec470 | total_pop | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | sum (4 rows) @@ -81,7 +81,7 @@ SELECT '2009 - 2013' ))).* ORDER BY 1 ASC; - colname | tablename | aggregate + colname | tablename | aggregate ---------+-----------+----------- (0 rows) @@ -91,7 +91,7 @@ SELECT cdb_observatory._OBS_LookupCensusHuman( Array['gini_index'] ); - _obs_lookupcensushuman + _obs_lookupcensushuman --------------------------------- {"\"us.census.acs\".B19083001"} (1 row) @@ -101,9 +101,9 @@ SELECT cdb_observatory._OBS_LookupCensusHuman( Array['cookies'] ); - _obs_lookupcensushuman + _obs_lookupcensushuman ------------------------ - + (1 row) -- OBS_BuildSnapshotQuery @@ -112,7 +112,7 @@ SELECT cdb_observatory._OBS_BuildSnapshotQuery( Array['total_pop','male_pop','female_pop','median_age'] ); - _obs_buildsnapshotquery + _obs_buildsnapshotquery ------------------------------------------------------------------------------------------------- SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age (1 row) @@ -122,11 +122,26 @@ SELECT cdb_observatory._OBS_BuildSnapshotQuery( Array['mandarin_orange'] ); - _obs_buildsnapshotquery + _obs_buildsnapshotquery ------------------------------------ SELECT vals[1] As mandarin_orange (1 row) +-- should give back "\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} +SELECT + select * from cdb_observatory._GetRelatedColumn( + Array[ + '"es.ine".pop_0_4', + '"us.census.acs".B01001001', + '"us.census.acs".B01001002' + ], + 'denominator' + ); + _getrelatedcolumn +------------------------------------------------------------- +{"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} +(1 row) + \i test/sql/drop_fixtures.sql SET client_min_messages TO NOTICE; \set ECHO none diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 9067f8e..677c369 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -75,4 +75,14 @@ SELECT Array['mandarin_orange'] ); +SELECT + select * from cdb_observatory._GetRelatedColumn( + Array[ + '"es.ine".pop_0_4', + '"us.census.acs".B01001001', + '"us.census.acs".B01001002' + ], + 'denominator' + ); + \i test/sql/drop_fixtures.sql From 6684c67e6168d21057b539f637bc1d8291c1bf01 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 21 Apr 2016 12:14:30 -0400 Subject: [PATCH 31/60] updating function name --- src/pg/sql/40_observatory_utility.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 22948a7..6eb22c1 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -152,7 +152,7 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION cdb_observatory._GetRelatedColumn(columns_ids text[], reltype text ) +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetRelatedColumn(columns_ids text[], reltype text ) RETURNS TEXT[] AS $$ DECLARE From b8148768b8282501ff06252c472d053bd6b68767 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 21 Apr 2016 12:22:20 -0400 Subject: [PATCH 32/60] fix whitespace and bug in test call --- .../expected/40_observatory_utility_test.out | 20 +++++++++---------- .../test/sql/40_observatory_utility_test.sql | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 9b54718..04978c3 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -29,7 +29,7 @@ SELECT CDB_LatLng(40.7128,-74.0059), '"us.census.tiger".census_tract' ); - _obs_geomtable + _obs_geomtable ---------------------------------------------- obs_a92e1111ad3177676471d66bb8036e6d057f271b (1 row) @@ -41,7 +41,7 @@ SELECT CDB_LatLng(40.7128,-74.0059), '"us.census.tiger".nonexistant_id' -- not in catalog ); - _obs_geomtable + _obs_geomtable ---------------- (1 row) @@ -54,7 +54,7 @@ SELECT -- ); -- OBS_GetColumnData -- should give back: --- colname | tablename | aggregate +-- colname | tablename | aggregate -- -----------|-----------------|----------- -- geoid | obs_{hex table} | null -- total_pop | obs_{hex table} | sum @@ -65,7 +65,7 @@ SELECT '2009 - 2013' ))).* ORDER BY colname, tablename ASC; - colname | tablename | aggregate + colname | tablename | aggregate -----------+----------------------------------------------+----------- geoid | obs_11ee8b82c877c073438bc935a91d3dfccef875d1 | geoid | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | @@ -81,7 +81,7 @@ SELECT '2009 - 2013' ))).* ORDER BY 1 ASC; - colname | tablename | aggregate + colname | tablename | aggregate ---------+-----------+----------- (0 rows) @@ -91,7 +91,7 @@ SELECT cdb_observatory._OBS_LookupCensusHuman( Array['gini_index'] ); - _obs_lookupcensushuman + _obs_lookupcensushuman --------------------------------- {"\"us.census.acs\".B19083001"} (1 row) @@ -101,7 +101,7 @@ SELECT cdb_observatory._OBS_LookupCensusHuman( Array['cookies'] ); - _obs_lookupcensushuman + _obs_lookupcensushuman ------------------------ (1 row) @@ -112,7 +112,7 @@ SELECT cdb_observatory._OBS_BuildSnapshotQuery( Array['total_pop','male_pop','female_pop','median_age'] ); - _obs_buildsnapshotquery + _obs_buildsnapshotquery ------------------------------------------------------------------------------------------------- SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age (1 row) @@ -122,7 +122,7 @@ SELECT cdb_observatory._OBS_BuildSnapshotQuery( Array['mandarin_orange'] ); - _obs_buildsnapshotquery + _obs_buildsnapshotquery ------------------------------------ SELECT vals[1] As mandarin_orange (1 row) @@ -137,7 +137,7 @@ SELECT ], 'denominator' ); - _getrelatedcolumn + _getrelatedcolumn ------------------------------------------------------------- {"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} (1 row) diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 677c369..6fbcb45 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -76,7 +76,7 @@ SELECT ); SELECT - select * from cdb_observatory._GetRelatedColumn( + select cdb_observatory._OBS_GetRelatedColumn( Array[ '"es.ine".pop_0_4', '"us.census.acs".B01001001', From 90a944070dac417e72cc799cdc23a6c678e2f886 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 21 Apr 2016 12:26:55 -0400 Subject: [PATCH 33/60] more test and whitespace fixes --- src/pg/test/expected/40_observatory_utility_test.out | 12 ++++++------ src/pg/test/sql/40_observatory_utility_test.sql | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 04978c3..f70d64f 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -43,7 +43,7 @@ SELECT ); _obs_geomtable ---------------- - + (1 row) -- future test: give back nulls when geometry doesn't intersect @@ -54,7 +54,7 @@ SELECT -- ); -- OBS_GetColumnData -- should give back: --- colname | tablename | aggregate +-- colname | tablename | aggregate -- -----------|-----------------|----------- -- geoid | obs_{hex table} | null -- total_pop | obs_{hex table} | sum @@ -67,9 +67,9 @@ SELECT ORDER BY colname, tablename ASC; colname | tablename | aggregate -----------+----------------------------------------------+----------- - geoid | obs_11ee8b82c877c073438bc935a91d3dfccef875d1 | - geoid | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | - geoid | obs_d34555209878e8c4b37cf0b2b3d072ff129ec470 | + geoid | obs_11ee8b82c877c073438bc935a91d3dfccef875d1 | + geoid | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | + geoid | obs_d34555209878e8c4b37cf0b2b3d072ff129ec470 | total_pop | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | sum (4 rows) @@ -103,7 +103,7 @@ SELECT ); _obs_lookupcensushuman ------------------------ - + (1 row) -- OBS_BuildSnapshotQuery diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 6fbcb45..37f0648 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -75,8 +75,7 @@ SELECT Array['mandarin_orange'] ); -SELECT - select cdb_observatory._OBS_GetRelatedColumn( +SELECT cdb_observatory._OBS_GetRelatedColumn( Array[ '"es.ine".pop_0_4', '"us.census.acs".B01001001', From afd1be30b52e822151a5b91aba7736f310c6ca28 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 21 Apr 2016 12:30:50 -0400 Subject: [PATCH 34/60] fixing whitespace --- src/pg/test/expected/40_observatory_utility_test.out | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index f70d64f..5fb3a2f 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -127,9 +127,7 @@ SELECT SELECT vals[1] As mandarin_orange (1 row) --- should give back "\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} -SELECT - select * from cdb_observatory._GetRelatedColumn( +SELECT cdb_observatory._OBS_GetRelatedColumn( Array[ '"es.ine".pop_0_4', '"us.census.acs".B01001001', @@ -137,9 +135,9 @@ SELECT ], 'denominator' ); - _getrelatedcolumn + _obs_getrelatedcolumn ------------------------------------------------------------- -{"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} + {"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} (1 row) \i test/sql/drop_fixtures.sql From 6fd8a43e44e67b48699476340c3191a04a9864dc Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 21 Apr 2016 14:17:31 -0400 Subject: [PATCH 35/60] adding standardizemeasurename --- src/pg/sql/40_observatory_utility.sql | 20 +++++++++++++++++++ .../expected/40_observatory_utility_test.out | 5 +++++ .../test/sql/40_observatory_utility_test.sql | 3 +++ 3 files changed, 28 insertions(+) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 6eb22c1..69ea27c 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -173,3 +173,23 @@ BEGIN return result; END; $$ LANGUAGE plpgsql; + +-- Function that replaces all non digits or letters with _ trims and lowercases the +-- passed measure name + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_StandardizeMeasureName(measure_name text) +RETURNS text +AS $$ +DECLARE + result text; +BEGIN + -- Turn non letter or digits to _ + result = regexp_replace(measure_name, '[^\dA-Za-z]+','_', 'g'); + -- Remove duplicate _'s + result = regexp_replace(result,'_{2,}','_', 'g'); + -- Trim _'s from beginning and end + result = trim(both '_' from result); + result = lower(result); + RETURN result; +END; +$$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 5fb3a2f..309f35e 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -140,6 +140,11 @@ SELECT cdb_observatory._OBS_GetRelatedColumn( {"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} (1 row) + _obs_standardizemeasurename +----------------------------- + test_343_2_qqq +(1 row) + \i test/sql/drop_fixtures.sql SET client_min_messages TO NOTICE; \set ECHO none diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 37f0648..9fc7d63 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -84,4 +84,7 @@ SELECT cdb_observatory._OBS_GetRelatedColumn( 'denominator' ); +-- should give back a standardized measure name +SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}'); + \i test/sql/drop_fixtures.sql From 54110795532396d7b09ffcfad10ecc1117cbecf6 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 21 Apr 2016 14:20:16 -0400 Subject: [PATCH 36/60] fix expectations --- src/pg/test/expected/40_observatory_utility_test.out | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 309f35e..05b78a5 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -140,6 +140,8 @@ SELECT cdb_observatory._OBS_GetRelatedColumn( {"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} (1 row) +-- should give back a standardized measure name +SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}'); _obs_standardizemeasurename ----------------------------- test_343_2_qqq From 00d43a78120b47b0dbc93cff1d021bf5ca6e7445 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 21 Apr 2016 15:16:04 -0400 Subject: [PATCH 37/60] Updating OBS_ColumnData to support more info --- src/pg/sql/40_observatory_utility.sql | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 6eb22c1..429c385 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -34,7 +34,12 @@ END; $$ LANGUAGE plpgsql; -- A type for use with the OBS_GetColumnData function -CREATE TYPE cdb_observatory.OBS_ColumnData AS (colname text, tablename text, aggregate text); +CREATE TYPE cdb_observatory.OBS_ColumnData AS ( + colname text, + tablename text, + aggregate text, + name text, + type text); -- A function that gets the column data for multiple columns @@ -60,7 +65,7 @@ BEGIN column_ids as ( select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a ) - SELECT array_agg(ROW(colname, tablename, aggregate)::cdb_observatory.OBS_ColumnData order by column_ids.no) + SELECT array_agg(ROW(colname, tablename, aggregate, name, type)::cdb_observatory.OBS_ColumnData order by column_ids.no) FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t WHERE column_ids.column_id = c.id AND c.id = ct.column_id From 1e625ce3efff2dc6f7034c0ddc1a167b0bc79596 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 21 Apr 2016 15:18:39 -0400 Subject: [PATCH 38/60] Updated version of OBS_Get to support json returning --- src/pg/sql/41_observatory_augmentation.sql | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index f5b9c8b..7966bb8 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -269,10 +269,10 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_Get( time_span text, geometry_level text ) -RETURNS TABLE(names text[], vals NUMERIC[]) +RETURNS SETOF JSON AS $$ DECLARE - results NUMERIC[]; + results numeric[]; geom_table_name text; names text[]; query text; @@ -290,10 +290,6 @@ BEGIN data_table_info := cdb_observatory._OBS_GetColumnData(geometry_level, column_ids, time_span); - - names := (SELECT array_agg((d).colname) - FROM unnest(data_table_info) As d); - IF ST_GeometryType(geom) = 'ST_Point' THEN results := cdb_observatory._OBS_GetPoints(geom, @@ -311,8 +307,19 @@ BEGIN THEN results := Array[]::numeric[]; END IF; - - RETURN QUERY SELECT names, results; + + RAISE NOTICE '%', results; + + RETURN QUERY + SELECT row_to_json(d) + FROM ( + SELECT (meta).aggregate as aggregate, + (meta).name as name, + (meta).type as type, + val as value + FROM (select unnest(data_table_info) as meta, + unnest(results) as val) b + ) d; END; $$ LANGUAGE plpgsql; From 7659ededaab337c554885dcadc0479501061065f Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 14:43:56 -0400 Subject: [PATCH 39/60] Changing OBS_GetColumnData to return json and more metadata --- src/pg/sql/40_observatory_utility.sql | 37 +++++++++++++-------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 429c385..4314e17 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -33,13 +33,6 @@ BEGIN END; $$ LANGUAGE plpgsql; --- A type for use with the OBS_GetColumnData function -CREATE TYPE cdb_observatory.OBS_ColumnData AS ( - colname text, - tablename text, - aggregate text, - name text, - type text); -- A function that gets the column data for multiple columns @@ -49,11 +42,10 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData( column_ids text[], timespan text ) -RETURNS cdb_observatory.OBS_ColumnData[] +RETURNS SETOF JSON AS $$ -DECLARE - result cdb_observatory.OBS_ColumnData[]; BEGIN + RETURN QUERY EXECUTE ' WITH geomref AS ( SELECT t.table_id id @@ -65,17 +57,24 @@ BEGIN column_ids as ( select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a ) - SELECT array_agg(ROW(colname, tablename, aggregate, name, type)::cdb_observatory.OBS_ColumnData order by column_ids.no) - FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t - WHERE column_ids.column_id = c.id - AND c.id = ct.column_id - AND t.id = ct.table_id - AND t.timespan = $3 - AND t.id in (SELECT id FROM geomref) + SELECT row_to_json(a) from ( + select colname, + tablename, + aggregate, + name, + type, + c.description + FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t + WHERE column_ids.column_id = c.id + AND c.id = ct.column_id + AND t.id = ct.table_id + AND t.timespan = $3 + AND t.id in (SELECT id FROM geomref) + order by column_ids.no + ) a ' USING geometry_id, column_ids, timespan - INTO result; - RETURN result; + RETURN; END; $$ LANGUAGE plpgsql; From 3394483a45119be5498479db946aef628fb651ed Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 14:45:22 -0400 Subject: [PATCH 40/60] migrating OBS_GET, OBS_GetPoints, OBS_GetPolygons and OBS_GetMeasure to all use json internals --- src/pg/sql/41_observatory_augmentation.sql | 123 +++++++++++++-------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 7966bb8..cdf707c 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -272,11 +272,11 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_Get( RETURNS SETOF JSON AS $$ DECLARE - results numeric[]; + results json[]; geom_table_name text; names text[]; query text; - data_table_info cdb_observatory.OBS_ColumnData[]; + data_table_info json[]; BEGIN geom_table_name := cdb_observatory._OBS_GeomTable(geom, geometry_level); @@ -287,9 +287,13 @@ BEGIN RETURN QUERY SELECT '{}'::text[], '{}'::NUMERIC[]; END IF; - data_table_info := cdb_observatory._OBS_GetColumnData(geometry_level, - column_ids, - time_span); + execute' + select array_agg( _obs_getcolumndata) from cdb_observatory._OBS_GetColumnData($1, + $2, + $3);' + INTO data_table_info + using geometry_level, column_ids, time_span; + IF ST_GeometryType(geom) = 'ST_Point' THEN results := cdb_observatory._OBS_GetPoints(geom, @@ -298,28 +302,19 @@ BEGIN ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + -- RAISE EXCEPTION 'polygons not supported for now'; results := cdb_observatory._OBS_GetPolygons(geom, geom_table_name, data_table_info); END IF; - - IF results IS NULL - THEN - results := Array[]::numeric[]; - END IF; - - RAISE NOTICE '%', results; RETURN QUERY - SELECT row_to_json(d) - FROM ( - SELECT (meta).aggregate as aggregate, - (meta).name as name, - (meta).type as type, - val as value - FROM (select unnest(data_table_info) as meta, - unnest(results) as val) b - ) d; + EXECUTE + $query$ + SELECT unnest($1) + $query$ + USING results; + END; $$ LANGUAGE plpgsql; @@ -329,12 +324,13 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetPoints( geom geometry, geom_table_name text, - data_table_info cdb_observatory.OBS_ColumnData[] + data_table_info json[] ) -RETURNS NUMERIC[] +RETURNS json[] AS $$ DECLARE result NUMERIC[]; + json_result json[]; query text; i int; geoid text; @@ -372,14 +368,14 @@ BEGIN THEN -- give back null values query := query || format('NULL::numeric '); - ELSIF ((data_table_info)[i]).aggregate != 'sum' + ELSIF ((data_table_info)[i])->>'aggregate' != 'sum' THEN -- give back full variable - query := query || format('%I ', ((data_table_info)[i]).colname); + query := query || format('%I ', ((data_table_info)[i])->>'colname'); ELSE -- give back variable normalized by area of geography query := query || format('%I/%s ', - ((data_table_info)[i]).colname, + ((data_table_info)[i])->>'colname', area); END IF; @@ -393,17 +389,32 @@ BEGIN FROM observatory.%I WHERE %I.geoid = %L ', - ((data_table_info)[1]).tablename, - ((data_table_info)[1]).tablename, + ((data_table_info)[1])->>'tablename', + ((data_table_info)[1])->>'tablename', geoid ); - + EXECUTE query INTO result USING geom; - - RETURN result; + + EXECUTE + $query$ + select array_agg(row_to_json(t)) from( + select values as value, + meta->>'name' as name, + meta->>'tablename' as tablename, + meta->>'aggregate' as aggregate, + meta->>'type' as type, + meta->>'description' as description + from (select unnest($1) as values, unnest($2) as meta) b + ) t + $query$ + INTO json_result + USING result, data_table_info; + + RETURN json_result; END; $$ LANGUAGE plpgsql; @@ -418,8 +429,7 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure( RETURNS JSON AS $$ DECLARE - names TEXT[]; - vals NUMERIC[]; + result json; BEGIN IF boundary_id IS NULL THEN @@ -432,14 +442,14 @@ BEGIN time_span := '2009 - 2013'; END IF; - EXECUTE ' - SELECT names, vals FROM cdb_observatory._OBS_Get($1, ARRAY[$2], $3, $4) LIMIT 1 + + EXECUTE ' + SELECT * FROM cdb_observatory._OBS_Get($1, ARRAY[$2], $3, $4) LIMIT 1 ' - INTO names, vals + INTO result USING geom, measure_id, time_span, boundary_id; - RETURN json_build_object('name', (names)[1], 'value', (vals)[1]); - + RETURN result; END; $$ LANGUAGE plpgsql; @@ -447,12 +457,13 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetPolygons( geom geometry, geom_table_name text, - data_table_info cdb_observatory.OBS_ColumnData[] + data_table_info json[] ) -RETURNS NUMERIC[] +RETURNS json[] AS $$ DECLARE - result NUMERIC[]; + result numeric[]; + json_result json[]; q_select text; q_sum text; q text; @@ -464,11 +475,11 @@ BEGIN FOR i IN 1..array_upper(data_table_info, 1) LOOP - q_select := q_select || format( '%I ', ((data_table_info)[i]).colname); + q_select := q_select || format( '%I ', ((data_table_info)[i])->>'colname'); - IF ((data_table_info)[i]).aggregate ='sum' + IF ((data_table_info)[i])->>'aggregate' ='sum' THEN - q_sum := q_sum || format('sum(overlap_fraction * COALESCE(%I, 0)) ',((data_table_info)[i]).colname,((data_table_info)[i]).colname); + q_sum := q_sum || format('sum(overlap_fraction * COALESCE(%I, 0)) ',((data_table_info)[i])->>'colname',((data_table_info)[i])->>'colname'); ELSE q_sum := q_sum || ' NULL::numeric '; END IF; @@ -492,17 +503,32 @@ BEGIN values As ( ', geom_table_name); - q := q || q_select || format('FROM observatory.%I ', ((data_table_info)[1].tablename)); + q := q || q_select || format('FROM observatory.%I ', ((data_table_info)[1]->>'tablename')); q := q || ' ) ' || q_sum || ' ]::numeric[] FROM _overlaps, values WHERE values.geoid = _overlaps.geoid'; - + EXECUTE q INTO result USING geom; - - RETURN result; + + EXECUTE + $query$ + select array_agg(row_to_json(t)) from( + select values as value, + meta->>'name' as name, + meta->>'tablename' as tablename, + meta->>'aggregate' as aggregate, + meta->>'type' as type, + meta->>'description' as description + from (select unnest($1) as values, unnest($2) as meta) b + ) t + $query$ + INTO json_result + USING result, data_table_info; + + RETURN json_result; END; $$ LANGUAGE plpgsql; @@ -730,4 +756,3 @@ BEGIN END; $$ LANGUAGE plpgsql; - From 69ac0d25f22ea212ad7f452d2e242f2efce29765 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 16:10:30 -0400 Subject: [PATCH 41/60] Adding tests for new json returning utility functions --- .../expected/40_observatory_utility_test.out | 61 ++++--------------- .../test/sql/40_observatory_utility_test.sql | 25 +++++--- 2 files changed, 29 insertions(+), 57 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 5fb3a2f..5cc88e7 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -46,44 +46,15 @@ SELECT (1 row) --- future test: give back nulls when geometry doesn't intersect --- SELECT --- cdb_observatory._OBS_GeomTable( --- CDB_LatLng(0,0), -- should give back null since it's in the ocean? --- '"us.census.tiger".census_tract' --- ); --- OBS_GetColumnData --- should give back: --- colname | tablename | aggregate --- -----------|-----------------|----------- --- geoid | obs_{hex table} | null --- total_pop | obs_{hex table} | sum -SELECT - (unnest(cdb_observatory._OBS_GetColumnData( - '"us.census.tiger".census_tract', - Array['"us.census.tiger".census_tract_geoid', '"us.census.acs".B01001001'], - '2009 - 2013' - ))).* -ORDER BY colname, tablename ASC; - colname | tablename | aggregate ------------+----------------------------------------------+----------- - geoid | obs_11ee8b82c877c073438bc935a91d3dfccef875d1 | - geoid | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | - geoid | obs_d34555209878e8c4b37cf0b2b3d072ff129ec470 | - total_pop | obs_ab038198aaab3f3cb055758638ee4de28ad70146 | sum -(4 rows) +test_get_obs_column_with_geoid_and_census_1 | test_get_obs_column_with_geoid_and_census_2 | test_get_obs_column_with_geoid_and_census_3 +---------------------------------------------+---------------------------------------------+--------------------------------------------- +t | t | t +(1 row) --- should be null-valued -SELECT - (unnest(cdb_observatory._OBS_GetColumnData( - '"us.census.tiger".census_tract', - Array['"us.census.tiger".baloney'], -- entry not in catalog - '2009 - 2013' - ))).* -ORDER BY 1 ASC; - colname | tablename | aggregate ----------+-----------+----------- -(0 rows) +obs_getcolumndatajson_missing_measure +--------------------------------------- +t +(1 row) -- OBS_LookupCensusHuman -- should give back: {"\"us.census.acs\".B19083001"} @@ -127,17 +98,11 @@ SELECT SELECT vals[1] As mandarin_orange (1 row) -SELECT cdb_observatory._OBS_GetRelatedColumn( - Array[ - '"es.ine".pop_0_4', - '"us.census.acs".B01001001', - '"us.census.acs".B01001002' - ], - 'denominator' - ); - _obs_getrelatedcolumn -------------------------------------------------------------- - {"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} +-- should give back a normalized name + SELECT cdb_observatory._OBS_NormalizeMeasureName('test 343 %% 2 qqq }}{{}}'); +_obs_normalizemeasurename +--------------------------- +test_343_2_qqq (1 row) \i test/sql/drop_fixtures.sql diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 37f0648..5fe5c40 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -32,22 +32,29 @@ SELECT -- -----------|-----------------|----------- -- geoid | obs_{hex table} | null -- total_pop | obs_{hex table} | sum +WITH result as ( SELECT - (unnest(cdb_observatory._OBS_GetColumnData( + array_agg(a) expected from cdb_observatory._OBS_GetColumnData( '"us.census.tiger".census_tract', Array['"us.census.tiger".census_tract_geoid', '"us.census.acs".B01001001'], - '2009 - 2013' - ))).* -ORDER BY colname, tablename ASC; + '2009 - 2013') a +) +select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, + (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2, + (expected)[3]::text = '{"colname":"geoid","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_3 +from result; + -- should be null-valued +WITH result as ( SELECT - (unnest(cdb_observatory._OBS_GetColumnData( + array_agg(a) expected from cdb_observatory._OBS_GetColumnData( '"us.census.tiger".census_tract', - Array['"us.census.tiger".baloney'], -- entry not in catalog - '2009 - 2013' - ))).* -ORDER BY 1 ASC; + Array['"us.census.tiger".baloney'], + '2009 - 2013') a +) +select expected is null as OBS_GetColumnDataJSON_missing_measure +from result; -- OBS_LookupCensusHuman -- should give back: {"\"us.census.acs\".B19083001"} From 7782bdeec23285e875b0d153dfd31bdcd22d5f59 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 16:10:41 -0400 Subject: [PATCH 42/60] adding tests fro new json returning get methods --- .../41_observatory_augmentation_test.out | 36 ++++++------- .../sql/41_observatory_augmentation_test.sql | 50 +++++++++++-------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 948b4b3..41bff69 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -41,34 +41,34 @@ Done. {female_pop,male_pop} | {} (1 row) - names | vals ---------------+---------- - {gini_index} | {0.3494} +obs_get_gini_index_at_test_point +---------------------------------- +t (1 row) - names | vals ---------------+------ - {gini_index} | {} +obs_get_gini_index_at_null_island +----------------------------------- +t (1 row) - _obs_getpoints --------------------- - {4809.33511352425} +obs_getpoints_for_test_point +------------------------------ +t (1 row) - _obs_getpoints ----------------- - +obs_getpoints_for_null_island +------------------------------- +t (1 row) - _obs_getpolygons --------------------- - {1570.72353789469} +obs_getpolygons_for_test_point +-------------------------------- +t (1 row) - _obs_getpolygons ------------------- - {NULL} +obs_getpolygons_for_null_island +--------------------------------- +t (1 row) segment_name | total_pop_quantile | male_pop_quantile | female_pop_quantile | median_age_quantile | white_pop_quantile | black_pop_quantile | asian_pop_quantile | hispanic_pop_quantile | not_us_citizen_pop_quantile | workers_16_and_over_quantile | commuters_by_car_truck_van_quantile | commuters_by_public_transportation_quantile | commuters_by_bus_quantile | commuters_by_subway_or_elevated_quantile | walked_to_work_quantile | worked_at_home_quantile | children_quantile | households_quantile | population_3_years_over_quantile | in_school_quantile | in_grades_1_to_4_quantile | in_grades_5_to_8_quantile | in_grades_9_to_12_quantile | in_undergrad_college_quantile | pop_25_years_over_quantile | high_school_diploma_quantile | bachelors_degree_quantile | masters_degree_quantile | pop_5_years_over_quantile | speak_only_english_at_home_quantile | speak_spanish_at_home_quantile | pop_determined_poverty_status_quantile | poverty_quantile | median_income_quantile | gini_index_quantile | income_per_capita_quantile | housing_units_quantile | vacant_housing_units_quantile | vacant_housing_units_for_rent_quantile | vacant_housing_units_for_sale_quantile | median_rent_quantile | percent_income_spent_on_rent_quantile | owner_occupied_housing_units_quantile | million_dollar_housing_units_quantile diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 1436597..5999383 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -41,42 +41,50 @@ FROM -- -----------|------- -- gini_index | 0.3494 -SELECT * FROM - cdb_observatory._OBS_Get( +WITH result as ( + SELECT _OBS_GetJSON::text as expected FROM + cdb_observatory._OBS_GetJSON( cdb_observatory._TestPoint(), Array['"us.census.acs".B19083001']::text[], '2009 - 2013', '"us.census.tiger".block_group' - ); + ) +) select expected = '{"value":0.3494,"name":"Gini Index","tablename":"obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb","aggregate":"","type":"Numeric","description":""}' + from result; -- gini index at null island -SELECT * FROM - cdb_observatory._OBS_Get( +WITH result as ( + SELECT count(_OBS_GetJSON) as expected FROM + cdb_observatory._OBS_GetJSON( CDB_LatLng(0, 0), Array['"us.census.acs".B19083001']::text[], '2009 - 2013', '"us.census.tiger".block_group' - ); - + ) +) select expected = 0 as OBS_Get_gini_index_at_null_island + from result; + -- OBS_GetPoints -- obs_getpoints -- -------------------- -- {4809.33511352425} SELECT - cdb_observatory._OBS_GetPoints( + (cdb_observatory._OBS_GetPoints( cdb_observatory._TestPoint(), 'obs_a92e1111ad3177676471d66bb8036e6d057f271b'::text, -- see example in obs_geomtable - Array[('total_pop','obs_ab038198aaab3f3cb055758638ee4de28ad70146','sum')::cdb_observatory.OBS_ColumnData] - ); - + (Array['{"colname":"total_pop","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json]) + ))[1]::text = '{"value":4809.33511352425,"name":"Total Population","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":"sum","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}' + as OBS_GetPoints_for_test_point; -- what happens at null island + SELECT - cdb_observatory._OBS_GetPoints( + (cdb_observatory._OBS_GetPoints( CDB_LatLng(0, 0), 'obs_a92e1111ad3177676471d66bb8036e6d057f271b'::text, -- see example in obs_geomtable - Array[('total_pop','obs_ab038198aaab3f3cb055758638ee4de28ad70146','sum')::cdb_observatory.OBS_ColumnData] - ); + (Array['{"colname":"total_pop","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json]) + ))[1]::text is null + as OBS_GetPoints_for_null_island; -- OBS_GetPolygons -- obs_getpolygons @@ -84,19 +92,21 @@ SELECT -- {12996.8172420752} SELECT - cdb_observatory._OBS_GetPolygons( + (cdb_observatory._OBS_GetPolygons( cdb_observatory._TestArea(), 'obs_a92e1111ad3177676471d66bb8036e6d057f271b'::text, -- see example in obs_geomtable - Array[('total_pop','obs_ab038198aaab3f3cb055758638ee4de28ad70146','sum')::cdb_observatory.OBS_ColumnData] -); + Array['{"colname":"total_pop","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json] +))[1]::text = '{"value":12996.8172420752,"name":"Total Population","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":"sum","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}' + as OBS_GetPolygons_for_test_point; -- see what happens around null island SELECT - cdb_observatory._OBS_GetPolygons( + (cdb_observatory._OBS_GetPolygons( ST_Buffer(CDB_LatLng(0, 0)::geography, 500)::geometry, 'obs_a92e1111ad3177676471d66bb8036e6d057f271b'::text, -- see example in obs_geomtable - Array[('total_pop','obs_ab038198aaab3f3cb055758638ee4de28ad70146','sum')::cdb_observatory.OBS_ColumnData] -); + Array['{"colname":"total_pop","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates."}'::json]) + )[1]->>'value' is null + as OBS_GetPolygons_for_null_island; SELECT * FROM cdb_observatory._OBS_GetSegmentSnapshot( From 2c46a72038facf2cf3e79201af1d2df646c04534 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 16:14:56 -0400 Subject: [PATCH 43/60] bug fix --- src/pg/test/sql/41_observatory_augmentation_test.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 5999383..63433c2 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -42,20 +42,21 @@ FROM -- gini_index | 0.3494 WITH result as ( - SELECT _OBS_GetJSON::text as expected FROM - cdb_observatory._OBS_GetJSON( + SELECT _OBS_Get::text as expected FROM + cdb_observatory._OBS_Get( cdb_observatory._TestPoint(), Array['"us.census.acs".B19083001']::text[], '2009 - 2013', '"us.census.tiger".block_group' ) ) select expected = '{"value":0.3494,"name":"Gini Index","tablename":"obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb","aggregate":"","type":"Numeric","description":""}' + as OBS_Get_gini_index_at_test_point from result; -- gini index at null island WITH result as ( - SELECT count(_OBS_GetJSON) as expected FROM - cdb_observatory._OBS_GetJSON( + SELECT count(_OBS_Get) as expected FROM + cdb_observatory._OBS_Get( CDB_LatLng(0, 0), Array['"us.census.acs".B19083001']::text[], '2009 - 2013', From a9b22caadf2c06d703ee80690dde924707fffce5 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 16:40:21 -0400 Subject: [PATCH 44/60] updating OBS_GetCategories to json internals --- src/pg/sql/41_observatory_augmentation.sql | 39 ++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index cdf707c..9899829 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -691,14 +691,14 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCategories( geometry_level text DEFAULT '"us.census.tiger".block_group', time_span text DEFAULT '2009 - 2013' ) -RETURNS TABLE(names text[], categories text[]) as $$ +RETURNS SETOF JSON as $$ DECLARE geom_table_name text; geoid text; names text[]; results text[]; query text; - data_table_info cdb_observatory.OBS_ColumnData[]; + data_table_info json[]; BEGIN geom_table_name := cdb_observatory._OBS_GeomTable(geom, geometry_level); @@ -709,13 +709,12 @@ BEGIN RETURN QUERY SELECT '{}'::text[], '{}'::text[]; END IF; - data_table_info := cdb_observatory._OBS_GetColumnData(geometry_level, - dimension_names, - time_span); - - - names := (SELECT array_agg((d).colname) - FROM unnest(data_table_info) As d); + execute' + select array_agg( _obs_getcolumndatajson) from cdb_observatory._OBS_GetColumnDataJSON($1, + $2, + $3);' + INTO data_table_info + using geometry_level, dimension_names, time_span; EXECUTE @@ -729,7 +728,7 @@ BEGIN query := 'SELECT ARRAY['; FOR i IN 1..array_upper(data_table_info, 1) LOOP - query = query || format('%I ', lower(((data_table_info)[i]).colname)); + query = query || format('%I ', lower(((data_table_info)[i])->>'colname')); IF i < array_upper(data_table_info, 1) THEN query := query || ','; @@ -740,8 +739,8 @@ BEGIN FROM observatory.%I WHERE %I.geoid = %L ', - ((data_table_info)[1]).tablename, - ((data_table_info)[1]).tablename, + ((data_table_info)[1])->>'tablename', + ((data_table_info)[1])->>'tablename', geoid ); @@ -749,9 +748,21 @@ BEGIN query INTO results USING geom; - + RETURN QUERY - SELECT names,results + EXECUTE + $query$ + select row_to_json(t) from( + select categories as category, + meta->>'name' as name, + meta->>'tablename' as tablename, + meta->>'aggregate' as aggregate, + meta->>'type' as type, + meta->>'description' as description + from (select unnest($1) as categories, unnest($2) as meta) b + ) t + $query$ + USING results, data_table_info; RETURN; END; From 0406d493a7ceb4d7930a748dcb5ffdb1bdb33c62 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 16:49:38 -0400 Subject: [PATCH 45/60] removing extra JSON from testing --- src/pg/test/sql/40_observatory_utility_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 5fe5c40..abdab50 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -53,7 +53,7 @@ SELECT Array['"us.census.tiger".baloney'], '2009 - 2013') a ) -select expected is null as OBS_GetColumnDataJSON_missing_measure +select expected is null as OBS_GetColumnData_missing_measure from result; -- OBS_LookupCensusHuman From aa29a287d15574d95cac61cfa173ae3ac610cc4c Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 22 Apr 2016 16:49:51 -0400 Subject: [PATCH 46/60] tests for OBS_GetCategories --- .../41_observatory_augmentation_test.out | 13 +++---- .../sql/41_observatory_augmentation_test.sql | 36 ++++++++++++------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 41bff69..5b13d3e 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -81,16 +81,17 @@ t | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | (1 row) - names | categories --------+--------------------------------- - {X10} | {"Wealthy, urban without Kids"} +getcategories_at_test_point_1 | getcategories_at_test_point_2 +-------------------------------+------------------------------- +t | t (1 row) - names | categories --------+------------ - {X10} | +getcategories_at_null_island +------------------------------ +t (1 row) + obs_getmeasure ---------------------------------------------------- {"name" : "total_pop", "value" : 9516.27915900609} diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 63433c2..137b021 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -122,19 +122,29 @@ SELECT * FROM '"us.census.tiger".census_tract' ); -SELECT * FROM - cdb_observatory._OBS_GetCategories( - cdb_observatory._TestPoint(), - Array['"us.census.spielman_singleton_segments".X10'], - '"us.census.tiger".census_tract' -); - -SELECT * FROM - cdb_observatory._OBS_GetCategories( - CDB_LatLng(0, 0), - Array['"us.census.spielman_singleton_segments".X10'], - '"us.census.tiger".census_tract' -); +WITH result as ( + SELECT array_agg(_OBS_GetCategories) as expected FROM + cdb_observatory._OBS_GetCategories( + cdb_observatory._TestPoint(), + Array['"us.census.spielman_singleton_segments".X10'], + '"us.census.tiger".census_tract' + ) +) + select (expected)[1]::text = '{"category":"Wealthy, urban without Kids","name":"SS_segment_10_clusters","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"type":"Text","description":"Sociodemographic classes from Spielman and Singleton 2015, 10 clusters"}' as GetCategories_at_test_point_1, + (expected)[2]::text ='{"category":"Wealthy, urban without Kids","name":"SS_segment_10_clusters","tablename":"obs_11ee8b82c877c073438bc935a91d3dfccef875d1","aggregate":null,"type":"Text","description":"Sociodemographic classes from Spielman and Singleton 2015, 10 clusters"}' as GetCategories_at_test_point_2 + from result; + +WITH result as ( + SELECT array_agg(_OBS_GetCategories) as expected FROM + cdb_observatory._OBS_GetCategories( + -- cdb_observatory._TestPoint(), + CDB_LatLng(0,0), + Array['"us.census.spielman_singleton_segments".X10'], + '"us.census.tiger".census_tract' + ) +) + select expected is null as GetCategories_at_null_island + from result; -- Point-based OBS_GetMeasure, default normalization (area) SELECT * FROM From 6709ce1589ff07b1bd9d2447c39a12e5d972cd58 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 09:24:25 -0400 Subject: [PATCH 47/60] removing extra json in function causing bug --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 9899829..02dafb0 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -710,7 +710,7 @@ BEGIN END IF; execute' - select array_agg( _obs_getcolumndatajson) from cdb_observatory._OBS_GetColumnDataJSON($1, + select array_agg( _obs_getcolumndata) from cdb_observatory._OBS_GetColumnDataJSON($1, $2, $3);' INTO data_table_info From 68f5bce80bb12f5417f743c97ab193e30b891c25 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 09:52:34 -0400 Subject: [PATCH 48/60] updating get census functions to work with new json internals --- src/pg/sql/41_observatory_augmentation.sql | 31 +++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 02dafb0..ff46e47 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -247,7 +247,7 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCensus( time_span text DEFAULT '2009 - 2013', geometry_level text DEFAULT '"us.census.tiger".block_group' ) -RETURNS TABLE(dimension text[], dimension_value NUMERIC[]) +RETURNS SETOF JSON AS $$ DECLARE ids text[]; @@ -256,10 +256,35 @@ BEGIN ids := cdb_observatory._OBS_LookupCensusHuman(dimension_names); RETURN QUERY - SELECT names, vals FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level); + SELECT * FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level); END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCensus( + geom geometry, + dimension_name text, + time_span text DEFAULT '2009 - 2013', + geometry_level text DEFAULT '"us.census.tiger".block_group' +) +RETURNS NUMERIC +AS $$ +DECLARE + ids Text[]; + result_json json; + result Numeric; +BEGIN + + ids := cdb_observatory._OBS_LookupCensusHuman(Array[dimension_name]); + result_json := (SELECT a FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level) as a limit 1); + EXECUTE + format('select $1::numeric as "%s"', result_json->>'name') + INTO result + USING + result_json->>'value'; + + return result; +END; +$$ LANGUAGE plpgsql; -- Base augmentation fucntion. @@ -710,7 +735,7 @@ BEGIN END IF; execute' - select array_agg( _obs_getcolumndata) from cdb_observatory._OBS_GetColumnDataJSON($1, + select array_agg( _obs_getcolumndata) from cdb_observatory._OBS_GetColumnData($1, $2, $3);' INTO data_table_info From 07b678e448861a27b916e1ab9fe6dd91898abdc3 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 10:12:43 -0400 Subject: [PATCH 49/60] test for obs get census --- .../41_observatory_augmentation_test.out | 18 +++---- .../sql/41_observatory_augmentation_test.sql | 52 ++++++++++++------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 5b13d3e..1a14f88 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -26,19 +26,19 @@ Done. 9516.27915900609 | 6152.51885204623 | 3363.76030695986 | 28.8 | 5301.51624447348 | 149.500458087105 | 230.000704749392 | 3835.26175169611 | 0 | 0 | 0 | 5681.01740730998 | 3323.51018362871 | 7107.02177675621 | 1040.753188991 | 69.0002114248176 | 793.502431385402 | 327.751004267883 | 2742.7584041365 | 931.502854235037 | 66304 | 0.3494 | 28291 | 3662.76122313407 | 339.251039505353 | 120.750369993431 | 0 | 1764 | 35.3 | 339.251039505353 | 0 | 224.250687130657 | 6549.27006773893 | 327.751004267883 | 28.750088093674 | 201.250616655718 | 621.001902823358 | 373.751145217762 | 1851.5056732326 | 1414.50433420876 | 1115.50341803455 | 615.251885204623 | | 57.500176187348 | 0 | 212.750651893187 | 408.251250930171 | 0 | 155.25047570584 | 109.250334755961 | 92.0002818997568 | 63.2501938060828 | 184.000563799514 | 621.001902823358 | 552.001691398541 | 327.751004267883 | 333.501021886618 | 126.500387612166 | | (1 row) - dimension | dimension_value -----------------------+------------------------------------- - {total_pop,male_pop} | {9516.27915900609,6152.51885204623} +test_obsgetcensuswithtestpointand2variables +--------------------------------------------- +t (1 row) - dimension | dimension_value ------------------------+----------------- - {female_pop,male_pop} | {NULL,NULL} +test_obsgetcensuswithnullislandarea +------------------------------------- +t (1 row) - dimension | dimension_value ------------------------+----------------- - {female_pop,male_pop} | {} +test_obsgetcensuswithnullisland +--------------------------------- +t (1 row) obs_get_gini_index_at_test_point diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 137b021..ad7cf1a 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -13,28 +13,42 @@ SELECT * FROM -- total_pop | 9516.27915900609 -- male_pop | 6152.51885204623 -SELECT * -FROM - cdb_observatory._OBS_GetCensus( - cdb_observatory._TestPoint(), - Array['total_pop','male_pop']::text[] - ); - +WITH result as ( + SELECT array_agg(_obs_getcensus->>'value') as b + FROM( select * from + cdb_observatory._OBS_GetCensus( + cdb_observatory._TestPoint(), + Array['total_pop','male_pop']::text[] + )) a +) +select b='{9516.27915900609,6152.51885204623}' + as test_obsGetCensusWithTestPointAnd2Variables + from result; -- what happens on null island? -- expect nulls back: {female_pop, male_pop} | {NULL, NULL} -SELECT * -FROM - cdb_observatory._OBS_GetCensus( - ST_Buffer(CDB_LatLng(0, 0)::geography, 5000)::geometry, - Array['female_pop','male_pop']::text[] - ); + +WITH result as ( + SELECT count(vals) non_null + FROM( select _OBS_GetCensus->>'value' vals from + cdb_observatory._OBS_GetCensus( + ST_Buffer(CDB_LatLng(0, 0)::geography, 5000)::geometry, + Array['total_pop','male_pop']::text[] + )) a +) +SELECT non_null = 0 as test_obsGetCensusWithNullIslandArea +FROM result; + -- expect nulls back {female_pop, male_pop} | {NULL, NULL} -SELECT * -FROM - cdb_observatory._OBS_GetCensus( - CDB_LatLng(0, 0), - Array['female_pop', 'male_pop']::text[] - ); +WITH result as ( + SELECT count(vals) non_null + FROM( select _OBS_GetCensus->>'value' vals from + cdb_observatory._OBS_GetCensus( + CDB_LatLng(0, 0), + Array['total_pop','male_pop']::text[] + )) a +) +SELECT non_null = 0 as test_obsGetCensusWithNullIsland +FROM result; -- -- names | vals From 511a15d9939cdaba82bb2651098134d9a71349d5 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 10:15:08 -0400 Subject: [PATCH 50/60] removing extra test statment --- src/pg/test/sql/40_observatory_utility_test.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index abdab50..6093624 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -40,8 +40,7 @@ SELECT '2009 - 2013') a ) select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, - (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2, - (expected)[3]::text = '{"colname":"geoid","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_3 + (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2 from result; From 20ec7ef25a2de1a07b54755fbc5fc605d77b8de1 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 10:18:18 -0400 Subject: [PATCH 51/60] whitespace fix --- .../expected/40_observatory_utility_test.out | 60 +++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 5cc88e7..f24f5f3 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -46,14 +46,46 @@ SELECT (1 row) -test_get_obs_column_with_geoid_and_census_1 | test_get_obs_column_with_geoid_and_census_2 | test_get_obs_column_with_geoid_and_census_3 ----------------------------------------------+---------------------------------------------+--------------------------------------------- -t | t | t +-- future test: give back nulls when geometry doesn't intersect +-- SELECT +-- cdb_observatory._OBS_GeomTable( +-- CDB_LatLng(0,0), -- should give back null since it's in the ocean? +-- '"us.census.tiger".census_tract' +-- ); +-- OBS_GetColumnData +-- should give back: +-- colname | tablename | aggregate +-- -----------|-----------------|----------- +-- geoid | obs_{hex table} | null +-- total_pop | obs_{hex table} | sum +WITH result as ( +SELECT + array_agg(a) expected from cdb_observatory._OBS_GetColumnData( + '"us.census.tiger".census_tract', + Array['"us.census.tiger".census_tract_geoid', '"us.census.acs".B01001001'], + '2009 - 2013') a +) +select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, + (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2 +from result; + test_get_obs_column_with_geoid_and_census_1 | test_get_obs_column_with_geoid_and_census_2 +---------------------------------------------+--------------------------------------------- + t | t (1 row) -obs_getcolumndatajson_missing_measure ---------------------------------------- -t +-- should be null-valued +WITH result as ( +SELECT + array_agg(a) expected from cdb_observatory._OBS_GetColumnData( + '"us.census.tiger".census_tract', + Array['"us.census.tiger".baloney'], + '2009 - 2013') a +) +select expected is null as OBS_GetColumnData_missing_measure +from result; + obs_getcolumndata_missing_measure +----------------------------------- + t (1 row) -- OBS_LookupCensusHuman @@ -98,11 +130,17 @@ SELECT SELECT vals[1] As mandarin_orange (1 row) --- should give back a normalized name - SELECT cdb_observatory._OBS_NormalizeMeasureName('test 343 %% 2 qqq }}{{}}'); -_obs_normalizemeasurename ---------------------------- -test_343_2_qqq +SELECT cdb_observatory._OBS_GetRelatedColumn( + Array[ + '"es.ine".pop_0_4', + '"us.census.acs".B01001001', + '"us.census.acs".B01001002' + ], + 'denominator' + ); + _obs_getrelatedcolumn +------------------------------------------------------------- + {"\"es.ine\".total_pop",NULL,"\"us.census.acs\".B01001001"} (1 row) \i test/sql/drop_fixtures.sql From ace34f6ad820bbbc7b3ff1cbdfbdffa411f09c0d Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 10:45:54 -0400 Subject: [PATCH 52/60] removing table returning census snapshot function for json returning one for now --- src/pg/sql/41_observatory_augmentation.sql | 508 +++++++++++------- .../sql/41_observatory_augmentation_test.sql | 12 +- 2 files changed, 307 insertions(+), 213 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index ff46e47..c958fe2 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -22,217 +22,313 @@ -- Creates a table of demographic snapshot -CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetDemographicSnapshot(geom geometry, time_span text default '2009 - 2013', geometry_level text default '"us.census.tiger".block_group') -RETURNS json +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetDemographicSnapshotJ(geom geometry, time_span text default '2009 - 2013', geometry_level text default '"us.census.tiger".block_group') +RETURNS SETOF JSON AS $$ + DECLARE + target_cols text[]; BEGIN - RETURN row_to_json(cdb_observatory._OBS_GetDemographicSnapshot(geom, time_span, geometry_level)); + target_cols := Array['total_pop', + 'male_pop', + 'female_pop', + 'median_age', + 'white_pop', + 'black_pop', + 'asian_pop', + 'hispanic_pop', + 'amerindian_pop', + 'other_race_pop', + 'two_or_more_races_pop', + 'not_hispanic_pop', + --'not_us_citizen_pop', + --'workers_16_and_over', + --'commuters_by_car_truck_van', + --'commuters_drove_alone', + --'commuters_by_carpool', + --'commuters_by_public_transportation', + --'commuters_by_bus', + --'commuters_by_subway_or_elevated', + --'walked_to_work', + --'worked_at_home', + --'children', + 'households', + --'population_3_years_over', + --'in_school', + --'in_grades_1_to_4', + --'in_grades_5_to_8', + --'in_grades_9_to_12', + --'in_undergrad_college', + 'pop_25_years_over', + 'high_school_diploma', + 'less_one_year_college', + 'one_year_more_college', + 'associates_degree', + 'bachelors_degree', + 'masters_degree', + --'pop_5_years_over', + --'speak_only_english_at_home', + --'speak_spanish_at_home', + --'pop_determined_poverty_status', + --'poverty', + 'median_income', + 'gini_index', + 'income_per_capita', + 'housing_units', + 'vacant_housing_units', + 'vacant_housing_units_for_rent', + 'vacant_housing_units_for_sale', + 'median_rent', + 'percent_income_spent_on_rent', + 'owner_occupied_housing_units', + 'million_dollar_housing_units', + 'mortgaged_housing_units', + --'pop_15_and_over', + --'pop_never_married', + --'pop_now_married', + --'pop_separated', + --'pop_widowed', + --'pop_divorced', + 'commuters_16_over', + 'commute_less_10_mins', + 'commute_10_14_mins', + 'commute_15_19_mins', + 'commute_20_24_mins', + 'commute_25_29_mins', + 'commute_30_34_mins', + 'commute_35_44_mins', + 'commute_45_59_mins', + 'commute_60_more_mins', + 'aggregate_travel_time_to_work', + 'income_less_10000', + 'income_10000_14999', + 'income_15000_19999', + 'income_20000_24999', + 'income_25000_29999', + 'income_30000_34999', + 'income_35000_39999', + 'income_40000_44999', + 'income_45000_49999', + 'income_50000_59999', + 'income_60000_74999', + 'income_75000_99999', + 'income_100000_124999', + 'income_125000_149999', + 'income_150000_199999', + 'income_200000_or_more', + 'land_area']; + RETURN QUERY + 'select * from cdb_observatory._OBS_GetCensus($1, $2 )' + USING geom, target_cols + RETURN; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetDemographicSnapshot(geom geometry, time_span text default '2009 - 2013', geometry_level text default '"us.census.tiger".block_group' ) -RETURNS TABLE( - total_pop NUMERIC, - male_pop NUMERIC, - female_pop NUMERIC, - median_age NUMERIC, - white_pop NUMERIC, - black_pop NUMERIC, - asian_pop NUMERIC, - hispanic_pop NUMERIC, - amerindian_pop NUMERIC, - other_race_pop NUMERIC, - two_or_more_races_pop NUMERIC, - not_hispanic_pop NUMERIC, - --not_us_citizen_pop NUMERIC, - --workers_16_and_over NUMERIC, - --commuters_by_car_truck_van NUMERIC, - --commuters_drove_alone NUMERIC, - --commuters_by_carpool NUMERIC, - --commuters_by_public_transportation NUMERIC, - --commuters_by_bus NUMERIC, - --commuters_by_subway_or_elevated NUMERIC, - --walked_to_work NUMERIC, - --worked_at_home NUMERIC, - --children NUMERIC, -- TODO we should be able to get this at BG - households NUMERIC, - --population_3_years_over NUMERIC, - --in_school NUMERIC, - --in_grades_1_to_4 NUMERIC, - --in_grades_5_to_8 NUMERIC, - --in_grades_9_to_12 NUMERIC, - --in_undergrad_college NUMERIC, - pop_25_years_over NUMERIC, - high_school_diploma NUMERIC, - less_one_year_college NUMERIC, - one_year_more_college NUMERIC, - associates_degree NUMERIC, - bachelors_degree NUMERIC, - masters_degree NUMERIC, - --pop_5_years_over NUMERIC, - --speak_only_english_at_home NUMERIC, - --speak_spanish_at_home NUMERIC, - --pop_determined_poverty_status NUMERIC, - --poverty NUMERIC, - median_income NUMERIC, - gini_index NUMERIC, - income_per_capita NUMERIC, - housing_units NUMERIC, - vacant_housing_units NUMERIC, - vacant_housing_units_for_rent NUMERIC, - vacant_housing_units_for_sale NUMERIC, - median_rent NUMERIC, - percent_income_spent_on_rent NUMERIC, - owner_occupied_housing_units NUMERIC, - million_dollar_housing_units NUMERIC, - mortgaged_housing_units NUMERIC, - --pop_15_and_over NUMERIC, - --pop_never_married NUMERIC, - --pop_now_married NUMERIC, - --pop_separated NUMERIC, - --pop_widowed NUMERIC, - --pop_divorced NUMERIC, - commuters_16_over NUMERIC, - commute_less_10_mins NUMERIC, - commute_10_14_mins NUMERIC, - commute_15_19_mins NUMERIC, - commute_20_24_mins NUMERIC, - commute_25_29_mins NUMERIC, - commute_30_34_mins NUMERIC, - commute_35_44_mins NUMERIC, - commute_45_59_mins NUMERIC, - commute_60_more_mins NUMERIC, - aggregate_travel_time_to_work NUMERIC, - income_less_10000 NUMERIC, - income_10000_14999 NUMERIC, - income_15000_19999 NUMERIC, - income_20000_24999 NUMERIC, - income_25000_29999 NUMERIC, - income_30000_34999 NUMERIC, - income_35000_39999 NUMERIC, - income_40000_44999 NUMERIC, - income_45000_49999 NUMERIC, - income_50000_59999 NUMERIC, - income_60000_74999 NUMERIC, - income_75000_99999 NUMERIC, - income_100000_124999 NUMERIC, - income_125000_149999 NUMERIC, - income_150000_199999 NUMERIC, - income_200000_or_more NUMERIC, - land_area NUMERIC) -AS $$ -DECLARE - target_cols text[]; - names text[]; - vals NUMERIC[]; - q text; -BEGIN - target_cols := Array['total_pop', - 'male_pop', - 'female_pop', - 'median_age', - 'white_pop', - 'black_pop', - 'asian_pop', - 'hispanic_pop', - 'amerindian_pop', - 'other_race_pop', - 'two_or_more_races_pop', - 'not_hispanic_pop', - --'not_us_citizen_pop', - --'workers_16_and_over', - --'commuters_by_car_truck_van', - --'commuters_drove_alone', - --'commuters_by_carpool', - --'commuters_by_public_transportation', - --'commuters_by_bus', - --'commuters_by_subway_or_elevated', - --'walked_to_work', - --'worked_at_home', - --'children', - 'households', - --'population_3_years_over', - --'in_school', - --'in_grades_1_to_4', - --'in_grades_5_to_8', - --'in_grades_9_to_12', - --'in_undergrad_college', - 'pop_25_years_over', - 'high_school_diploma', - 'less_one_year_college', - 'one_year_more_college', - 'associates_degree', - 'bachelors_degree', - 'masters_degree', - --'pop_5_years_over', - --'speak_only_english_at_home', - --'speak_spanish_at_home', - --'pop_determined_poverty_status', - --'poverty', - 'median_income', - 'gini_index', - 'income_per_capita', - 'housing_units', - 'vacant_housing_units', - 'vacant_housing_units_for_rent', - 'vacant_housing_units_for_sale', - 'median_rent', - 'percent_income_spent_on_rent', - 'owner_occupied_housing_units', - 'million_dollar_housing_units', - 'mortgaged_housing_units', - --'pop_15_and_over', - --'pop_never_married', - --'pop_now_married', - --'pop_separated', - --'pop_widowed', - --'pop_divorced', - 'commuters_16_over', - 'commute_less_10_mins', - 'commute_10_14_mins', - 'commute_15_19_mins', - 'commute_20_24_mins', - 'commute_25_29_mins', - 'commute_30_34_mins', - 'commute_35_44_mins', - 'commute_45_59_mins', - 'commute_60_more_mins', - 'aggregate_travel_time_to_work', - 'income_less_10000', - 'income_10000_14999', - 'income_15000_19999', - 'income_20000_24999', - 'income_25000_29999', - 'income_30000_34999', - 'income_35000_39999', - 'income_40000_44999', - 'income_45000_49999', - 'income_50000_59999', - 'income_60000_74999', - 'income_75000_99999', - 'income_100000_124999', - 'income_125000_149999', - 'income_150000_199999', - 'income_200000_or_more', - 'land_area']; - - q := 'WITH a As ( - SELECT - dimension As names, - dimension_value As vals - FROM cdb_observatory._OBS_GetCensus($1,$2,$3,$4) - )' || - cdb_observatory._OBS_BuildSnapshotQuery(target_cols) || - ' FROM a'; - - RETURN QUERY - EXECUTE - q - USING geom, target_cols, time_span, geometry_level; - - RETURN; -END; -$$ LANGUAGE plpgsql; +-- CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetDemographicSnapshot(geom geometry, time_span text default '2009 - 2013', geometry_level text default '"us.census.tiger".block_group' ) +-- RETURNS TABLE( +-- total_pop NUMERIC, +-- male_pop NUMERIC, +-- female_pop NUMERIC, +-- median_age NUMERIC, +-- white_pop NUMERIC, +-- black_pop NUMERIC, +-- asian_pop NUMERIC, +-- hispanic_pop NUMERIC, +-- amerindian_pop NUMERIC, +-- other_race_pop NUMERIC, +-- two_or_more_races_pop NUMERIC, +-- not_hispanic_pop NUMERIC, +-- --not_us_citizen_pop NUMERIC, +-- --workers_16_and_over NUMERIC, +-- --commuters_by_car_truck_van NUMERIC, +-- --commuters_drove_alone NUMERIC, +-- --commuters_by_carpool NUMERIC, +-- --commuters_by_public_transportation NUMERIC, +-- --commuters_by_bus NUMERIC, +-- --commuters_by_subway_or_elevated NUMERIC, +-- --walked_to_work NUMERIC, +-- --worked_at_home NUMERIC, +-- --children NUMERIC, -- TODO we should be able to get this at BG +-- households NUMERIC, +-- --population_3_years_over NUMERIC, +-- --in_school NUMERIC, +-- --in_grades_1_to_4 NUMERIC, +-- --in_grades_5_to_8 NUMERIC, +-- --in_grades_9_to_12 NUMERIC, +-- --in_undergrad_college NUMERIC, +-- pop_25_years_over NUMERIC, +-- high_school_diploma NUMERIC, +-- less_one_year_college NUMERIC, +-- one_year_more_college NUMERIC, +-- associates_degree NUMERIC, +-- bachelors_degree NUMERIC, +-- masters_degree NUMERIC, +-- --pop_5_years_over NUMERIC, +-- --speak_only_english_at_home NUMERIC, +-- --speak_spanish_at_home NUMERIC, +-- --pop_determined_poverty_status NUMERIC, +-- --poverty NUMERIC, +-- median_income NUMERIC, +-- gini_index NUMERIC, +-- income_per_capita NUMERIC, +-- housing_units NUMERIC, +-- vacant_housing_units NUMERIC, +-- vacant_housing_units_for_rent NUMERIC, +-- vacant_housing_units_for_sale NUMERIC, +-- median_rent NUMERIC, +-- percent_income_spent_on_rent NUMERIC, +-- owner_occupied_housing_units NUMERIC, +-- million_dollar_housing_units NUMERIC, +-- mortgaged_housing_units NUMERIC, +-- --pop_15_and_over NUMERIC, +-- --pop_never_married NUMERIC, +-- --pop_now_married NUMERIC, +-- --pop_separated NUMERIC, +-- --pop_widowed NUMERIC, +-- --pop_divorced NUMERIC, +-- commuters_16_over NUMERIC, +-- commute_less_10_mins NUMERIC, +-- commute_10_14_mins NUMERIC, +-- commute_15_19_mins NUMERIC, +-- commute_20_24_mins NUMERIC, +-- commute_25_29_mins NUMERIC, +-- commute_30_34_mins NUMERIC, +-- commute_35_44_mins NUMERIC, +-- commute_45_59_mins NUMERIC, +-- commute_60_more_mins NUMERIC, +-- aggregate_travel_time_to_work NUMERIC, +-- income_less_10000 NUMERIC, +-- income_10000_14999 NUMERIC, +-- income_15000_19999 NUMERIC, +-- income_20000_24999 NUMERIC, +-- income_25000_29999 NUMERIC, +-- income_30000_34999 NUMERIC, +-- income_35000_39999 NUMERIC, +-- income_40000_44999 NUMERIC, +-- income_45000_49999 NUMERIC, +-- income_50000_59999 NUMERIC, +-- income_60000_74999 NUMERIC, +-- income_75000_99999 NUMERIC, +-- income_100000_124999 NUMERIC, +-- income_125000_149999 NUMERIC, +-- income_150000_199999 NUMERIC, +-- income_200000_or_more NUMERIC, +-- land_area NUMERIC) +-- AS $$ +-- DECLARE +-- target_cols text[]; +-- names text[]; +-- vals NUMERIC[]; +-- q text; +-- BEGIN +-- target_cols := Array['total_pop', +-- 'male_pop', +-- 'female_pop', +-- 'median_age', +-- 'white_pop', +-- 'black_pop', +-- 'asian_pop', +-- 'hispanic_pop', +-- 'amerindian_pop', +-- 'other_race_pop', +-- 'two_or_more_races_pop', +-- 'not_hispanic_pop', +-- --'not_us_citizen_pop', +-- --'workers_16_and_over', +-- --'commuters_by_car_truck_van', +-- --'commuters_drove_alone', +-- --'commuters_by_carpool', +-- --'commuters_by_public_transportation', +-- --'commuters_by_bus', +-- --'commuters_by_subway_or_elevated', +-- --'walked_to_work', +-- --'worked_at_home', +-- --'children', +-- 'households', +-- --'population_3_years_over', +-- --'in_school', +-- --'in_grades_1_to_4', +-- --'in_grades_5_to_8', +-- --'in_grades_9_to_12', +-- --'in_undergrad_college', +-- 'pop_25_years_over', +-- 'high_school_diploma', +-- 'less_one_year_college', +-- 'one_year_more_college', +-- 'associates_degree', +-- 'bachelors_degree', +-- 'masters_degree', +-- --'pop_5_years_over', +-- --'speak_only_english_at_home', +-- --'speak_spanish_at_home', +-- --'pop_determined_poverty_status', +-- --'poverty', +-- 'median_income', +-- 'gini_index', +-- 'income_per_capita', +-- 'housing_units', +-- 'vacant_housing_units', +-- 'vacant_housing_units_for_rent', +-- 'vacant_housing_units_for_sale', +-- 'median_rent', +-- 'percent_income_spent_on_rent', +-- 'owner_occupied_housing_units', +-- 'million_dollar_housing_units', +-- 'mortgaged_housing_units', +-- --'pop_15_and_over', +-- --'pop_never_married', +-- --'pop_now_married', +-- --'pop_separated', +-- --'pop_widowed', +-- --'pop_divorced', +-- 'commuters_16_over', +-- 'commute_less_10_mins', +-- 'commute_10_14_mins', +-- 'commute_15_19_mins', +-- 'commute_20_24_mins', +-- 'commute_25_29_mins', +-- 'commute_30_34_mins', +-- 'commute_35_44_mins', +-- 'commute_45_59_mins', +-- 'commute_60_more_mins', +-- 'aggregate_travel_time_to_work', +-- 'income_less_10000', +-- 'income_10000_14999', +-- 'income_15000_19999', +-- 'income_20000_24999', +-- 'income_25000_29999', +-- 'income_30000_34999', +-- 'income_35000_39999', +-- 'income_40000_44999', +-- 'income_45000_49999', +-- 'income_50000_59999', +-- 'income_60000_74999', +-- 'income_75000_99999', +-- 'income_100000_124999', +-- 'income_125000_149999', +-- 'income_150000_199999', +-- 'income_200000_or_more', +-- 'land_area']; +-- +-- q := +-- $query$ +-- WITH a As ( +-- SELECT +-- array_agg(_OBS_GetCensusJ->>'name') As names, +-- array_agg(_OBS_GetCensusJ->>'value') As vals +-- FROM cdb_observatory._OBS_GetCensusJ($1,$2,$3,$4) +-- )$query$ || +-- cdb_observatory._OBS_BuildSnapshotQuery(target_cols) || +-- ' FROM a' +-- ; +-- +-- RETURN QUERY +-- EXECUTE +-- q +-- USING geom, target_cols, time_span, geometry_level; +-- +-- RETURN; +-- END; +-- $$ LANGUAGE plpgsql; --Base functions for performing augmentation diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index ad7cf1a..04699be 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -1,12 +1,10 @@ \i test/sql/load_fixtures.sql -- -SELECT * FROM - cdb_observatory._OBS_GetDemographicSnapshot( - cdb_observatory._TestPoint(), - '2009 - 2013', - '"us.census.tiger".block_group' - ) As snapshot; - +WITH result as( + Select count(OBS_GetDemographicSnapshot->>'value') expected_columns + FROM cdb_observatory.OBS_GetDemographicSnapshot(_test_point()) +) select expected_columns ='58' as OBS_GetDemographicSnapshot_test_no_returns +FROM result -- -- dimension | dimension_value -- ----------|---------------- From eda10dfa61bb84fac3670ffe7a024c92788eda2a Mon Sep 17 00:00:00 2001 From: "Andrew W. Hill" Date: Mon, 25 Apr 2016 10:48:01 -0400 Subject: [PATCH 53/60] Update methods.md removed JSON response stuff --- doc/methods.md | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/doc/methods.md b/doc/methods.md index 8d52e44..8e1ae08 100644 --- a/doc/methods.md +++ b/doc/methods.md @@ -18,27 +18,24 @@ normalize | for measures that are are **sums** (e.g. population) the default nor #### Returns -A JSON object containing the following properties +A NUMERIC value containing the following properties Key | Description --- | --- value | the raw or normalized measure -name | the human readable name of the measure -description | a brief description of the measure -type | the data type (text, number, boolean) #### Example Add a Measure to an empty column based on point locations in your table ```SQL -UPDATE tablename SET local_male_population = OBS_GetUSCensusMeasure(the_geom, 'Male Population') -> 'value' +UPDATE tablename SET local_male_population = OBS_GetUSCensusMeasure(the_geom, 'Male Population') ``` Get a measure at a single point location ```SQL -SELECT * FROM json_each(OBS_GetUSCensusMeasure(CDB_LatLng(40.7, -73.9), 'Male Population')) +SELECT OBS_GetUSCensusMeasure(CDB_LatLng(40.7, -73.9), 'Male Population') ``` + +## OBS_GetUSCensusCategory(point_geometry, measure_name); + +The ```OBS_GetUSCensusCategory(point_geometry, category_name)``` method returns a categorical measure based on a subset of the US Census variables at a point location. It requires a different function from ```OBS_GetUSCensusMeasure``` because this function will always return TEXT, whereas ```OBS_GetUSCensusMeasure``` will always returna NUMERIC value. + +#### Arguments + +Name |Description +--- | --- +point_geometry | a WGS84 point geometry (the_geom) +measure_name | a human readable string name of a US Census variable. The glossary of measure_names is [available below]('measure_name table'). +#### Returns + +A NUMERIC value containing the following properties + +Key | Description +--- | --- +value | the raw or normalized measure + +#### Example + +Add a Measure to an empty column based on point locations in your table + +```SQL +UPDATE tablename SET local_male_population = OBS_GetUSCensusCategory(the_geom, 'Spielman Singleton Category 10') +``` + +Get a measure at a single point location + +```SQL +SELECT OBS_GetUSCensusCategory(CDB_LatLng(40.7, -73.9), 'Spielman Singleton Category 10') +``` + + + ## OBS_GetMeasure(point_geometry, measure_id); The ```OBS_GetMeasure(point_geometry, measure_id)``` method returns any Data Observatory measure at a point location. From 21d306898be80f29e1522e0965769b989a40c57f Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 25 Apr 2016 13:24:39 -0400 Subject: [PATCH 55/60] typo --- src/pg/sql/41_observatory_augmentation.sql | 87 ++++++---------------- 1 file changed, 21 insertions(+), 66 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index c958fe2..0c157aa 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -117,6 +117,7 @@ AS $$ 'income_200000_or_more', 'land_area']; RETURN QUERY + EXECUTE 'select * from cdb_observatory._OBS_GetCensus($1, $2 )' USING geom, target_cols RETURN; @@ -653,71 +654,21 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION OBS_GetSegmentSnapshot(geom geometry, geometry_level text default '"us.census.tiger".census_tract') -RETURNS json -AS $$ - BEGIN - RETURN row_to_json(cdb_observatory._OBS_GetSegmentSnapshot(geom, geometry_level)); -END; -$$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION _OBS_GetSegmentSnapshot( + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetSegmentSnapshot( geom geometry, geometry_level text DEFAULT '"us.census.tiger".census_tract' - ) -RETURNS TABLE( - segment_name TEXT, - total_pop_quantile NUMERIC, - male_pop_quantile NUMERIC, - female_pop_quantile NUMERIC, - median_age_quantile NUMERIC, - white_pop_quantile NUMERIC, - black_pop_quantile NUMERIC, - asian_pop_quantile NUMERIC, - hispanic_pop_quantile NUMERIC, - not_us_citizen_pop_quantile NUMERIC, - workers_16_and_over_quantile NUMERIC, - commuters_by_car_truck_van_quantile NUMERIC, - commuters_by_public_transportation_quantile NUMERIC, - commuters_by_bus_quantile NUMERIC, - commuters_by_subway_or_elevated_quantile NUMERIC, - walked_to_work_quantile NUMERIC, - worked_at_home_quantile NUMERIC, - children_quantile NUMERIC, - households_quantile NUMERIC, - population_3_years_over_quantile NUMERIC, - in_school_quantile NUMERIC, - in_grades_1_to_4_quantile NUMERIC, - in_grades_5_to_8_quantile NUMERIC, - in_grades_9_to_12_quantile NUMERIC, - in_undergrad_college_quantile NUMERIC, - pop_25_years_over_quantile NUMERIC, - high_school_diploma_quantile NUMERIC, - bachelors_degree_quantile NUMERIC, - masters_degree_quantile NUMERIC, - pop_5_years_over_quantile NUMERIC, - speak_only_english_at_home_quantile NUMERIC, - speak_spanish_at_home_quantile NUMERIC, - pop_determined_poverty_status_quantile NUMERIC, - poverty_quantile NUMERIC, - median_income_quantile NUMERIC, - gini_index_quantile NUMERIC, - income_per_capita_quantile NUMERIC, - housing_units_quantile NUMERIC, - vacant_housing_units_quantile NUMERIC, - vacant_housing_units_for_rent_quantile NUMERIC, - vacant_housing_units_for_sale_quantile NUMERIC, - median_rent_quantile NUMERIC, - percent_income_spent_on_rent_quantile NUMERIC, - owner_occupied_housing_units_quantile NUMERIC, - million_dollar_housing_units_quantile NUMERIC -) +) +RETURNS JSON AS $$ DECLARE target_cols text[]; - seg_name Text; - geom_id Text; - q Text; + result json; + seg_name Text; + geom_id Text; + q Text; + segment_name Text; BEGIN target_cols := Array[ '"us.census.acs".B01001001_quantile', @@ -768,7 +719,7 @@ target_cols := Array[ EXECUTE $query$ - SELECT (categories)[1] + SELECT (_OBS_GetCategories)->>'name' FROM cdb_observatory._OBS_GetCategories( $1, Array['"us.census.spielman_singleton_segments".X10'], @@ -782,8 +733,8 @@ target_cols := Array[ format($query$ WITH a As ( SELECT - names As names, - vals As vals + array_agg(_OBS_GET->>'name') As names, + array_agg(_OBS_GET->>'value') As vals FROM cdb_observatory._OBS_Get($1, $2, '2009 - 2013', @@ -792,14 +743,18 @@ target_cols := Array[ ), percentiles As ( %s FROM a) - SELECT $4, percentiles.* - FROM percentiles - $query$, cdb_observatory._OBS_BuildSnapshotQuery(target_cols)); + SELECT row_to_json(r) FROM + ( SELECT $4 as segment_name, percentiles.* + FROM percentiles) r + $query$, cdb_observatory._OBS_BuildSnapshotQuery(target_cols)) results; - RETURN QUERY + EXECUTE q + into result USING geom, target_cols, geometry_level, segment_name; + + return result; END; $$ LANGUAGE plpgsql; From 82a838ff74383c22bfabb52b5b0fe3c399796c31 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Mon, 25 Apr 2016 14:56:45 -0400 Subject: [PATCH 56/60] whitespace fixes --- .../41_observatory_augmentation_test.out | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 1a14f88..1836a52 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -26,49 +26,49 @@ Done. 9516.27915900609 | 6152.51885204623 | 3363.76030695986 | 28.8 | 5301.51624447348 | 149.500458087105 | 230.000704749392 | 3835.26175169611 | 0 | 0 | 0 | 5681.01740730998 | 3323.51018362871 | 7107.02177675621 | 1040.753188991 | 69.0002114248176 | 793.502431385402 | 327.751004267883 | 2742.7584041365 | 931.502854235037 | 66304 | 0.3494 | 28291 | 3662.76122313407 | 339.251039505353 | 120.750369993431 | 0 | 1764 | 35.3 | 339.251039505353 | 0 | 224.250687130657 | 6549.27006773893 | 327.751004267883 | 28.750088093674 | 201.250616655718 | 621.001902823358 | 373.751145217762 | 1851.5056732326 | 1414.50433420876 | 1115.50341803455 | 615.251885204623 | | 57.500176187348 | 0 | 212.750651893187 | 408.251250930171 | 0 | 155.25047570584 | 109.250334755961 | 92.0002818997568 | 63.2501938060828 | 184.000563799514 | 621.001902823358 | 552.001691398541 | 327.751004267883 | 333.501021886618 | 126.500387612166 | | (1 row) -test_obsgetcensuswithtestpointand2variables + test_obsgetcensuswithtestpointand2variables --------------------------------------------- -t + t (1 row) -test_obsgetcensuswithnullislandarea + test_obsgetcensuswithnullislandarea ------------------------------------- -t + t (1 row) -test_obsgetcensuswithnullisland + test_obsgetcensuswithnullisland --------------------------------- -t + t (1 row) -obs_get_gini_index_at_test_point + obs_get_gini_index_at_test_point ---------------------------------- -t + t (1 row) -obs_get_gini_index_at_null_island + obs_get_gini_index_at_null_island ----------------------------------- -t + t (1 row) -obs_getpoints_for_test_point + obs_getpoints_for_test_point ------------------------------ -t + t (1 row) -obs_getpoints_for_null_island + obs_getpoints_for_null_island ------------------------------- -t + t (1 row) -obs_getpolygons_for_test_point + obs_getpolygons_for_test_point -------------------------------- -t + t (1 row) -obs_getpolygons_for_null_island + obs_getpolygons_for_null_island --------------------------------- -t + t (1 row) segment_name | total_pop_quantile | male_pop_quantile | female_pop_quantile | median_age_quantile | white_pop_quantile | black_pop_quantile | asian_pop_quantile | hispanic_pop_quantile | not_us_citizen_pop_quantile | workers_16_and_over_quantile | commuters_by_car_truck_van_quantile | commuters_by_public_transportation_quantile | commuters_by_bus_quantile | commuters_by_subway_or_elevated_quantile | walked_to_work_quantile | worked_at_home_quantile | children_quantile | households_quantile | population_3_years_over_quantile | in_school_quantile | in_grades_1_to_4_quantile | in_grades_5_to_8_quantile | in_grades_9_to_12_quantile | in_undergrad_college_quantile | pop_25_years_over_quantile | high_school_diploma_quantile | bachelors_degree_quantile | masters_degree_quantile | pop_5_years_over_quantile | speak_only_english_at_home_quantile | speak_spanish_at_home_quantile | pop_determined_poverty_status_quantile | poverty_quantile | median_income_quantile | gini_index_quantile | income_per_capita_quantile | housing_units_quantile | vacant_housing_units_quantile | vacant_housing_units_for_rent_quantile | vacant_housing_units_for_sale_quantile | median_rent_quantile | percent_income_spent_on_rent_quantile | owner_occupied_housing_units_quantile | million_dollar_housing_units_quantile @@ -81,14 +81,14 @@ t | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | (1 row) -getcategories_at_test_point_1 | getcategories_at_test_point_2 + getcategories_at_test_point_1 | getcategories_at_test_point_2 -------------------------------+------------------------------- -t | t + t | t (1 row) -getcategories_at_null_island + getcategories_at_null_island ------------------------------ -t + t (1 row) From 0535d3e305465f10eafcbb8c96340e72422917cc Mon Sep 17 00:00:00 2001 From: John Krauss Date: Mon, 25 Apr 2016 15:00:56 -0400 Subject: [PATCH 57/60] fixes for first test in 41 --- src/pg/test/sql/41_observatory_augmentation_test.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 04699be..9b89436 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -2,9 +2,9 @@ -- WITH result as( Select count(OBS_GetDemographicSnapshot->>'value') expected_columns - FROM cdb_observatory.OBS_GetDemographicSnapshot(_test_point()) + FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint()) ) select expected_columns ='58' as OBS_GetDemographicSnapshot_test_no_returns -FROM result +FROM result; -- -- dimension | dimension_value -- ----------|---------------- From 0d811f6eb3a09d074f2b1f6afc8cd35cd2080944 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Mon, 25 Apr 2016 15:10:09 -0400 Subject: [PATCH 58/60] uncomment some public-facing code --- src/pg/sql/41_observatory_augmentation.sql | 2 +- src/pg/test/expected/41_observatory_augmentation_test.out | 4 ---- src/pg/test/sql/41_observatory_augmentation_test.sql | 5 ++--- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 0c157aa..a736fb5 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -22,7 +22,7 @@ -- Creates a table of demographic snapshot -CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetDemographicSnapshotJ(geom geometry, time_span text default '2009 - 2013', geometry_level text default '"us.census.tiger".block_group') +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetDemographicSnapshot(geom geometry, time_span text default '2009 - 2013', geometry_level text default '"us.census.tiger".block_group') RETURNS SETOF JSON AS $$ DECLARE diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 1836a52..27ab610 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -21,10 +21,6 @@ Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... Done. Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... Done. - total_pop | male_pop | female_pop | median_age | white_pop | black_pop | asian_pop | hispanic_pop | amerindian_pop | other_race_pop | two_or_more_races_pop | not_hispanic_pop | households | pop_25_years_over | high_school_diploma | less_one_year_college | one_year_more_college | associates_degree | bachelors_degree | masters_degree | median_income | gini_index | income_per_capita | housing_units | vacant_housing_units | vacant_housing_units_for_rent | vacant_housing_units_for_sale | median_rent | percent_income_spent_on_rent | owner_occupied_housing_units | million_dollar_housing_units | mortgaged_housing_units | commuters_16_over | commute_less_10_mins | commute_10_14_mins | commute_15_19_mins | commute_20_24_mins | commute_25_29_mins | commute_30_34_mins | commute_35_44_mins | commute_45_59_mins | commute_60_more_mins | aggregate_travel_time_to_work | income_less_10000 | income_10000_14999 | income_15000_19999 | income_20000_24999 | income_25000_29999 | income_30000_34999 | income_35000_39999 | income_40000_44999 | income_45000_49999 | income_50000_59999 | income_60000_74999 | income_75000_99999 | income_100000_124999 | income_125000_149999 | income_150000_199999 | income_200000_or_more | land_area -------------------+------------------+------------------+------------+------------------+------------------+------------------+------------------+----------------+----------------+-----------------------+------------------+------------------+-------------------+---------------------+-----------------------+-----------------------+-------------------+------------------+------------------+---------------+------------+-------------------+------------------+----------------------+-------------------------------+-------------------------------+-------------+------------------------------+------------------------------+------------------------------+-------------------------+-------------------+----------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------------------+-------------------------------+-------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------------------+----------------------+----------------------+-----------------------+----------- - 9516.27915900609 | 6152.51885204623 | 3363.76030695986 | 28.8 | 5301.51624447348 | 149.500458087105 | 230.000704749392 | 3835.26175169611 | 0 | 0 | 0 | 5681.01740730998 | 3323.51018362871 | 7107.02177675621 | 1040.753188991 | 69.0002114248176 | 793.502431385402 | 327.751004267883 | 2742.7584041365 | 931.502854235037 | 66304 | 0.3494 | 28291 | 3662.76122313407 | 339.251039505353 | 120.750369993431 | 0 | 1764 | 35.3 | 339.251039505353 | 0 | 224.250687130657 | 6549.27006773893 | 327.751004267883 | 28.750088093674 | 201.250616655718 | 621.001902823358 | 373.751145217762 | 1851.5056732326 | 1414.50433420876 | 1115.50341803455 | 615.251885204623 | | 57.500176187348 | 0 | 212.750651893187 | 408.251250930171 | 0 | 155.25047570584 | 109.250334755961 | 92.0002818997568 | 63.2501938060828 | 184.000563799514 | 621.001902823358 | 552.001691398541 | 327.751004267883 | 333.501021886618 | 126.500387612166 | | -(1 row) test_obsgetcensuswithtestpointand2variables --------------------------------------------- diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 9b89436..ee6c46c 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -122,14 +122,14 @@ SELECT as OBS_GetPolygons_for_null_island; SELECT * FROM - cdb_observatory._OBS_GetSegmentSnapshot( + cdb_observatory.OBS_GetSegmentSnapshot( cdb_observatory._TestPoint(), '"us.census.tiger".census_tract' ); -- segmentation around null island SELECT * FROM - cdb_observatory._OBS_GetSegmentSnapshot( + cdb_observatory.OBS_GetSegmentSnapshot( CDB_LatLng(0, 0), '"us.census.tiger".census_tract' ); @@ -149,7 +149,6 @@ WITH result as ( WITH result as ( SELECT array_agg(_OBS_GetCategories) as expected FROM cdb_observatory._OBS_GetCategories( - -- cdb_observatory._TestPoint(), CDB_LatLng(0,0), Array['"us.census.spielman_singleton_segments".X10'], '"us.census.tiger".census_tract' From 3e99b2deeb6bdc3b91f2701ddf2f818c518d9c8d Mon Sep 17 00:00:00 2001 From: John Krauss Date: Mon, 25 Apr 2016 15:19:38 -0400 Subject: [PATCH 59/60] add in missing test --- src/pg/test/expected/41_observatory_augmentation_test.out | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 27ab610..f4a39d6 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -21,6 +21,10 @@ Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... Done. Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... Done. + obs_getdemographicsnapshot_test_no_returns +-------------------------------------------- + t +(1 row) test_obsgetcensuswithtestpointand2variables --------------------------------------------- From b93ea03786ae0d6f542d7d2d1ff170faba16a166 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Mon, 25 Apr 2016 15:31:39 -0400 Subject: [PATCH 60/60] update tests for segmentation to work with JSON --- .../expected/41_observatory_augmentation_test.out | 12 ++++++------ src/pg/test/sql/41_observatory_augmentation_test.sql | 10 ++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index f4a39d6..7e06252 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -71,14 +71,14 @@ Done. t (1 row) - segment_name | total_pop_quantile | male_pop_quantile | female_pop_quantile | median_age_quantile | white_pop_quantile | black_pop_quantile | asian_pop_quantile | hispanic_pop_quantile | not_us_citizen_pop_quantile | workers_16_and_over_quantile | commuters_by_car_truck_van_quantile | commuters_by_public_transportation_quantile | commuters_by_bus_quantile | commuters_by_subway_or_elevated_quantile | walked_to_work_quantile | worked_at_home_quantile | children_quantile | households_quantile | population_3_years_over_quantile | in_school_quantile | in_grades_1_to_4_quantile | in_grades_5_to_8_quantile | in_grades_9_to_12_quantile | in_undergrad_college_quantile | pop_25_years_over_quantile | high_school_diploma_quantile | bachelors_degree_quantile | masters_degree_quantile | pop_5_years_over_quantile | speak_only_english_at_home_quantile | speak_spanish_at_home_quantile | pop_determined_poverty_status_quantile | poverty_quantile | median_income_quantile | gini_index_quantile | income_per_capita_quantile | housing_units_quantile | vacant_housing_units_quantile | vacant_housing_units_for_rent_quantile | vacant_housing_units_for_sale_quantile | median_rent_quantile | percent_income_spent_on_rent_quantile | owner_occupied_housing_units_quantile | million_dollar_housing_units_quantile ------------------------------+--------------------+-------------------+---------------------+---------------------+--------------------+--------------------+--------------------+-----------------------+-----------------------------+------------------------------+-------------------------------------+---------------------------------------------+---------------------------+------------------------------------------+-------------------------+-------------------------+--------------------+---------------------+----------------------------------+--------------------+---------------------------+---------------------------+----------------------------+-------------------------------+----------------------------+------------------------------+---------------------------+-------------------------+---------------------------+-------------------------------------+--------------------------------+----------------------------------------+-------------------+------------------------+---------------------+----------------------------+------------------------+-------------------------------+----------------------------------------+----------------------------------------+----------------------+---------------------------------------+---------------------------------------+--------------------------------------- - Wealthy, urban without Kids | 0.234783783783784 | 0.422405405405405 | 0.0987567567567568 | 0.0715 | 0.295310810810811 | 0.407189189189189 | 0.625608108108108 | 0.795202702702703 | 0.703797297297297 | 0.59227027027027 | 0.0180540540540541 | 0.993756756756757 | 0.728162162162162 | 0.995972972972973 | 0.929135135135135 | 0.625432432432432 | 0.0386081081081081 | 0.157121621621622 | 0.241878378378378 | 0.173783783783784 | 0.0380675675675676 | 0.0308108108108108 | 0.0486216216216216 | 0.479743243243243 | 0.297675675675676 | 0.190351351351351 | 0.802513513513514 | 0.757148648648649 | 0.255405405405405 | 0.196094594594595 | 0.816851351351351 | 0.252513513513514 | 0.560054054054054 | 0.777472972972973 | 0.336932432432432 | 0.655378378378378 | 0.141810810810811 | 0.362824324324324 | 0.463837837837838 | 0 | 0.939040540540541 | 0.419445945945946 | 0.0387972972972973 | 0 + test_point_segmentation +------------------------- + t (1 row) - segment_name | total_pop_quantile | male_pop_quantile | female_pop_quantile | median_age_quantile | white_pop_quantile | black_pop_quantile | asian_pop_quantile | hispanic_pop_quantile | not_us_citizen_pop_quantile | workers_16_and_over_quantile | commuters_by_car_truck_van_quantile | commuters_by_public_transportation_quantile | commuters_by_bus_quantile | commuters_by_subway_or_elevated_quantile | walked_to_work_quantile | worked_at_home_quantile | children_quantile | households_quantile | population_3_years_over_quantile | in_school_quantile | in_grades_1_to_4_quantile | in_grades_5_to_8_quantile | in_grades_9_to_12_quantile | in_undergrad_college_quantile | pop_25_years_over_quantile | high_school_diploma_quantile | bachelors_degree_quantile | masters_degree_quantile | pop_5_years_over_quantile | speak_only_english_at_home_quantile | speak_spanish_at_home_quantile | pop_determined_poverty_status_quantile | poverty_quantile | median_income_quantile | gini_index_quantile | income_per_capita_quantile | housing_units_quantile | vacant_housing_units_quantile | vacant_housing_units_for_rent_quantile | vacant_housing_units_for_sale_quantile | median_rent_quantile | percent_income_spent_on_rent_quantile | owner_occupied_housing_units_quantile | million_dollar_housing_units_quantile ---------------+--------------------+-------------------+---------------------+---------------------+--------------------+--------------------+--------------------+-----------------------+-----------------------------+------------------------------+-------------------------------------+---------------------------------------------+---------------------------+------------------------------------------+-------------------------+-------------------------+-------------------+---------------------+----------------------------------+--------------------+---------------------------+---------------------------+----------------------------+-------------------------------+----------------------------+------------------------------+---------------------------+-------------------------+---------------------------+-------------------------------------+--------------------------------+----------------------------------------+------------------+------------------------+---------------------+----------------------------+------------------------+-------------------------------+----------------------------------------+----------------------------------------+----------------------+---------------------------------------+---------------------------------------+--------------------------------------- - | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | + null_island_segmentation +-------------------------- + t (1 row) getcategories_at_test_point_1 | getcategories_at_test_point_2 diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index ee6c46c..5675ca7 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -121,18 +121,16 @@ SELECT )[1]->>'value' is null as OBS_GetPolygons_for_null_island; -SELECT * FROM - cdb_observatory.OBS_GetSegmentSnapshot( +SELECT cdb_observatory.OBS_GetSegmentSnapshot( cdb_observatory._TestPoint(), '"us.census.tiger".census_tract' -); +)::text = '{"segment_name":"SS_segment_10_clusters","\"us.census.acs\".B01001001_quantile":"0.234783783783784","\"us.census.acs\".B01001002_quantile":"0.422405405405405","\"us.census.acs\".B01001026_quantile":"0.0987567567567568","\"us.census.acs\".B01002001_quantile":"0.0715","\"us.census.acs\".B03002003_quantile":"0.295310810810811","\"us.census.acs\".B03002004_quantile":"0.407189189189189","\"us.census.acs\".B03002006_quantile":"0.625608108108108","\"us.census.acs\".B03002012_quantile":"0.795202702702703","\"us.census.acs\".B05001006_quantile":"0.703797297297297","\"us.census.acs\".B08006001_quantile":"0.59227027027027","\"us.census.acs\".B08006002_quantile":"0.0180540540540541","\"us.census.acs\".B08006008_quantile":"0.993756756756757","\"us.census.acs\".B08006009_quantile":"0.728162162162162","\"us.census.acs\".B08006011_quantile":"0.995972972972973","\"us.census.acs\".B08006015_quantile":"0.929135135135135","\"us.census.acs\".B08006017_quantile":"0.625432432432432","\"us.census.acs\".B09001001_quantile":"0.0386081081081081","\"us.census.acs\".B11001001_quantile":"0.157121621621622","\"us.census.acs\".B14001001_quantile":"0.241878378378378","\"us.census.acs\".B14001002_quantile":"0.173783783783784","\"us.census.acs\".B14001005_quantile":"0.0380675675675676","\"us.census.acs\".B14001006_quantile":"0.0308108108108108","\"us.census.acs\".B14001007_quantile":"0.0486216216216216","\"us.census.acs\".B14001008_quantile":"0.479743243243243","\"us.census.acs\".B15003001_quantile":"0.297675675675676","\"us.census.acs\".B15003017_quantile":"0.190351351351351","\"us.census.acs\".B15003022_quantile":"0.802513513513514","\"us.census.acs\".B15003023_quantile":"0.757148648648649","\"us.census.acs\".B16001001_quantile":"0.255405405405405","\"us.census.acs\".B16001002_quantile":"0.196094594594595","\"us.census.acs\".B16001003_quantile":"0.816851351351351","\"us.census.acs\".B17001001_quantile":"0.252513513513514","\"us.census.acs\".B17001002_quantile":"0.560054054054054","\"us.census.acs\".B19013001_quantile":"0.777472972972973","\"us.census.acs\".B19083001_quantile":"0.336932432432432","\"us.census.acs\".B19301001_quantile":"0.655378378378378","\"us.census.acs\".B25001001_quantile":"0.141810810810811","\"us.census.acs\".B25002003_quantile":"0.362824324324324","\"us.census.acs\".B25004002_quantile":"0.463837837837838","\"us.census.acs\".B25004004_quantile":"0","\"us.census.acs\".B25058001_quantile":"0.939040540540541","\"us.census.acs\".B25071001_quantile":"0.419445945945946","\"us.census.acs\".B25075001_quantile":"0.0387972972972973","\"us.census.acs\".B25075025_quantile":"0"}' as test_point_segmentation; -- segmentation around null island -SELECT * FROM - cdb_observatory.OBS_GetSegmentSnapshot( +SELECT cdb_observatory.OBS_GetSegmentSnapshot( CDB_LatLng(0, 0), '"us.census.tiger".census_tract' -); +)::text = '{"segment_name":null,"\"us.census.acs\".B01001001_quantile":null,"\"us.census.acs\".B01001002_quantile":null,"\"us.census.acs\".B01001026_quantile":null,"\"us.census.acs\".B01002001_quantile":null,"\"us.census.acs\".B03002003_quantile":null,"\"us.census.acs\".B03002004_quantile":null,"\"us.census.acs\".B03002006_quantile":null,"\"us.census.acs\".B03002012_quantile":null,"\"us.census.acs\".B05001006_quantile":null,"\"us.census.acs\".B08006001_quantile":null,"\"us.census.acs\".B08006002_quantile":null,"\"us.census.acs\".B08006008_quantile":null,"\"us.census.acs\".B08006009_quantile":null,"\"us.census.acs\".B08006011_quantile":null,"\"us.census.acs\".B08006015_quantile":null,"\"us.census.acs\".B08006017_quantile":null,"\"us.census.acs\".B09001001_quantile":null,"\"us.census.acs\".B11001001_quantile":null,"\"us.census.acs\".B14001001_quantile":null,"\"us.census.acs\".B14001002_quantile":null,"\"us.census.acs\".B14001005_quantile":null,"\"us.census.acs\".B14001006_quantile":null,"\"us.census.acs\".B14001007_quantile":null,"\"us.census.acs\".B14001008_quantile":null,"\"us.census.acs\".B15003001_quantile":null,"\"us.census.acs\".B15003017_quantile":null,"\"us.census.acs\".B15003022_quantile":null,"\"us.census.acs\".B15003023_quantile":null,"\"us.census.acs\".B16001001_quantile":null,"\"us.census.acs\".B16001002_quantile":null,"\"us.census.acs\".B16001003_quantile":null,"\"us.census.acs\".B17001001_quantile":null,"\"us.census.acs\".B17001002_quantile":null,"\"us.census.acs\".B19013001_quantile":null,"\"us.census.acs\".B19083001_quantile":null,"\"us.census.acs\".B19301001_quantile":null,"\"us.census.acs\".B25001001_quantile":null,"\"us.census.acs\".B25002003_quantile":null,"\"us.census.acs\".B25004002_quantile":null,"\"us.census.acs\".B25004004_quantile":null,"\"us.census.acs\".B25058001_quantile":null,"\"us.census.acs\".B25071001_quantile":null,"\"us.census.acs\".B25075001_quantile":null,"\"us.census.acs\".B25075025_quantile":null}' as null_island_segmentation; WITH result as ( SELECT array_agg(_OBS_GetCategories) as expected FROM