cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into docs-768-obs-docs-structure

Browse Source
This commit is contained in:
Andy Eschbacher
2016-05-13 09:36:46 -04:00
parent d99312aa06 3230a9996b
commit 7e7f572bf1
10 changed files with 13775 additions and 10708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
scripts/generate_fixtures.py Normal file
View File

@@ -0,0 +1,84 @@
from sqldumpr import Dumpr
def get_tablename_query(column_id, boundary_id, timespan):
"""
given a column_id, boundary-id (us.census.tiger.block_group), and
timespan, give back the current table hash from the data observatory
"""
q = """
SELECT t.tablename, geoid_ct.colname colname
FROM obs_table t,
obs_column_table geoid_ct,
obs_column_table data_ct
WHERE
t.id = geoid_ct.table_id AND
t.id = data_ct.table_id AND
geoid_ct.column_id =
(SELECT source_id
FROM obs_column_to_column
WHERE target_id = '{boundary_id}'
AND reltype = 'geom_ref'
) AND
data_ct.column_id = '{column_id}' AND
timespan = '{timespan}'
""".replace('\n','')
return q.format(column_id=column_id,
boundary_id=boundary_id,
timespan=timespan)
def select_star(tablename):
return "SELECT * FROM {}".format(tablename)
cdb = Dumpr('observatory.cartodb.com','')
metadata = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag', 'obs_tag', 'obs_column_to_column']
fixtures = [
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'),
('us.census.tiger.block_group', 'us.census.tiger.block_group', '2014'),
('us.census.tiger.zcta5', 'us.census.tiger.zcta5', '2014'),
('us.census.tiger.county', 'us.census.tiger.county', '2014'),
('us.census.acs.B01001001', 'us.census.tiger.census_tract', '2009 - 2013'),
('us.census.acs.B01001001_quantile', 'us.census.tiger.census_tract', '2009 - 2013'),
('us.census.acs.B01001001', 'us.census.tiger.block_group', '2009 - 2013'),
('us.census.acs.B01001001', 'us.census.tiger.block_group', '2010 - 2014'),
('us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2009 - 2013'),
('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2014-01'),
('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2016-03'),
]
unique_tables = set()
for f in fixtures:
column_id, boundary_id, timespan = f
tablename_query = get_tablename_query(*f)
resp = cdb.query(tablename_query).json()['rows'][0]
tablename = resp['tablename']
colname = resp['colname']
table_colname = (tablename, colname, boundary_id, )
if table_colname not in unique_tables:
print table_colname
unique_tables.add(table_colname)
print unique_tables
with open('src/pg/test/fixtures/load_fixtures.sql', 'w') as outfile:
with open('src/pg/test/fixtures/drop_fixtures.sql', 'w') as dropfiles:
outfile.write('SET client_min_messages TO WARNING;\n\set ECHO none\n')
dropfiles.write('SET client_min_messages TO WARNING;\n\set ECHO none\n')
for tablename in metadata:
cdb.dump(select_star(tablename), tablename, outfile, schema='observatory')
dropfiles.write('DROP TABLE IF EXISTS observatory.{};\n'.format(tablename))
print tablename
for tablename, colname, boundary_id in unique_tables:
if 'zcta5' in boundary_id:
where = '11%'
else:
where = '36047%'
print ' '.join([select_star(tablename), "WHERE {} LIKE '{}'".format(colname, where)])
cdb.dump(' '.join([select_star(tablename), "WHERE {} LIKE '{}'".format(colname, where)]),
tablename, outfile, schema='observatory')
dropfiles.write('DROP TABLE IF EXISTS observatory.{};\n'.format(tablename))

38
src/pg/sql/40_observatory_utility.sql
View File

@@ -48,14 +48,43 @@ CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData(
RETURNS SETOF JSON
AS $$
BEGIN
-- figure out highest-weight geometry_id/timespan pair for the first data column
-- TODO this should be done for each data column separately
IF geometry_id IS NULL OR timespan IS NULL THEN
EXECUTE '
SELECT data_t.timespan timespan, geom_c.id boundary_id
FROM observatory.obs_table data_t,
observatory.obs_column_table data_ct,
observatory.obs_column data_c,
observatory.obs_column_table geoid_ct,
observatory.obs_column_to_column c2c,
observatory.obs_column geom_c
WHERE data_c.id = $2
AND data_ct.column_id = data_c.id
AND data_ct.table_id = data_t.id
AND geoid_ct.table_id = data_t.id
AND geoid_ct.column_id = c2c.source_id
AND c2c.reltype = ''geom_ref''
AND geom_c.id = c2c.target_id
AND CASE WHEN $3 IS NULL THEN True ELSE $3 = timespan END
AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END
ORDER BY geom_c.weight DESC,
data_t.timespan DESC
LIMIT 1
' INTO timespan, geometry_id
USING geometry_id, (column_ids)[1], timespan;
END IF;
RETURN QUERY
EXECUTE '
WITH geomref AS (
SELECT t.table_id id
FROM observatory.OBS_column_to_column c2c, observatory.OBS_column_table t
SELECT ct.table_id id
FROM observatory.OBS_column_to_column c2c,
observatory.OBS_column_table ct
WHERE c2c.reltype = ''geom_ref''
AND c2c.target_id = $1
AND c2c.source_id = t.column_id
AND c2c.source_id = ct.column_id
),
column_ids as (
select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a
@@ -66,7 +95,8 @@ BEGIN
aggregate,
name,
type,
c.description
c.description,
$1 AS boundary_id
FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t
WHERE column_ids.column_id = c.id
AND c.id = ct.column_id

79
src/pg/sql/41_observatory_augmentation.sql
View File

@@ -155,20 +155,29 @@ DECLARE
data_table_info json[];
BEGIN
geom_table_name := cdb_observatory._OBS_GeomTable(geom, geometry_level);
IF geom_table_name IS NULL
THEN
RAISE NOTICE 'Point % is outside of the data region', ST_AsText(geom);
RETURN QUERY SELECT '{}'::text[], '{}'::NUMERIC[];
END IF;
EXECUTE
'SELECT array_agg(_obs_getcolumndata)
FROM cdb_observatory._OBS_GetColumnData($1, $2, $3);'
INTO data_table_info
USING geometry_level, column_ids, time_span;
IF geometry_level IS NULL THEN
geometry_level = data_table_info[1]->>'boundary_id';
END IF;
geom_table_name := cdb_observatory._OBS_GeomTable(geom, geometry_level);
IF geom_table_name IS NULL
THEN
RAISE NOTICE 'Point % is outside of the data region', ST_AsText(geom);
-- TODO this should return JSON
RETURN QUERY SELECT '{}'::text[], '{}'::NUMERIC[];
END IF;
IF data_table_info IS NULL THEN
RAISE NOTICE 'Cannot find data table for boundary ID %, column_ids %, and time_span %', geometry_level, column_ids, time_span;
END IF;
IF ST_GeometryType(geom) = 'ST_Point'
THEN
RAISE NOTICE 'geom_table_name %, data_table_info %', geom_table_name, data_table_info::json[];
@@ -198,7 +207,7 @@ $$ LANGUAGE plpgsql;
-- otherwise normalize it to the census block area and return that
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetPoints(
geom geometry(Geometry, 4326),
geom_table_name text,
geom_table_name text, -- TODO: change to boundary_id
data_table_info json[]
)
RETURNS json[]
@@ -209,14 +218,42 @@ DECLARE
query text;
i int;
geoid text;
data_geoid_colname text;
geom_geoid_colname text;
area NUMERIC;
BEGIN
-- TODO: does 'geoid' need to be generalized to geom_ref??
-- TODO we're assuming our geom_table has only one geom_ref column
-- we *really* should pass in both geom_table_name and boundary_id
-- TODO tablename should not be passed here (use boundary_id)
EXECUTE
format('SELECT geoid
FROM observatory.%I
WHERE ST_Within($1, the_geom)',
format('SELECT ct.colname
FROM observatory.obs_column_to_column c2c,
observatory.obs_column_table ct,
observatory.obs_table t
WHERE c2c.reltype = ''geom_ref''
AND ct.column_id = c2c.source_id
AND ct.table_id = t.id
AND t.tablename = %L'
, (data_table_info)[1]->>'tablename')
INTO data_geoid_colname;
EXECUTE
format('SELECT ct.colname
FROM observatory.obs_column_to_column c2c,
observatory.obs_column_table ct,
observatory.obs_table t
WHERE c2c.reltype = ''geom_ref''
AND ct.column_id = c2c.source_id
AND ct.table_id = t.id
AND t.tablename = %L'
, geom_table_name)
INTO geom_geoid_colname;
EXECUTE
format('SELECT %I
FROM observatory.%I
WHERE ST_Within($1, the_geom)',
geom_geoid_colname,
geom_table_name)
USING geom
INTO geoid;
@@ -226,8 +263,9 @@ BEGIN
EXECUTE
format('SELECT ST_Area(the_geom::geography) / (1000 * 1000)
FROM observatory.%I
WHERE geoid = %L',
WHERE %I = %L',
geom_table_name,
geom_geoid_colname,
geoid)
INTO area;
@@ -262,10 +300,11 @@ BEGIN
query := query || format(' ]::numeric[]
FROM observatory.%I
WHERE %I.geoid = %L
WHERE %I.%I = %L
',
((data_table_info)[1])->>'tablename',
((data_table_info)[1])->>'tablename',
data_geoid_colname,
geoid
);
@@ -310,16 +349,6 @@ DECLARE
vals NUMERIC[];
BEGIN
IF boundary_id IS NULL THEN
-- TODO we should determine best boundary for this geom
boundary_id := 'us.census.tiger.block_group';
END IF;
IF time_span IS NULL THEN
-- TODO we should determine latest timespan for this measure
time_span := '2010 - 2014';
END IF;
IF normalize ILIKE 'area' THEN
measure_ids := ARRAY[measure_id];
ELSIF normalize ILIKE 'denominator' THEN

11
src/pg/sql/42_observatory_exploration.sql
View File

@@ -89,14 +89,15 @@ DECLARE
timespan_query TEXT DEFAULT '';
BEGIN
IF time_span != null THEN
timespan_query = format('AND timespan = %L', time_span);
IF timespan != NULL
THEN
timespan_query = format('AND timespan = %L', timespan);
END IF;
RETURN QUERY
EXECUTE
$string$
select
SELECT
column_id,
obs_column.description,
timespan,
@@ -109,11 +110,11 @@ BEGIN
observatory.OBS_column_table.column_id = observatory.obs_column.id AND
observatory.OBS_column_table.table_id = observatory.obs_table.id
AND
observatory.OBS_column.type='Geometry'
observatory.OBS_column.type = 'Geometry'
AND
$1 && bounds::box2d
$string$ || timespan_query
USING geom
USING geom;
RETURN;
END
$$ LANGUAGE plpgsql;

49
src/pg/sql/44_observatory_geometries.sql
View File

@@ -72,9 +72,9 @@ BEGIN
-- return the first boundary in intersections
EXECUTE format(
'SELECT t.the_geom
FROM observatory.%s As t
WHERE ST_Intersects($1, t.the_geom)
'SELECT the_geom
FROM observatory.%I
WHERE ST_Intersects($1, the_geom)
LIMIT 1', target_table)
INTO boundary
USING geom;
@@ -113,6 +113,7 @@ AS $$
DECLARE
output_id text;
target_table text;
geoid_colname text;
BEGIN
-- If not point, raise error
@@ -139,21 +140,33 @@ BEGIN
LIMIT 1;
END IF;
-- if no tables are found, raise error
-- if no tables are found, raise notice and return null
IF target_table IS NULL
THEN
RAISE NOTICE 'Warning: No boundaries found for ''%''', boundary_id;
RETURN NULL::text;
END IF;
RAISE NOTICE 'target_table: %', target_table;
EXECUTE
format('SELECT ct.colname
FROM observatory.obs_column_to_column c2c,
observatory.obs_column_table ct,
observatory.obs_table t
WHERE c2c.reltype = ''geom_ref''
AND ct.column_id = c2c.source_id
AND ct.table_id = t.id
AND t.tablename = %L'
, target_table)
INTO geoid_colname;
RAISE NOTICE 'target_table: %, geoid_colname: %', target_table, geoid_colname;
-- return name of geometry id column
EXECUTE format(
'SELECT t.geoid
FROM observatory.%s As t
WHERE ST_Intersects($1, t.the_geom)
LIMIT 1', target_table)
'SELECT %I
FROM observatory.%I
WHERE ST_Intersects($1, the_geom)
LIMIT 1', geoid_colname, target_table)
INTO output_id
USING geom;
@@ -210,9 +223,9 @@ BEGIN
-- retrieve boundary
EXECUTE
format(
'SELECT t.%s
FROM observatory.%I As t
WHERE t.%s = $1
'SELECT %I
FROM observatory.%I
WHERE %I = $1
LIMIT 1', geom_colname, target_table, geoid_colname)
INTO boundary
USING geometry_id;
@@ -268,9 +281,9 @@ BEGIN
-- return first boundary in intersections
RETURN QUERY
EXECUTE format(
'SELECT t.%s, t.%s
FROM observatory.%s As t
WHERE ST_%s($1, t.the_geom)
'SELECT %I, %I
FROM observatory.%I
WHERE ST_%s($1, the_geom)
', geom_colname, geoid_colname, target_table, overlap_type)
USING geom;
@@ -410,9 +423,9 @@ BEGIN
-- return first boundary in intersections
RETURN QUERY
EXECUTE format(
'SELECT ST_PointOnSurface(t.%s) As %s, t.%s
FROM observatory.%s As t
WHERE ST_%s($1, t.the_geom)
'SELECT ST_PointOnSurface(%I) As %s, %I
FROM observatory.%I
WHERE ST_%s($1, the_geom)
', geom_colname, geom_colname, geoid_colname, target_table, overlap_type)
USING geom;

6
src/pg/test/expected/41_observatory_augmentation_test.out
View File

@@ -34,6 +34,12 @@ t
getcategories_at_null_island
t
(1 row)
obs_getmeasure_zhvi_point_test
t
(1 row)
obs_getmeasure_zhvi_point_default_latest_test
t
(1 row)
obs_getmeasure_total_pop_point_test
t
(1 row)

13
src/pg/test/fixtures/drop_fixtures.sql vendored
View File

@@ -6,11 +6,14 @@ DROP TABLE IF EXISTS observatory.obs_column;
DROP TABLE IF EXISTS observatory.obs_column_tag;
DROP TABLE IF EXISTS observatory.obs_tag;
DROP TABLE IF EXISTS observatory.obs_column_to_column;
DROP TABLE IF EXISTS observatory.obs_1babf5a26a1ecda5fb74963e88408f71d0364b81;
DROP TABLE IF EXISTS observatory.obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb;
DROP TABLE IF EXISTS observatory.obs_ab038198aaab3f3cb055758638ee4de28ad70146;
DROP TABLE IF EXISTS observatory.obs_d34555209878e8c4b37cf0b2b3d072ff129ec470;
DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7;
DROP TABLE IF EXISTS observatory.obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d;
DROP TABLE IF EXISTS observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd;
DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7;
DROP TABLE IF EXISTS observatory.obs_1babf5a26a1ecda5fb74963e88408f71d0364b81;
DROP TABLE IF EXISTS observatory.obs_8764a6b439a4f8714f54d4b3a157bc5e36519066;
DROP TABLE IF EXISTS observatory.obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb;
DROP TABLE IF EXISTS observatory.obs_d34555209878e8c4b37cf0b2b3d072ff129ec470;
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;
DROP TABLE IF EXISTS observatory.obs_144e8b4f906885b2e057ac4842644a553ae49c6e;
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
DROP TABLE IF EXISTS observatory.obs_ab038198aaab3f3cb055758638ee4de28ad70146;

24187
src/pg/test/fixtures/load_fixtures.sql vendored
View File

File diff suppressed because one or more lines are too long

4
src/pg/test/sql/40_observatory_utility_test.sql
View File

@@ -40,8 +40,8 @@ SELECT
Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'],
'2009 - 2013') a
)
select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null}' as test_get_obs_column_with_geoid_and_census_1,
(expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null}' as test_get_obs_column_with_geoid_and_census_2
select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null,"boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_1,
(expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null,"boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_2
from result;
-- should be null-valued

12
src/pg/test/sql/41_observatory_augmentation_test.sql
View File

@@ -133,6 +133,18 @@ WITH result as (
select expected[0] is NULL as GetCategories_at_null_island
from result;
-- Point-based OBS_GetMeasure with zillow
SELECT abs(OBS_GetMeasure_zhvi_point - 583600) / 583600 < 0.001 AS OBS_GetMeasure_zhvi_point_test FROM cdb_observatory.OBS_GetMeasure(
ST_SetSRID(ST_Point(-73.94602417945862, 40.6768220087458), 4326),
'us.zillow.AllHomes_Zhvi', 'area', 'us.census.tiger.zcta5', '2014-01'
) As t(OBS_GetMeasure_zhvi_point);
-- Point-based OBS_GetMeasure with zillow default to latest
SELECT abs(OBS_GetMeasure_zhvi_point_default_latest - 972900) / 972900 < 0.001 AS OBS_GetMeasure_zhvi_point_default_latest_test FROM cdb_observatory.OBS_GetMeasure(
ST_SetSRID(ST_Point(-73.94602417945862, 40.6768220087458), 4326),
'us.zillow.AllHomes_Zhvi'
) As t(OBS_GetMeasure_zhvi_point_default_latest);
-- Point-based OBS_GetMeasure, default normalization (area)
-- is result within 0.1% of expected
SELECT abs(OBS_GetMeasure_total_pop_point - 10923.093200390833950) / 10923.093200390833950 < 0.001 As OBS_GetMeasure_total_pop_point_test FROM