From bbc6f9ef36bff1e486fc3c05c214b2308cc0632a Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 12:20:01 -0400 Subject: [PATCH] getmeasure bypassing several older functions, areas not yet implemented --- src/pg/sql/40_observatory_utility.sql | 97 ----------------- src/pg/sql/41_observatory_augmentation.sql | 100 ++++++++++++++---- .../expected/40_observatory_utility_test.out | 9 -- .../test/sql/40_observatory_utility_test.sql | 32 ------ 4 files changed, 77 insertions(+), 161 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index de942b5..3acec90 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -37,81 +37,6 @@ END; $$ LANGUAGE plpgsql; - --- A function that gets the column data for multiple columns --- Old: OBS_GetColumnData -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData( - geometry_id text, - column_ids text[], - timespan text -) -RETURNS SETOF JSON -AS $$ -BEGIN - - -- figure out highest-weight geometry_id/timespan pair for the first data column - -- TODO this should be done for each data column separately - IF geometry_id IS NULL OR timespan IS NULL THEN - EXECUTE ' - SELECT data_t.timespan timespan, geom_c.id boundary_id - FROM observatory.obs_table data_t, - observatory.obs_column_table data_ct, - observatory.obs_column data_c, - observatory.obs_column_table geoid_ct, - observatory.obs_column_to_column c2c, - observatory.obs_column geom_c - WHERE data_c.id = $2 - AND data_ct.column_id = data_c.id - AND data_ct.table_id = data_t.id - AND geoid_ct.table_id = data_t.id - AND geoid_ct.column_id = c2c.source_id - AND c2c.reltype = ''geom_ref'' - AND geom_c.id = c2c.target_id - AND CASE WHEN $3 IS NULL THEN True ELSE $3 = timespan END - AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END - ORDER BY geom_c.weight DESC, - data_t.timespan DESC - LIMIT 1 - ' INTO timespan, geometry_id - USING geometry_id, (column_ids)[1], timespan; - END IF; - - RETURN QUERY - EXECUTE ' - WITH geomref AS ( - SELECT ct.table_id id - FROM observatory.OBS_column_to_column c2c, - observatory.OBS_column_table ct - WHERE c2c.reltype = ''geom_ref'' - AND c2c.target_id = $1 - AND c2c.source_id = ct.column_id - ), - column_ids as ( - select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a - ) - SELECT row_to_json(a) from ( - select colname, - tablename, - aggregate, - name, - type, - c.description, - $1 AS boundary_id - FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t - WHERE column_ids.column_id = c.id - AND c.id = ct.column_id - AND t.id = ct.table_id - AND t.timespan = $3 - AND t.id in (SELECT id FROM geomref) - order by column_ids.no - ) a - ' - USING geometry_id, column_ids, timespan - RETURN; - -END; -$$ LANGUAGE plpgsql; - --Test point cause Stuart always seems to make random points in the water CREATE OR REPLACE FUNCTION cdb_observatory._TestPoint() RETURNS geometry(Point, 4326) @@ -158,28 +83,6 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetRelatedColumn(columns_ids text[], reltype text ) -RETURNS TEXT[] -AS $$ -DECLARE - result TEXT[]; -BEGIN - EXECUTE ' - With ids as ( - select row_number() over() as no, id from (select unnest($1) as id) t - ) - select array_agg(target_id order by no) - FROM ids - LEFT JOIN observatory.obs_column_to_column - on source_id = id - where reltype = $2 or reltype is null - ' - INTO result - using columns_ids, reltype; - return result; -END; -$$ LANGUAGE plpgsql; - -- Function that replaces all non digits or letters with _ trims and lowercases the -- passed measure name diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3c1e552..38fb503 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -345,37 +345,91 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure( RETURNS NUMERIC AS $$ DECLARE + map_type TEXT; + numer_aggregate TEXT; + numer_colname TEXT; + numer_geomref_colname TEXT; + numer_tablename TEXT; + denom_colname TEXT; + denom_geomref_colname TEXT; + denom_tablename TEXT; + geom_colname TEXT; + geom_geomref_colname TEXT; + geom_tablename TEXT; result NUMERIC; - measure_ids TEXT[]; - denominator_id TEXT; - vals NUMERIC[]; + sql TEXT; BEGIN - IF normalize ILIKE 'area' THEN - measure_ids := ARRAY[measure_id]; - ELSIF normalize ILIKE 'denominator' THEN - EXECUTE 'SELECT (cdb_observatory._OBS_GetRelatedColumn(ARRAY[$1], ''denominator''))[1] - ' INTO denominator_id - USING measure_id; - measure_ids := ARRAY[measure_id, denominator_id]; - ELSIF normalize ILIKE 'none' THEN - -- TODO we need a switch on obs_get to disable area normalization - RAISE EXCEPTION 'No normalization not yet supported.'; + EXECUTE + $query$ + SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, + denom_colname, denom_geomref_colname, denom_tablename, + geom_colname, geom_geomref_colname, geom_tablename + FROM observatory.obs_meta + WHERE (geom_id = $1 OR ($1 = '')) + AND numer_id = $2 + AND (numer_timespan = $3 OR ($3 = '')) + ORDER BY geom_weight DESC, numer_timespan DESC + LIMIT 1 + $query$ + INTO numer_colname, numer_geomref_colname, numer_table, + denom_colname, denom_geomref_colname, denom_table, + geom_colname, geom_geomref_colname, geom_table + USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); + + IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN + -- area normalized + map_type := 'areaNormalized'; + ELSIF normalize ILIKE 'denominator' AND numer_aggregate ILIKE 'sum' THEN + -- denominated + map_type := 'denominated'; ELSE - RAISE EXCEPTION 'Only valid inputs for "normalize" are "area" (default) and "denominator".'; + -- predenominated + map_type := 'predenominated'; END IF; - EXECUTE ' - SELECT ARRAY_AGG(val) FROM (SELECT (cdb_observatory._OBS_Get($1, $2, $3, $4)->>''value'')::NUMERIC val) b - ' - INTO vals - USING geom, measure_ids, time_span, boundary_id; - - IF normalize ILIKE 'denominator' THEN - RETURN (vals)[1]/(vals)[2]; + IF ST_GeometryType(geom) = 'ST_Point' THEN + IF map_type = 'areaNormalied' THEN + sql = format('SELECT numer.%I / (ST_Area(geom.%I::Geography) * 1000000) + FROM observatory.%I numer, observatory.%I geom + WHERE numer.%I = geom.%I + AND ST_WITHIN(%L, geom.%I)', + numer_colname, geom_colname, numer_tablename, + geom_tablename, numer_geomref_colname, + geom_geomref_colname, geom, geom_colname); + ELSIF map_type = 'denominated' THEN + sql = format('SELECT numer.%I / denom.%I + FROM observatory.%I numer, observatory.%I geom, observatory.%I denom + WHERE numer.%I = geom.%I + AND geom.%I = denom.%I + AND ST_WITHIN(%L, geom.%I)', + numer_colname, denom_colname, numer_tablename, + geom_tablename, denom_tablename, + numer_geomref_colname, geom_geomref_colname, + geom_geomref_colname, denom_geomref_colname, + geom, geom_colname); + ELSIF map_type = 'predenominated' THEN + sql = format('SELECT numer.%I + FROM observatory.%I numer, observatory.%I geom + WHERE numer.%I = geom.%I + AND ST_WITHIN(%L, geom.%I)', + numer_colname, numer_tablename, + geom_tablename, numer_geomref_colname, + geom_geomref_colname, geom, geom_colname); + END IF; + ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + IF map_type = 'areaNormalied' THEN + ELSIF map_type = 'denominated' THEN + ELSIF map_type = 'predenominated' THEN + END IF; ELSE - RETURN (vals)[1]; + RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', + ST_GeometryType(geom); END IF; + + EXECUTE sql INTO result; + RETURN result; + END; $$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 9203060..1a04d17 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -9,21 +9,12 @@ t _obs_geomtable_with_null_response t (1 row) -test_get_obs_column_with_geoid_and_census_1|test_get_obs_column_with_geoid_and_census_2 -t|t -(1 row) -obs_getcolumndata_missing_measure -t -(1 row) _obs_buildsnapshotquery_test_1 t (1 row) _obs_buildsnapshotquery_test_2 t (1 row) -_obs_getrelatedcolumn_test -t -(1 row) _obs_standardizemeasurename_test t (1 row) diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 769f0fb..2324cb5 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -29,29 +29,6 @@ SELECT -- 'us.census.tiger.census_tract' -- ); -WITH result as ( -SELECT - array_agg(a) expected from cdb_observatory._OBS_GetColumnData( - 'us.census.tiger.census_tract', - Array['us.census.spielman_singleton_segments.X55', 'us.census.acs.B01003001'], - '2010 - 2014') a -) -select -(expected)[1]::text = '{"colname":"x55","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"name":"Spielman-Singleton Segments: 55 Clusters","type":"Text","description":"Sociodemographic classes from Spielman and Singleton 2015, 55 clusters","boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_1, -(expected)[2]::text = '{"colname":"total_pop","tablename":"obs_b393b5b88c6adda634b2071a8005b03c551b609a","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_2 -from result; - --- should be null-valued -WITH result as ( -SELECT - array_agg(a) expected from cdb_observatory._OBS_GetColumnData( - 'us.census.tiger.census_tract', - Array['us.census.tiger.baloney'], - '2010 - 2014') a -) -select expected is null as OBS_GetColumnData_missing_measure -from result; - -- OBS_BuildSnapshotQuery -- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age SELECT @@ -65,15 +42,6 @@ SELECT Array['mandarin_orange'] ) = 'SELECT vals[1] As mandarin_orange' As _OBS_BuildSnapshotQuery_test_2; -SELECT cdb_observatory._OBS_GetRelatedColumn( - Array[ - 'es.ine.t3_1', - 'us.census.acs.B01003001', - 'us.census.acs.B01001002' - ], - 'denominator' - ) = '{es.ine.t1_1,NULL,us.census.acs.B01003001}' As _OBS_GetRelatedColumn_test; - -- should give back a standardized measure name SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}') = 'test_343_2_qqq' As _OBS_StandardizeMeasureName_test;