From bbc6f9ef36bff1e486fc3c05c214b2308cc0632a Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 12:20:01 -0400 Subject: [PATCH 01/26] getmeasure bypassing several older functions, areas not yet implemented --- src/pg/sql/40_observatory_utility.sql | 97 ----------------- src/pg/sql/41_observatory_augmentation.sql | 100 ++++++++++++++---- .../expected/40_observatory_utility_test.out | 9 -- .../test/sql/40_observatory_utility_test.sql | 32 ------ 4 files changed, 77 insertions(+), 161 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index de942b5..3acec90 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -37,81 +37,6 @@ END; $$ LANGUAGE plpgsql; - --- A function that gets the column data for multiple columns --- Old: OBS_GetColumnData -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData( - geometry_id text, - column_ids text[], - timespan text -) -RETURNS SETOF JSON -AS $$ -BEGIN - - -- figure out highest-weight geometry_id/timespan pair for the first data column - -- TODO this should be done for each data column separately - IF geometry_id IS NULL OR timespan IS NULL THEN - EXECUTE ' - SELECT data_t.timespan timespan, geom_c.id boundary_id - FROM observatory.obs_table data_t, - observatory.obs_column_table data_ct, - observatory.obs_column data_c, - observatory.obs_column_table geoid_ct, - observatory.obs_column_to_column c2c, - observatory.obs_column geom_c - WHERE data_c.id = $2 - AND data_ct.column_id = data_c.id - AND data_ct.table_id = data_t.id - AND geoid_ct.table_id = data_t.id - AND geoid_ct.column_id = c2c.source_id - AND c2c.reltype = ''geom_ref'' - AND geom_c.id = c2c.target_id - AND CASE WHEN $3 IS NULL THEN True ELSE $3 = timespan END - AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END - ORDER BY geom_c.weight DESC, - data_t.timespan DESC - LIMIT 1 - ' INTO timespan, geometry_id - USING geometry_id, (column_ids)[1], timespan; - END IF; - - RETURN QUERY - EXECUTE ' - WITH geomref AS ( - SELECT ct.table_id id - FROM observatory.OBS_column_to_column c2c, - observatory.OBS_column_table ct - WHERE c2c.reltype = ''geom_ref'' - AND c2c.target_id = $1 - AND c2c.source_id = ct.column_id - ), - column_ids as ( - select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a - ) - SELECT row_to_json(a) from ( - select colname, - tablename, - aggregate, - name, - type, - c.description, - $1 AS boundary_id - FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t - WHERE column_ids.column_id = c.id - AND c.id = ct.column_id - AND t.id = ct.table_id - AND t.timespan = $3 - AND t.id in (SELECT id FROM geomref) - order by column_ids.no - ) a - ' - USING geometry_id, column_ids, timespan - RETURN; - -END; -$$ LANGUAGE plpgsql; - --Test point cause Stuart always seems to make random points in the water CREATE OR REPLACE FUNCTION cdb_observatory._TestPoint() RETURNS geometry(Point, 4326) @@ -158,28 +83,6 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetRelatedColumn(columns_ids text[], reltype text ) -RETURNS TEXT[] -AS $$ -DECLARE - result TEXT[]; -BEGIN - EXECUTE ' - With ids as ( - select row_number() over() as no, id from (select unnest($1) as id) t - ) - select array_agg(target_id order by no) - FROM ids - LEFT JOIN observatory.obs_column_to_column - on source_id = id - where reltype = $2 or reltype is null - ' - INTO result - using columns_ids, reltype; - return result; -END; -$$ LANGUAGE plpgsql; - -- Function that replaces all non digits or letters with _ trims and lowercases the -- passed measure name diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3c1e552..38fb503 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -345,37 +345,91 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure( RETURNS NUMERIC AS $$ DECLARE + map_type TEXT; + numer_aggregate TEXT; + numer_colname TEXT; + numer_geomref_colname TEXT; + numer_tablename TEXT; + denom_colname TEXT; + denom_geomref_colname TEXT; + denom_tablename TEXT; + geom_colname TEXT; + geom_geomref_colname TEXT; + geom_tablename TEXT; result NUMERIC; - measure_ids TEXT[]; - denominator_id TEXT; - vals NUMERIC[]; + sql TEXT; BEGIN - IF normalize ILIKE 'area' THEN - measure_ids := ARRAY[measure_id]; - ELSIF normalize ILIKE 'denominator' THEN - EXECUTE 'SELECT (cdb_observatory._OBS_GetRelatedColumn(ARRAY[$1], ''denominator''))[1] - ' INTO denominator_id - USING measure_id; - measure_ids := ARRAY[measure_id, denominator_id]; - ELSIF normalize ILIKE 'none' THEN - -- TODO we need a switch on obs_get to disable area normalization - RAISE EXCEPTION 'No normalization not yet supported.'; + EXECUTE + $query$ + SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, + denom_colname, denom_geomref_colname, denom_tablename, + geom_colname, geom_geomref_colname, geom_tablename + FROM observatory.obs_meta + WHERE (geom_id = $1 OR ($1 = '')) + AND numer_id = $2 + AND (numer_timespan = $3 OR ($3 = '')) + ORDER BY geom_weight DESC, numer_timespan DESC + LIMIT 1 + $query$ + INTO numer_colname, numer_geomref_colname, numer_table, + denom_colname, denom_geomref_colname, denom_table, + geom_colname, geom_geomref_colname, geom_table + USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); + + IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN + -- area normalized + map_type := 'areaNormalized'; + ELSIF normalize ILIKE 'denominator' AND numer_aggregate ILIKE 'sum' THEN + -- denominated + map_type := 'denominated'; ELSE - RAISE EXCEPTION 'Only valid inputs for "normalize" are "area" (default) and "denominator".'; + -- predenominated + map_type := 'predenominated'; END IF; - EXECUTE ' - SELECT ARRAY_AGG(val) FROM (SELECT (cdb_observatory._OBS_Get($1, $2, $3, $4)->>''value'')::NUMERIC val) b - ' - INTO vals - USING geom, measure_ids, time_span, boundary_id; - - IF normalize ILIKE 'denominator' THEN - RETURN (vals)[1]/(vals)[2]; + IF ST_GeometryType(geom) = 'ST_Point' THEN + IF map_type = 'areaNormalied' THEN + sql = format('SELECT numer.%I / (ST_Area(geom.%I::Geography) * 1000000) + FROM observatory.%I numer, observatory.%I geom + WHERE numer.%I = geom.%I + AND ST_WITHIN(%L, geom.%I)', + numer_colname, geom_colname, numer_tablename, + geom_tablename, numer_geomref_colname, + geom_geomref_colname, geom, geom_colname); + ELSIF map_type = 'denominated' THEN + sql = format('SELECT numer.%I / denom.%I + FROM observatory.%I numer, observatory.%I geom, observatory.%I denom + WHERE numer.%I = geom.%I + AND geom.%I = denom.%I + AND ST_WITHIN(%L, geom.%I)', + numer_colname, denom_colname, numer_tablename, + geom_tablename, denom_tablename, + numer_geomref_colname, geom_geomref_colname, + geom_geomref_colname, denom_geomref_colname, + geom, geom_colname); + ELSIF map_type = 'predenominated' THEN + sql = format('SELECT numer.%I + FROM observatory.%I numer, observatory.%I geom + WHERE numer.%I = geom.%I + AND ST_WITHIN(%L, geom.%I)', + numer_colname, numer_tablename, + geom_tablename, numer_geomref_colname, + geom_geomref_colname, geom, geom_colname); + END IF; + ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + IF map_type = 'areaNormalied' THEN + ELSIF map_type = 'denominated' THEN + ELSIF map_type = 'predenominated' THEN + END IF; ELSE - RETURN (vals)[1]; + RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', + ST_GeometryType(geom); END IF; + + EXECUTE sql INTO result; + RETURN result; + END; $$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 9203060..1a04d17 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -9,21 +9,12 @@ t _obs_geomtable_with_null_response t (1 row) -test_get_obs_column_with_geoid_and_census_1|test_get_obs_column_with_geoid_and_census_2 -t|t -(1 row) -obs_getcolumndata_missing_measure -t -(1 row) _obs_buildsnapshotquery_test_1 t (1 row) _obs_buildsnapshotquery_test_2 t (1 row) -_obs_getrelatedcolumn_test -t -(1 row) _obs_standardizemeasurename_test t (1 row) diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 769f0fb..2324cb5 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -29,29 +29,6 @@ SELECT -- 'us.census.tiger.census_tract' -- ); -WITH result as ( -SELECT - array_agg(a) expected from cdb_observatory._OBS_GetColumnData( - 'us.census.tiger.census_tract', - Array['us.census.spielman_singleton_segments.X55', 'us.census.acs.B01003001'], - '2010 - 2014') a -) -select -(expected)[1]::text = '{"colname":"x55","tablename":"obs_65f29658e096ca1485bf683f65fdbc9f05ec3c5d","aggregate":null,"name":"Spielman-Singleton Segments: 55 Clusters","type":"Text","description":"Sociodemographic classes from Spielman and Singleton 2015, 55 clusters","boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_1, -(expected)[2]::text = '{"colname":"total_pop","tablename":"obs_b393b5b88c6adda634b2071a8005b03c551b609a","aggregate":"sum","name":"Total Population","type":"Numeric","description":"The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.","boundary_id":"us.census.tiger.census_tract"}' as test_get_obs_column_with_geoid_and_census_2 -from result; - --- should be null-valued -WITH result as ( -SELECT - array_agg(a) expected from cdb_observatory._OBS_GetColumnData( - 'us.census.tiger.census_tract', - Array['us.census.tiger.baloney'], - '2010 - 2014') a -) -select expected is null as OBS_GetColumnData_missing_measure -from result; - -- OBS_BuildSnapshotQuery -- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age SELECT @@ -65,15 +42,6 @@ SELECT Array['mandarin_orange'] ) = 'SELECT vals[1] As mandarin_orange' As _OBS_BuildSnapshotQuery_test_2; -SELECT cdb_observatory._OBS_GetRelatedColumn( - Array[ - 'es.ine.t3_1', - 'us.census.acs.B01003001', - 'us.census.acs.B01001002' - ], - 'denominator' - ) = '{es.ine.t1_1,NULL,us.census.acs.B01003001}' As _OBS_GetRelatedColumn_test; - -- should give back a standardized measure name SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}') = 'test_343_2_qqq' As _OBS_StandardizeMeasureName_test; From 976e119abb48857f24c18f5296c1577c11e7ff02 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 12:21:53 -0400 Subject: [PATCH 02/26] fix typo --- src/pg/sql/41_observatory_augmentation.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 38fb503..62f561f 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -372,9 +372,9 @@ BEGIN ORDER BY geom_weight DESC, numer_timespan DESC LIMIT 1 $query$ - INTO numer_colname, numer_geomref_colname, numer_table, - denom_colname, denom_geomref_colname, denom_table, - geom_colname, geom_geomref_colname, geom_table + INTO numer_colname, numer_geomref_colname, numer_tablename, + denom_colname, denom_geomref_colname, denom_tablename, + geom_colname, geom_geomref_colname, geom_tablename USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN From fb083f4b9e6f169cabc0693512ff753460a9492e Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 13:30:05 -0400 Subject: [PATCH 03/26] fix typo --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 62f561f..1c64f20 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -389,7 +389,7 @@ BEGIN END IF; IF ST_GeometryType(geom) = 'ST_Point' THEN - IF map_type = 'areaNormalied' THEN + IF map_type = 'areaNormalized' THEN sql = format('SELECT numer.%I / (ST_Area(geom.%I::Geography) * 1000000) FROM observatory.%I numer, observatory.%I geom WHERE numer.%I = geom.%I @@ -418,7 +418,7 @@ BEGIN geom_geomref_colname, geom, geom_colname); END IF; ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN - IF map_type = 'areaNormalied' THEN + IF map_type = 'areaNormalized' THEN ELSIF map_type = 'denominated' THEN ELSIF map_type = 'predenominated' THEN END IF; From af2259bb0aee90e691d3becd864d36e2c1420a07 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 13:39:33 -0400 Subject: [PATCH 04/26] fix wrong number of variables INTO --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 1c64f20..ed8a2e3 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -372,7 +372,7 @@ BEGIN ORDER BY geom_weight DESC, numer_timespan DESC LIMIT 1 $query$ - INTO numer_colname, numer_geomref_colname, numer_tablename, + INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, denom_colname, denom_geomref_colname, denom_tablename, geom_colname, geom_geomref_colname, geom_tablename USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); From e89a88aa8359559676017d78e8775fe2a4af89ad Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 15:59:27 -0400 Subject: [PATCH 05/26] use subselects as joins are horrifically slow over FDW --- src/pg/sql/41_observatory_augmentation.sql | 43 +++++++++++----------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index ed8a2e3..cfbb9fe 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -390,38 +390,39 @@ BEGIN IF ST_GeometryType(geom) = 'ST_Point' THEN IF map_type = 'areaNormalized' THEN - sql = format('SELECT numer.%I / (ST_Area(geom.%I::Geography) * 1000000) - FROM observatory.%I numer, observatory.%I geom - WHERE numer.%I = geom.%I - AND ST_WITHIN(%L, geom.%I)', - numer_colname, geom_colname, numer_tablename, - geom_tablename, numer_geomref_colname, - geom_geomref_colname, geom, geom_colname); + sql = format('WITH _geom AS (SELECT ST_Area(geom.%I::Geography) / 1000000 area, geom.%I geom_ref + FROM observatory.%I geom + WHERE ST_Within(%L, geom.%I) + LIMIT 1) + SELECT numer.%I / (SELECT area FROM _geom) + FROM observatory.%I numer + WHERE numer.%I = (SELECT geom_ref FROM _geom)', + geom_colname, geom_geomref_colname, geom_tablename, + geom, geom_colname, numer_colname, numer_tablename, + numer_geomref_colname); ELSIF map_type = 'denominated' THEN - sql = format('SELECT numer.%I / denom.%I - FROM observatory.%I numer, observatory.%I geom, observatory.%I denom - WHERE numer.%I = geom.%I - AND geom.%I = denom.%I - AND ST_WITHIN(%L, geom.%I)', - numer_colname, denom_colname, numer_tablename, - geom_tablename, denom_tablename, + sql = format('SELECT numer.%I / NULLIF((SELECT denom.%I FROM observatory.%I denom WHERE denom.%I = numer.%I LIMIT 1), 0) + FROM observatory.%I numer + WHERE numer.%I = (SELECT geom.%I FROM observatory.%I geom WHERE ST_Within(%L, geom.%I) LIMIT 1)', + numer_colname, denom_colname, denom_tablename, + denom_geomref_colname, numer_geomref_colname, + numer_tablename, numer_geomref_colname, geom_geomref_colname, - geom_geomref_colname, denom_geomref_colname, - geom, geom_colname); + geom_tablename, geom, geom_colname); ELSIF map_type = 'predenominated' THEN sql = format('SELECT numer.%I - FROM observatory.%I numer, observatory.%I geom - WHERE numer.%I = geom.%I - AND ST_WITHIN(%L, geom.%I)', + FROM observatory.%I numer + WHERE numer.%I = (SELECT geom.%I FROM observatory.%I geom WHERE ST_Within(%L, geom.%I) LIMIT 1)', numer_colname, numer_tablename, - geom_tablename, numer_geomref_colname, - geom_geomref_colname, geom, geom_colname); + numer_geomref_colname, geom_geomref_colname, geom_tablename, + geom, geom_colname); END IF; ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN IF map_type = 'areaNormalized' THEN ELSIF map_type = 'denominated' THEN ELSIF map_type = 'predenominated' THEN END IF; + RAISE EXCEPTION 'Polygon/multipolygon TODO'; ELSE RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', ST_GeometryType(geom); From da127baa3c2f2e1fddc5e9fcc948436a837e43a1 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 16:24:33 -0400 Subject: [PATCH 06/26] implementation for polygon/multipolygon weighted getmeasure --- src/pg/sql/41_observatory_augmentation.sql | 54 +++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index cfbb9fe..a45cc74 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -358,13 +358,14 @@ DECLARE geom_tablename TEXT; result NUMERIC; sql TEXT; + numer_name TEXT; BEGIN EXECUTE $query$ SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, denom_colname, denom_geomref_colname, denom_tablename, - geom_colname, geom_geomref_colname, geom_tablename + geom_colname, geom_geomref_colname, geom_tablename, numer_name FROM observatory.obs_meta WHERE (geom_id = $1 OR ($1 = '')) AND numer_id = $2 @@ -374,17 +375,14 @@ BEGIN $query$ INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, denom_colname, denom_geomref_colname, denom_tablename, - geom_colname, geom_geomref_colname, geom_tablename + geom_colname, geom_geomref_colname, geom_tablename, numer_name USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN - -- area normalized map_type := 'areaNormalized'; ELSIF normalize ILIKE 'denominator' AND numer_aggregate ILIKE 'sum' THEN - -- denominated map_type := 'denominated'; ELSE - -- predenominated map_type := 'predenominated'; END IF; @@ -419,10 +417,54 @@ BEGIN END IF; ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN IF map_type = 'areaNormalized' THEN + sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) + / ST_Area(geom.%I) overlap, geom.%I geom_ref + FROM observatory.%I geom + WHERE %L && geom.%I) + SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / + ST_Area(%L::Geography) + FROM observatory.%I numer + WHERE numer.%I IN (SELECT geom_ref FROM _geom)', + geom, geom_colname, geom_colname, + geom_geomref_colname, geom_tablename, + geom, geom_colname, numer_colname, numer_geomref_colname, + geom, numer_tablename, + numer_geomref_colname); ELSIF map_type = 'denominated' THEN + sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) + / ST_Area(geom.%I) overlap, geom.%I geom_ref + FROM observatory.%I geom + WHERE %L && geom.%I), + _denom AS (SELECT denom.%I, denom.%I geom_ref + FROM observatory.%I denom + WHERE %L && denom.%I) + SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / + SUM(denom.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = denom%I)) + FROM observatory.%I numer + WHERE numer.%I IN (SELECT geom_ref FROM _geom)', + geom, geom_colname, geom_colname, + geom_geomref_colname, geom_tablename, + geom, geom_colname, numer_colname, geom, numer_tablename, + numer_geomref_colname); ELSIF map_type = 'predenominated' THEN + IF numer_aggregate NOT ILIKE 'sum' THEN + RAISE EXCEPTION 'Cannot calculate "%" (%) for custom area as it cannot be summed, use ST_PointOnSurface instead', + numer_name, numer_id; + ELSE + sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) + / ST_Area(geom.%I) overlap, geom.%I geom_ref + FROM observatory.%I geom + WHERE %L && geom.%I) + SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) + FROM observatory.%I numer + WHERE numer.%I IN (SELECT geom_ref FROM _geom)', + geom, geom_colname, geom_colname, + geom_geomref_colname, geom_tablename, + geom, geom_colname, numer_colname, numer_geomref_colname, + numer_tablename, + numer_geomref_colname); + END IF; END IF; - RAISE EXCEPTION 'Polygon/multipolygon TODO'; ELSE RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', ST_GeometryType(geom); From a946ab9d03b3c69fae8c75c44a6b05d912a33ea5 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 16:28:50 -0400 Subject: [PATCH 07/26] fix params in denominated polygon getmeasure --- src/pg/sql/41_observatory_augmentation.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index a45cc74..a28d170 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -437,14 +437,17 @@ BEGIN WHERE %L && geom.%I), _denom AS (SELECT denom.%I, denom.%I geom_ref FROM observatory.%I denom - WHERE %L && denom.%I) + WHERE denom.%I IN (SELECT geom_ref FROM _geom)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / - SUM(denom.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = denom%I)) + SUM(denom.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = denom.%I)) FROM observatory.%I numer WHERE numer.%I IN (SELECT geom_ref FROM _geom)', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, - geom, geom_colname, numer_colname, geom, numer_tablename, + geom, geom_colname, + denom_colname, denom_geomref_colname, + denom_tablename, denom_geomref_colname, + numer_colname, geom, numer_tablename, numer_geomref_colname); ELSIF map_type = 'predenominated' THEN IF numer_aggregate NOT ILIKE 'sum' THEN From 7c8c5cca0a719c658a65bc13673e77e82d22ad43 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 16:42:02 -0400 Subject: [PATCH 08/26] fix params --- src/pg/sql/41_observatory_augmentation.sql | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index a28d170..cd0d9ed 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -439,15 +439,22 @@ BEGIN FROM observatory.%I denom WHERE denom.%I IN (SELECT geom_ref FROM _geom)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / - SUM(denom.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = denom.%I)) + SUM(SELECT _denom.%I * (SELECT _geom.overlap + FROM _geom WHERE _geom.geom_ref = _denom.geom_ref) + WHERE _denom.geom_ref = numer.%I) FROM observatory.%I numer WHERE numer.%I IN (SELECT geom_ref FROM _geom)', - geom, geom_colname, geom_colname, - geom_geomref_colname, geom_tablename, + geom, geom_colname, + geom_colname, geom_geomref_colname, + geom_tablename, geom, geom_colname, denom_colname, denom_geomref_colname, - denom_tablename, denom_geomref_colname, - numer_colname, geom, numer_tablename, + denom_tablename, + denom_geomref_colname, + numer_colname, numer_geomref_colname, + denom_colname, + numer_geomref_colname, + numer_tablename, numer_geomref_colname); ELSIF map_type = 'predenominated' THEN IF numer_aggregate NOT ILIKE 'sum' THEN From 80cdc5e8ca5440067ca0f5f309257e239c149f45 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 16:48:06 -0400 Subject: [PATCH 09/26] fix predicate --- src/pg/sql/41_observatory_augmentation.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index cd0d9ed..5ab007a 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -439,9 +439,10 @@ BEGIN FROM observatory.%I denom WHERE denom.%I IN (SELECT geom_ref FROM _geom)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / - SUM(SELECT _denom.%I * (SELECT _geom.overlap - FROM _geom WHERE _geom.geom_ref = _denom.geom_ref) - WHERE _denom.geom_ref = numer.%I) + SUM((SELECT _denom.%I * (SELECT _geom.overlap + FROM _geom + WHERE _geom.geom_ref = _denom.geom_ref) + FROM _denom WHERE _denom.geom_ref = numer.%I)) FROM observatory.%I numer WHERE numer.%I IN (SELECT geom_ref FROM _geom)', geom, geom_colname, From 86529ada5a216e3e254aca860a3c33db9b60f7a6 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 16:53:50 -0400 Subject: [PATCH 10/26] use st_overlaps instead of && --- src/pg/sql/41_observatory_augmentation.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 5ab007a..fb338d0 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -420,7 +420,7 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE %L && geom.%I) + WHERE ST_Overlaps(%L, geom.%I)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / ST_Area(%L::Geography) FROM observatory.%I numer @@ -434,7 +434,7 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE %L && geom.%I), + WHERE ST_Overlaps(%L, geom.%I)), _denom AS (SELECT denom.%I, denom.%I geom_ref FROM observatory.%I denom WHERE denom.%I IN (SELECT geom_ref FROM _geom)) @@ -465,7 +465,7 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE %L && geom.%I) + WHERE ST_Overlaps(%L, geom.%I)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) FROM observatory.%I numer WHERE numer.%I IN (SELECT geom_ref FROM _geom)', From a7de1f222875ba7e08e6636ee1214d77dd0e1a36 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 16:57:27 -0400 Subject: [PATCH 11/26] intersects, not overlaps --- src/pg/sql/41_observatory_augmentation.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index fb338d0..3b2f196 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -420,7 +420,7 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE ST_Overlaps(%L, geom.%I)) + WHERE ST_Intersects(%L, geom.%I)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / ST_Area(%L::Geography) FROM observatory.%I numer @@ -434,7 +434,7 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE ST_Overlaps(%L, geom.%I)), + WHERE ST_Intersects(%L, geom.%I)), _denom AS (SELECT denom.%I, denom.%I geom_ref FROM observatory.%I denom WHERE denom.%I IN (SELECT geom_ref FROM _geom)) @@ -465,7 +465,7 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE ST_Overlaps(%L, geom.%I)) + WHERE ST_Intersects(%L, geom.%I)) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) FROM observatory.%I numer WHERE numer.%I IN (SELECT geom_ref FROM _geom)', From ff173a01520cc16386714a2ecbe8d561f21bbced Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:12:16 -0400 Subject: [PATCH 12/26] filter so theres some overlap --- src/pg/sql/41_observatory_augmentation.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3b2f196..6aae9c8 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -465,7 +465,8 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE ST_Intersects(%L, geom.%I)) + WHERE ST_Intersects(%L, geom.%I) + AND ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) > 0) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) FROM observatory.%I numer WHERE numer.%I IN (SELECT geom_ref FROM _geom)', From c07d9f68337ed023ac807a7b97f05702822df8cb Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:13:45 -0400 Subject: [PATCH 13/26] add missing params --- src/pg/sql/41_observatory_augmentation.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 6aae9c8..eedf71a 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -472,7 +472,9 @@ BEGIN WHERE numer.%I IN (SELECT geom_ref FROM _geom)', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, - geom, geom_colname, numer_colname, numer_geomref_colname, + geom, geom_colname, + geom, geom_colname, geom_colname, + numer_colname, numer_geomref_colname, numer_tablename, numer_geomref_colname); END IF; From fc6317161fcbd11a26e4b1282c516eca13500ee3 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:26:01 -0400 Subject: [PATCH 14/26] avoid joins --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index eedf71a..3388934 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -469,7 +469,7 @@ BEGIN AND ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) > 0) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) FROM observatory.%I numer - WHERE numer.%I IN (SELECT geom_ref FROM _geom)', + WHERE numer.%I = ANY ((SELECT geom_ref FROM _geom)::TEXT[])', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, geom, geom_colname, From 6fa9d5c96a52ad0e684aeccaae3483431a919462 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:27:03 -0400 Subject: [PATCH 15/26] add missing array_agg --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3388934..060169a 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -469,7 +469,7 @@ BEGIN AND ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) > 0) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) FROM observatory.%I numer - WHERE numer.%I = ANY ((SELECT geom_ref FROM _geom)::TEXT[])', + WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, geom, geom_colname, From b690478aff22699f1a427f024f54e56f5da79e2b Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:36:27 -0400 Subject: [PATCH 16/26] use IN ANY to avoid joins elsewhere, and filter by nonzero overlap for all getmeasure polygon queries --- src/pg/sql/41_observatory_augmentation.sql | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 060169a..64880d9 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -420,21 +420,25 @@ BEGIN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE ST_Intersects(%L, geom.%I)) + WHERE ST_Intersects(%L, geom.%I) + AND ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) > 0) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / ST_Area(%L::Geography) FROM observatory.%I numer - WHERE numer.%I IN (SELECT geom_ref FROM _geom)', + WHERE numer.%I = ANY (SELECT ARRAY_AGG(geom_ref) FROM _geom)', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, - geom, geom_colname, numer_colname, numer_geomref_colname, + geom, geom_colname, + geom, geom_colname, geom_colname, + numer_colname, numer_geomref_colname, geom, numer_tablename, numer_geomref_colname); ELSIF map_type = 'denominated' THEN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref FROM observatory.%I geom - WHERE ST_Intersects(%L, geom.%I)), + WHERE ST_Intersects(%L, geom.%I) + AND ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) > 0), _denom AS (SELECT denom.%I, denom.%I geom_ref FROM observatory.%I denom WHERE denom.%I IN (SELECT geom_ref FROM _geom)) @@ -444,11 +448,12 @@ BEGIN WHERE _geom.geom_ref = _denom.geom_ref) FROM _denom WHERE _denom.geom_ref = numer.%I)) FROM observatory.%I numer - WHERE numer.%I IN (SELECT geom_ref FROM _geom)', + WHERE numer.%I = ANY (SELECT ARRAY_AGG(geom_ref) FROM _geom)', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, geom, geom_colname, + geom, geom_colname, geom_colname, denom_colname, denom_geomref_colname, denom_tablename, denom_geomref_colname, From 75e765f2566e7064c1d9a42966de63775a67e427 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:38:05 -0400 Subject: [PATCH 17/26] explicit type casts for = ANY --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 64880d9..3147c39 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -425,7 +425,7 @@ BEGIN SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / ST_Area(%L::Geography) FROM observatory.%I numer - WHERE numer.%I = ANY (SELECT ARRAY_AGG(geom_ref) FROM _geom)', + WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, geom, geom_colname, @@ -448,7 +448,7 @@ BEGIN WHERE _geom.geom_ref = _denom.geom_ref) FROM _denom WHERE _denom.geom_ref = numer.%I)) FROM observatory.%I numer - WHERE numer.%I = ANY (SELECT ARRAY_AGG(geom_ref) FROM _geom)', + WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom::TEXT[])', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, From f4854260854cf6ef4e8ded3d83a7e32e034626ae Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:38:54 -0400 Subject: [PATCH 18/26] fix syntax error --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3147c39..3004784 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -448,7 +448,7 @@ BEGIN WHERE _geom.geom_ref = _denom.geom_ref) FROM _denom WHERE _denom.geom_ref = numer.%I)) FROM observatory.%I numer - WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom::TEXT[])', + WHERE numer.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])', geom, geom_colname, geom_colname, geom_geomref_colname, geom_tablename, From a4a20e9c1d72094b2b4f20b20884c7b2ec2f0e8f Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:41:05 -0400 Subject: [PATCH 19/26] prevent internal join for denominated getmeasure by polygon --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3004784..6140542 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -441,7 +441,7 @@ BEGIN AND ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) > 0), _denom AS (SELECT denom.%I, denom.%I geom_ref FROM observatory.%I denom - WHERE denom.%I IN (SELECT geom_ref FROM _geom)) + WHERE denom.%I = ANY ((SELECT ARRAY_AGG(geom_ref) FROM _geom)::TEXT[])) SELECT SUM(numer.%I * (SELECT _geom.overlap FROM _geom WHERE _geom.geom_ref = numer.%I)) / SUM((SELECT _denom.%I * (SELECT _geom.overlap FROM _geom From 3a37b98b7232c7315aac7f53d774826c16d7934b Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 17:54:19 -0400 Subject: [PATCH 20/26] we still needt hese for getdemographicsnapshot --- src/pg/sql/40_observatory_utility.sql | 97 +++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 3acec90..de942b5 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -37,6 +37,81 @@ END; $$ LANGUAGE plpgsql; + +-- A function that gets the column data for multiple columns +-- Old: OBS_GetColumnData +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData( + geometry_id text, + column_ids text[], + timespan text +) +RETURNS SETOF JSON +AS $$ +BEGIN + + -- figure out highest-weight geometry_id/timespan pair for the first data column + -- TODO this should be done for each data column separately + IF geometry_id IS NULL OR timespan IS NULL THEN + EXECUTE ' + SELECT data_t.timespan timespan, geom_c.id boundary_id + FROM observatory.obs_table data_t, + observatory.obs_column_table data_ct, + observatory.obs_column data_c, + observatory.obs_column_table geoid_ct, + observatory.obs_column_to_column c2c, + observatory.obs_column geom_c + WHERE data_c.id = $2 + AND data_ct.column_id = data_c.id + AND data_ct.table_id = data_t.id + AND geoid_ct.table_id = data_t.id + AND geoid_ct.column_id = c2c.source_id + AND c2c.reltype = ''geom_ref'' + AND geom_c.id = c2c.target_id + AND CASE WHEN $3 IS NULL THEN True ELSE $3 = timespan END + AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END + ORDER BY geom_c.weight DESC, + data_t.timespan DESC + LIMIT 1 + ' INTO timespan, geometry_id + USING geometry_id, (column_ids)[1], timespan; + END IF; + + RETURN QUERY + EXECUTE ' + WITH geomref AS ( + SELECT ct.table_id id + FROM observatory.OBS_column_to_column c2c, + observatory.OBS_column_table ct + WHERE c2c.reltype = ''geom_ref'' + AND c2c.target_id = $1 + AND c2c.source_id = ct.column_id + ), + column_ids as ( + select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a + ) + SELECT row_to_json(a) from ( + select colname, + tablename, + aggregate, + name, + type, + c.description, + $1 AS boundary_id + FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t + WHERE column_ids.column_id = c.id + AND c.id = ct.column_id + AND t.id = ct.table_id + AND t.timespan = $3 + AND t.id in (SELECT id FROM geomref) + order by column_ids.no + ) a + ' + USING geometry_id, column_ids, timespan + RETURN; + +END; +$$ LANGUAGE plpgsql; + --Test point cause Stuart always seems to make random points in the water CREATE OR REPLACE FUNCTION cdb_observatory._TestPoint() RETURNS geometry(Point, 4326) @@ -83,6 +158,28 @@ BEGIN END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetRelatedColumn(columns_ids text[], reltype text ) +RETURNS TEXT[] +AS $$ +DECLARE + result TEXT[]; +BEGIN + EXECUTE ' + With ids as ( + select row_number() over() as no, id from (select unnest($1) as id) t + ) + select array_agg(target_id order by no) + FROM ids + LEFT JOIN observatory.obs_column_to_column + on source_id = id + where reltype = $2 or reltype is null + ' + INTO result + using columns_ids, reltype; + return result; +END; +$$ LANGUAGE plpgsql; + -- Function that replaces all non digits or letters with _ trims and lowercases the -- passed measure name From adc663b563d1555a59238195836afdb52838da31 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 18:14:09 -0400 Subject: [PATCH 21/26] default to area normalization for point, no normalization for polygon getmeasures --- src/pg/sql/41_observatory_augmentation.sql | 26 ++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 6140542..a397f73 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -338,13 +338,14 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure( geom geometry(Geometry, 4326), measure_id TEXT, - normalize TEXT DEFAULT 'area', -- TODO none/null + normalize TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL ) RETURNS NUMERIC AS $$ DECLARE + geom_type TEXT; map_type TEXT; numer_aggregate TEXT; numer_colname TEXT; @@ -378,15 +379,29 @@ BEGIN geom_colname, geom_geomref_colname, geom_tablename, numer_name USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); + IF ST_GeometryType(geom) = 'ST_Point' THEN + geom_type := 'point'; + ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + geom_type := 'polygon'; + ELSE + RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', + ST_GeometryType(geom); + END IF; + IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN map_type := 'areaNormalized'; ELSIF normalize ILIKE 'denominator' AND numer_aggregate ILIKE 'sum' THEN map_type := 'denominated'; ELSE - map_type := 'predenominated'; + -- defaults: area normalization for point and none for polygon + IF geom_type = 'point' THEN + map_type := 'areaNormalized'; + ELSIF geom_type = 'polygon' THEN + map_type := 'predenominated'; + END IF; END IF; - IF ST_GeometryType(geom) = 'ST_Point' THEN + IF geom_type = 'point' THEN IF map_type = 'areaNormalized' THEN sql = format('WITH _geom AS (SELECT ST_Area(geom.%I::Geography) / 1000000 area, geom.%I geom_ref FROM observatory.%I geom @@ -415,7 +430,7 @@ BEGIN numer_geomref_colname, geom_geomref_colname, geom_tablename, geom, geom_colname); END IF; - ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + ELSIF geom_type = 'polygon' THEN IF map_type = 'areaNormalized' THEN sql = format('WITH _geom AS (SELECT ST_Area(ST_Intersection(%L, geom.%I)) / ST_Area(geom.%I) overlap, geom.%I geom_ref @@ -484,9 +499,6 @@ BEGIN numer_geomref_colname); END IF; END IF; - ELSE - RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', - ST_GeometryType(geom); END IF; EXECUTE sql INTO result; From cbf1c5e67dd7ec4188a0fa3cf793bdd6d4cba3e0 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 18:16:53 -0400 Subject: [PATCH 22/26] fix default normalizations --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index a397f73..bf01221 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -390,11 +390,11 @@ BEGIN IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN map_type := 'areaNormalized'; - ELSIF normalize ILIKE 'denominator' AND numer_aggregate ILIKE 'sum' THEN + ELSIF normalize ILIKE 'denominator' THEN map_type := 'denominated'; ELSE -- defaults: area normalization for point and none for polygon - IF geom_type = 'point' THEN + IF geom_type = 'point' AND numer_aggregate ILIKE 'sum' THEN map_type := 'areaNormalized'; ELSIF geom_type = 'polygon' THEN map_type := 'predenominated'; From f581278b8a844717199745c922478b87efebbae3 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 18:24:45 -0400 Subject: [PATCH 23/26] default is now NULL --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index bf01221..5c8de45 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -623,7 +623,7 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetUSCensusMeasure( geom geometry(Geometry, 4326), name TEXT, - normalize TEXT DEFAULT 'area', + normalize TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL ) @@ -689,7 +689,7 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetPopulation( geom geometry(Geometry, 4326), - normalize TEXT DEFAULT 'area', + normalize TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL ) From 1d008ccbe934cc9cf9c5b4b8afd8bc0ccbb5783b Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 18:33:53 -0400 Subject: [PATCH 24/26] should not try to use area normalization for zhvi --- src/pg/test/sql/41_observatory_augmentation_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 763231a..0a6808e 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -131,7 +131,7 @@ WITH result as ( -- Point-based OBS_GetMeasure with zillow SELECT abs(OBS_GetMeasure_zhvi_point - 583600) / 583600 < 0.001 AS OBS_GetMeasure_zhvi_point_test FROM cdb_observatory.OBS_GetMeasure( ST_SetSRID(ST_Point(-73.94602417945862, 40.6768220087458), 4326), - 'us.zillow.AllHomes_Zhvi', 'area', 'us.census.tiger.zcta5', '2014-01' + 'us.zillow.AllHomes_Zhvi', null, 'us.census.tiger.zcta5', '2014-01' ) As t(OBS_GetMeasure_zhvi_point); -- Point-based OBS_GetMeasure with zillow default to latest From 7bb1bbd8040c4d8c4d3e039410ad90e73150b376 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 18:37:17 -0400 Subject: [PATCH 25/26] handle predenomination of points properly --- src/pg/sql/41_observatory_augmentation.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 5c8de45..e34e5c8 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -393,10 +393,11 @@ BEGIN ELSIF normalize ILIKE 'denominator' THEN map_type := 'denominated'; ELSE - -- defaults: area normalization for point and none for polygon + -- defaults: area normalization for point if it's possible and none for + -- polygon or non-summable point IF geom_type = 'point' AND numer_aggregate ILIKE 'sum' THEN map_type := 'areaNormalized'; - ELSIF geom_type = 'polygon' THEN + ELSE map_type := 'predenominated'; END IF; END IF; From c09e0b6e83558bf1bf4920239dff92eadd95d4b3 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 13 Jul 2016 18:42:25 -0400 Subject: [PATCH 26/26] can eliminate getrelatedcolumn --- src/pg/sql/40_observatory_utility.sql | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index de942b5..39e1950 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -158,28 +158,6 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetRelatedColumn(columns_ids text[], reltype text ) -RETURNS TEXT[] -AS $$ -DECLARE - result TEXT[]; -BEGIN - EXECUTE ' - With ids as ( - select row_number() over() as no, id from (select unnest($1) as id) t - ) - select array_agg(target_id order by no) - FROM ids - LEFT JOIN observatory.obs_column_to_column - on source_id = id - where reltype = $2 or reltype is null - ' - INTO result - using columns_ids, reltype; - return result; -END; -$$ LANGUAGE plpgsql; - -- Function that replaces all non digits or letters with _ trims and lowercases the -- passed measure name