From 84dec8bdf4ab53ad070805e7ae9762d8c59a8117 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 12:09:42 -0400 Subject: [PATCH 01/14] simplify obs_getcategory and use obs_meta --- src/pg/sql/41_observatory_augmentation.sql | 43 +++++++++++++--------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 40e9d1e..1d0a4a2 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -393,7 +393,6 @@ DECLARE colname TEXT; measure_val NUMERIC; data_geoid_colname TEXT; - test_query TEXT; BEGIN EXECUTE @@ -434,27 +433,37 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetCategory( RETURNS TEXT AS $$ DECLARE - denominator_id TEXT; - categories TEXT[]; + target_table TEXT; + colname TEXT; + category_val TEXT; + data_geoid_colname TEXT; BEGIN - IF boundary_id IS NULL THEN - -- TODO we should determine best boundary for this geom - boundary_id := 'us.census.tiger.census_tract'; - END IF; + EXECUTE + $query$ + SELECT numer_colname, numer_geomref_colname, numer_tablename + FROM observatory.obs_meta + WHERE (geom_id = $1 OR ($1 = '')) + AND numer_id = $2 + AND (numer_timespan = $3 OR ($3 = '')) + ORDER BY geom_weight DESC, numer_timespan DESC + LIMIT 1 + $query$ + INTO colname, data_geoid_colname, target_table + USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); - IF time_span IS NULL THEN - -- TODO we should determine latest timespan for this measure - time_span := '2010 - 2014'; - END IF; + RAISE DEBUG 'target_table %, colname %', target_table, colname; - EXECUTE ' - SELECT ARRAY_AGG(val) FROM (SELECT (cdb_observatory._OBS_GetCategories($1, $2, $3, $4))->>''category'' val LIMIT 1) b - ' - INTO categories - USING geom, ARRAY[category_id], boundary_id, time_span; + EXECUTE format( + 'SELECT %I + FROM observatory.%I data + WHERE data.%I = %L', + colname, + target_table, + data_geoid_colname, geom_ref) + INTO category_val; - RETURN (categories)[1]; + RETURN category_val; END; $$ LANGUAGE plpgsql; From 965fb947046841ea3d51a6fc4b32368a4ff157fd Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 12:22:06 -0400 Subject: [PATCH 02/14] fix bugs in obs_getcategory implementation --- src/pg/sql/41_observatory_augmentation.sql | 28 +++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 1d0a4a2..e98a3be 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -433,15 +433,19 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetCategory( RETURNS TEXT AS $$ DECLARE - target_table TEXT; + data_table TEXT; + geom_table TEXT; colname TEXT; + data_geomref_colname TEXT; + geom_geomref_colname TEXT; + geom_colname TEXT; category_val TEXT; - data_geoid_colname TEXT; BEGIN EXECUTE $query$ - SELECT numer_colname, numer_geomref_colname, numer_tablename + SELECT numer_colname, numer_geomref_colname, numer_tablename, + geom_geomref_colname, geom_colname, geom_tablename FROM observatory.obs_meta WHERE (geom_id = $1 OR ($1 = '')) AND numer_id = $2 @@ -449,18 +453,24 @@ BEGIN ORDER BY geom_weight DESC, numer_timespan DESC LIMIT 1 $query$ - INTO colname, data_geoid_colname, target_table - USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); + INTO colname, data_geomref_colname, data_table, geom_geomref_colname, + geom_colname, geom_table + USING COALESCE(boundary_id, ''), category_id, COALESCE(time_span, ''); RAISE DEBUG 'target_table %, colname %', target_table, colname; EXECUTE format( 'SELECT %I - FROM observatory.%I data - WHERE data.%I = %L', + FROM observatory.%I data, observatory.%I geom + WHERE data.%I = geom.%I + AND ST_WITHIN(%L, geom.%I) ', colname, - target_table, - data_geoid_colname, geom_ref) + data_table, + geom_table + data_geomref_colname, + geom_geomref_colname, + geom, + geom_colname) INTO category_val; RETURN category_val; From 26b61a6ddbc5faea72075e3e49be504b8a655f4d Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 12:26:10 -0400 Subject: [PATCH 03/14] minor formatting --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index e98a3be..cd47156 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -453,8 +453,8 @@ BEGIN ORDER BY geom_weight DESC, numer_timespan DESC LIMIT 1 $query$ - INTO colname, data_geomref_colname, data_table, geom_geomref_colname, - geom_colname, geom_table + INTO colname, data_geomref_colname, data_table, + geom_geomref_colname, geom_colname, geom_table USING COALESCE(boundary_id, ''), category_id, COALESCE(time_span, ''); RAISE DEBUG 'target_table %, colname %', target_table, colname; From af39a37b43ac26a3f824d63b4bccc95eb47cafdd Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 12:34:32 -0400 Subject: [PATCH 04/14] fix missing comma --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index cd47156..1b5d5b2 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -466,7 +466,7 @@ BEGIN AND ST_WITHIN(%L, geom.%I) ', colname, data_table, - geom_table + geom_table, data_geomref_colname, geom_geomref_colname, geom, From 62c08864af6252ba4676d93e082ecb54e5014b54 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 12:37:30 -0400 Subject: [PATCH 05/14] remove bad "target_table" notice --- src/pg/sql/41_observatory_augmentation.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 1b5d5b2..80d56d1 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -457,8 +457,6 @@ BEGIN geom_geomref_colname, geom_colname, geom_table USING COALESCE(boundary_id, ''), category_id, COALESCE(time_span, ''); - RAISE DEBUG 'target_table %, colname %', target_table, colname; - EXECUTE format( 'SELECT %I FROM observatory.%I data, observatory.%I geom From 8654c22c87affd439423be4f5b6b3e35b4850366 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 14:06:43 -0400 Subject: [PATCH 06/14] handle area categories properly --- src/pg/sql/41_observatory_augmentation.sql | 42 +++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 80d56d1..277cc00 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -457,19 +457,35 @@ BEGIN geom_geomref_colname, geom_colname, geom_table USING COALESCE(boundary_id, ''), category_id, COALESCE(time_span, ''); - EXECUTE format( - 'SELECT %I - FROM observatory.%I data, observatory.%I geom - WHERE data.%I = geom.%I - AND ST_WITHIN(%L, geom.%I) ', - colname, - data_table, - geom_table, - data_geomref_colname, - geom_geomref_colname, - geom, - geom_colname) - INTO category_val; + IF ST_GeometryType(geom) = 'ST_Point' THEN + EXECUTE format( + 'SELECT %I + FROM observatory.%I data, observatory.%I geom + WHERE data.%I = geom.%I + AND ST_WITHIN(%L, geom.%I) ', + colname, data_table, geom_table, data_geomref_colname, + geom_geomref_colname, geom, geom_colname) + INTO category_val; + ELSE + -- favor the category with the most area + EXECUTE format( + 'WITH _overlaps AS ( + SELECT ST_Area( + ST_Intersection(%I, a.%I) + ) / ST_Area(a.%I) AS overlap_fraction, %I geomref + FROM observatory.%I as a + WHERE %I && a.%I, + SELECT %I category + FROM observatory.%I data + WHERE data.%I = geomref + GROUP BY category + ORDER BY SUM(overlap_fraction) + LIMIT 1', + geom, geom_colname, geom_colname, geom_geomref_colname, + geom_table, geom, geom_colname, + colname, data_table, data_geomref_colname) + INTO category_val; + END IF; RETURN category_val; From 61854a070d5cd0a94ca14f71ba89de121bdd4346 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 14:11:02 -0400 Subject: [PATCH 07/14] fix wrong quoting --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 277cc00..e3796ee 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -471,10 +471,10 @@ BEGIN EXECUTE format( 'WITH _overlaps AS ( SELECT ST_Area( - ST_Intersection(%I, a.%I) + ST_Intersection(%L, a.%I) ) / ST_Area(a.%I) AS overlap_fraction, %I geomref FROM observatory.%I as a - WHERE %I && a.%I, + WHERE %L && a.%I, SELECT %I category FROM observatory.%I data WHERE data.%I = geomref From 62a2c259a76616bb872727fc8ed668513ee52d14 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 14:13:09 -0400 Subject: [PATCH 08/14] fix typo --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index e3796ee..1f2dde3 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -474,7 +474,7 @@ BEGIN ST_Intersection(%L, a.%I) ) / ST_Area(a.%I) AS overlap_fraction, %I geomref FROM observatory.%I as a - WHERE %L && a.%I, + WHERE %L && a.%I SELECT %I category FROM observatory.%I data WHERE data.%I = geomref From 1e9c3fb860342d2ad9e46355036e4ee9408f6006 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 14:15:00 -0400 Subject: [PATCH 09/14] fix typo --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 1f2dde3..22f97c6 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -474,7 +474,7 @@ BEGIN ST_Intersection(%L, a.%I) ) / ST_Area(a.%I) AS overlap_fraction, %I geomref FROM observatory.%I as a - WHERE %L && a.%I + WHERE %L && a.%I), SELECT %I category FROM observatory.%I data WHERE data.%I = geomref From 5bb42855289a9b9ad142e1e886d12d9a533ab1fb Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 14:16:28 -0400 Subject: [PATCH 10/14] fix typo --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 22f97c6..3cf01dd 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -474,7 +474,7 @@ BEGIN ST_Intersection(%L, a.%I) ) / ST_Area(a.%I) AS overlap_fraction, %I geomref FROM observatory.%I as a - WHERE %L && a.%I), + WHERE %L && a.%I) SELECT %I category FROM observatory.%I data WHERE data.%I = geomref From 4fc02f99e26ccaa6837b56ab7940c454c69e9df3 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 14:37:43 -0400 Subject: [PATCH 11/14] choose largest segment in the polygon --- src/pg/sql/41_observatory_augmentation.sql | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3cf01dd..f5dfe68 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -440,6 +440,7 @@ DECLARE geom_geomref_colname TEXT; geom_colname TEXT; category_val TEXT; + category_share NUMERIC; BEGIN EXECUTE @@ -469,22 +470,21 @@ BEGIN ELSE -- favor the category with the most area EXECUTE format( - 'WITH _overlaps AS ( + 'SELECT %I category, SUM(overlap_fraction) category_share + FROM observatory.%I data, ( SELECT ST_Area( - ST_Intersection(%L, a.%I) - ) / ST_Area(a.%I) AS overlap_fraction, %I geomref + ST_Intersection(%L, a.%I) + ) / ST_Area(%L) AS overlap_fraction, %I geomref FROM observatory.%I as a - WHERE %L && a.%I) - SELECT %I category - FROM observatory.%I data - WHERE data.%I = geomref - GROUP BY category - ORDER BY SUM(overlap_fraction) + WHERE %L && a.%I) _overlaps + WHERE data.%I = _overlaps.geomref + GROUP BY category + ORDER BY SUM(overlap_fraction) DESC LIMIT 1', - geom, geom_colname, geom_colname, geom_geomref_colname, - geom_table, geom, geom_colname, - colname, data_table, data_geomref_colname) - INTO category_val; + colname, data_table, + geom, geom_colname, geom, geom_geomref_colname, + geom_table, geom, geom_colname, data_geomref_colname) + INTO category_val, category_share; END IF; RETURN category_val; From 4b26eeda656c93ae240f5ffded01bd669b4a49b6 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 15:33:26 -0400 Subject: [PATCH 12/14] fix to correct segment for testarea area --- src/pg/test/sql/41_observatory_augmentation_test.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index ff4f822..763231a 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -172,7 +172,7 @@ SELECT cdb_observatory.OBS_GetCategory( -- Poly-based OBS_GetCategory SELECT cdb_observatory.OBS_GetCategory( - cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Low income, mix of minorities' As obs_getcategory_polygon; + cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Wealthy, urban without Kids' As obs_getcategory_polygon; -- Point-based OBS_GetPopulation, default normalization (area) SELECT (abs(OBS_GetPopulation - 10923.093200390833950) / 10923.093200390833950) < 0.001 As OBS_GetPopulation FROM @@ -201,7 +201,7 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory( -- Area-based OBS_GetUSCensusCategory SELECT cdb_observatory.OBS_GetUSCensusCategory( - cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Low income, mix of minorities' As OBS_GetUSCensusCategory_polygon; + cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Wealthy, urban without Kids' As OBS_GetUSCensusCategory_polygon; -- OBS_GetMeasureById tests From 81176d1df2c5976c670a2a1896d0624bc7c7c210 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 16:06:19 -0400 Subject: [PATCH 13/14] fix possible ambiguity in category colname --- src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index f5dfe68..ac71cf5 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -470,11 +470,11 @@ BEGIN ELSE -- favor the category with the most area EXECUTE format( - 'SELECT %I category, SUM(overlap_fraction) category_share + 'SELECT data.%I category, SUM(overlap_fraction) category_share FROM observatory.%I data, ( SELECT ST_Area( ST_Intersection(%L, a.%I) - ) / ST_Area(%L) AS overlap_fraction, %I geomref + ) / ST_Area(%L) AS overlap_fraction, a.%I geomref FROM observatory.%I as a WHERE %L && a.%I) _overlaps WHERE data.%I = _overlaps.geomref From 6a39bedee762a144785a861643fa958ec7e8e14c Mon Sep 17 00:00:00 2001 From: John Krauss Date: Tue, 12 Jul 2016 16:10:00 -0400 Subject: [PATCH 14/14] fix ambiguous colname for categories in points too --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index ac71cf5..59a2288 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -460,7 +460,7 @@ BEGIN IF ST_GeometryType(geom) = 'ST_Point' THEN EXECUTE format( - 'SELECT %I + 'SELECT data.%I FROM observatory.%I data, observatory.%I geom WHERE data.%I = geom.%I AND ST_WITHIN(%L, geom.%I) ',