From be4b5abbfa4b324f0bc90f7b1135a40bb666b701 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Mon, 7 Nov 2016 23:57:33 +0000 Subject: [PATCH] use highest ranked geom for obs_getmeasure, simplify scoring --- src/pg/sql/41_observatory_augmentation.sql | 49 ++++++++++++------- src/pg/sql/42_observatory_exploration.sql | 13 ++--- .../42_observatory_exploration_test.out | 12 +++++ .../sql/42_observatory_exploration_test.sql | 48 +++++++++++++++--- 4 files changed, 93 insertions(+), 29 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 5341945..aa5e4e4 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -360,6 +360,7 @@ DECLARE geom_colname TEXT; geom_geomref_colname TEXT; geom_tablename TEXT; + geom_id TEXT; result NUMERIC; sql TEXT; numer_name TEXT; @@ -370,23 +371,6 @@ BEGIN geom := ST_SnapToGrid(geom, 0.000001); - EXECUTE - $query$ - SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, - denom_colname, denom_geomref_colname, denom_tablename, - geom_colname, geom_geomref_colname, geom_tablename, numer_name - FROM observatory.obs_meta - WHERE (geom_id = $1 OR ($1 = '')) - AND numer_id = $2 - AND (numer_timespan = $3 OR ($3 = '')) - ORDER BY geom_weight DESC, numer_timespan DESC - LIMIT 1 - $query$ - INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, - denom_colname, denom_geomref_colname, denom_tablename, - geom_colname, geom_geomref_colname, geom_tablename, numer_name - USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''); - IF ST_GeometryType(geom) = 'ST_Point' THEN geom_type := 'point'; ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN @@ -397,6 +381,37 @@ BEGIN ST_GeometryType(geom); END IF; + + EXECUTE + $query$ + WITH meta AS (SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, + denom_colname, denom_geomref_colname, denom_tablename, + geom_colname, geom_geomref_colname, geom_tablename, + numer_name, geom_id + FROM observatory.obs_meta + WHERE (geom_id = $1 OR ($1 = '')) + AND numer_id = $2 + AND (numer_timespan = $3 OR ($3 = ''))), + scores AS (SELECT * + FROM cdb_observatory._OBS_GetGeometryScores($4, + (SELECT Array_Agg(geom_id) FROM meta), 500)) + SELECT meta.* + FROM meta, scores + WHERE meta.geom_id = scores.geom_id + ORDER BY score DESC + LIMIT 1 + $query$ + INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename, + denom_colname, denom_geomref_colname, denom_tablename, + geom_colname, geom_geomref_colname, geom_tablename, numer_name, geom_id + USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''), + CASE WHEN ST_GeometryType(geom) = 'ST_Point' THEN + st_buffer(geom::geography, 10)::geometry(geometry, 4326) + ELSE geom + END; + + raise notice 'Using boundary %', geom_id; + IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN map_type := 'areaNormalized'; ELSIF normalize ILIKE 'denominator' THEN diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index a3b87ed..8560875 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -416,7 +416,8 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores( bounds Geometry(Geometry, 4326) DEFAULT NULL, - filter_geom_ids TEXT[] DEFAULT NULL + filter_geom_ids TEXT[] DEFAULT NULL, + desired_num_geoms INTEGER DEFAULT 3000 ) RETURNS TABLE ( score NUMERIC, numtiles BIGINT, @@ -432,9 +433,9 @@ BEGIN RETURN QUERY EXECUTE format($string$ SELECT - (1 / (abs(numgeoms - 3000) - * (1 / Coalesce(NullIf(notnull_percent, 0), 1)) - * (1 / Coalesce(NullIf(percentfill, 0), 0.0001)) + (1 / (abs(numgeoms - $3) + --* (1 / Coalesce(NullIf(notnull_percent, 0), 1)) + --* (1 / Coalesce(NullIf(percentfill, 0), 0.0001)) ))::Numeric AS score, * FROM ( @@ -460,7 +461,7 @@ BEGIN END)::Numeric AS notnull_percent , (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0 THEN (ST_SummaryStatsAgg(clipped_tile, 2, True)).sum - ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0) + ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0) * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(cdb_observatory.FIRST(tile), 0, 0)) * cdb_observatory.FIRST(pixels)) END)::Numeric AS numgeoms , (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0 THEN (ST_SummaryStatsAgg(clipped_tile, 3, True)).mean @@ -481,7 +482,7 @@ BEGIN GROUP BY a.column_id, a.table_id ORDER BY a.column_id, a.table_id ) foo - $string$) USING bounds, filter_geom_ids; + $string$) USING bounds, filter_geom_ids, desired_num_geoms; RETURN; END $$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index 94ae652..3be1542 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -174,6 +174,18 @@ t _obs_geometryscores_numgeoms_2500km_buffer t (1 row) +_obs_geometryscores_500km_buffer_50_geoms +t +(1 row) +_obs_geometryscores_500km_buffer_500_geoms +t +(1 row) +_obs_geometryscores_500km_buffer_2500_geoms +t +(1 row) +_obs_geometryscores_500km_buffer_25000_geoms +t +(1 row) _total_pop_in_legacy_builder_metadata t (1 row) diff --git a/src/pg/test/sql/42_observatory_exploration_test.sql b/src/pg/test/sql/42_observatory_exploration_test.sql index a0358b8..47df681 100644 --- a/src/pg/test/sql/42_observatory_exploration_test.sql +++ b/src/pg/test/sql/42_observatory_exploration_test.sql @@ -352,7 +352,7 @@ AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014; SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', - 'us.census.tiger.county', 'us.census.tiger.zcta5'] + 'us.census.tiger.zcta5', 'us.census.tiger.county'] AS _obs_geometryscores_500m_buffer FROM cdb_observatory._OBS_GetGeometryScores( ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326), @@ -361,7 +361,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', - 'us.census.tiger.county', 'us.census.tiger.zcta5'] + 'us.census.tiger.zcta5', 'us.census.tiger.county'] AS _obs_geometryscores_5km_buffer FROM cdb_observatory._OBS_GetGeometryScores( ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326), @@ -369,8 +369,8 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = 'us.census.tiger.zcta5', 'us.census.tiger.county']); SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = - ARRAY['us.census.tiger.census_tract', 'us.census.tiger.county', - 'us.census.tiger.zcta5', 'us.census.tiger.block_group'] + ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5', + 'us.census.tiger.county', 'us.census.tiger.block_group'] AS _obs_geometryscores_50km_buffer FROM cdb_observatory._OBS_GetGeometryScores( ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), @@ -396,7 +396,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = 'us.census.tiger.zcta5', 'us.census.tiger.county']); SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text - = '{ "us.census.tiger.block_group" : 3960, "us.census.tiger.census_tract" : 1444, "us.census.tiger.zcta5" : 178, "us.census.tiger.county" : 4 }' + = '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }' AS _obs_geometryscores_numgeoms_500m_buffer FROM cdb_observatory._OBS_GetGeometryScores( ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326), @@ -404,7 +404,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text 'us.census.tiger.zcta5', 'us.census.tiger.county']); SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text = - '{ "us.census.tiger.block_group" : 3960, "us.census.tiger.census_tract" : 1444, "us.census.tiger.zcta5" : 178, "us.census.tiger.county" : 4 }' + '{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }' AS _obs_geometryscores_numgeoms_5km_buffer FROM cdb_observatory._OBS_GetGeometryScores( ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326), @@ -435,6 +435,42 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text = ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', 'us.census.tiger.zcta5', 'us.census.tiger.county']); +SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = + ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5', + 'us.census.tiger.census_tract', 'us.census.tiger.block_group'] + AS _obs_geometryscores_500km_buffer_50_geoms + FROM cdb_observatory._OBS_GetGeometryScores( + ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), + ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', + 'us.census.tiger.zcta5', 'us.census.tiger.county'], 50); + +SELECT ARRAY_AGG(geom_id ORDER BY score DESC) + = ARRAY['us.census.tiger.zcta5', 'us.census.tiger.county', + 'us.census.tiger.census_tract', 'us.census.tiger.block_group'] + AS _obs_geometryscores_500km_buffer_500_geoms + FROM cdb_observatory._OBS_GetGeometryScores( + ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), + ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', + 'us.census.tiger.zcta5', 'us.census.tiger.county'], 500); + +SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = + ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5', + 'us.census.tiger.county', 'us.census.tiger.block_group'] + AS _obs_geometryscores_500km_buffer_2500_geoms + FROM cdb_observatory._OBS_GetGeometryScores( + ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), + ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', + 'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500); + +SELECT ARRAY_AGG(geom_id ORDER BY score DESC) = + ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', + 'us.census.tiger.zcta5', 'us.census.tiger.county'] + AS _obs_geometryscores_500km_buffer_25000_geoms + FROM cdb_observatory._OBS_GetGeometryScores( + ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326), + ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract', + 'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000); + -- -- OBS_LegacyBuilderMetadata tests --