cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use highest ranked geom for obs_getmeasure, simplify scoring

Browse Source
This commit is contained in:
John Krauss
2016-11-07 23:57:33 +00:00
parent 8dad88a6b3
commit be4b5abbfa
4 changed files with 93 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/pg/sql/41_observatory_augmentation.sql
View File

@@ -360,6 +360,7 @@ DECLARE
geom_colname TEXT;
geom_geomref_colname TEXT;
geom_tablename TEXT;
geom_id TEXT;
result NUMERIC;
sql TEXT;
numer_name TEXT;
@@ -370,23 +371,6 @@ BEGIN
geom := ST_SnapToGrid(geom, 0.000001);
EXECUTE
$query$
SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name
FROM observatory.obs_meta
WHERE (geom_id = $1 OR ($1 = ''))
AND numer_id = $2
AND (numer_timespan = $3 OR ($3 = ''))
ORDER BY geom_weight DESC, numer_timespan DESC
LIMIT 1
$query$
INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name
USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, '');
IF ST_GeometryType(geom) = 'ST_Point' THEN
geom_type := 'point';
ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN
@@ -397,6 +381,37 @@ BEGIN
ST_GeometryType(geom);
END IF;
EXECUTE
$query$
WITH meta AS (SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename,
numer_name, geom_id
FROM observatory.obs_meta
WHERE (geom_id = $1 OR ($1 = ''))
AND numer_id = $2
AND (numer_timespan = $3 OR ($3 = ''))),
scores AS (SELECT *
FROM cdb_observatory._OBS_GetGeometryScores($4,
(SELECT Array_Agg(geom_id) FROM meta), 500))
SELECT meta.*
FROM meta, scores
WHERE meta.geom_id = scores.geom_id
ORDER BY score DESC
LIMIT 1
$query$
INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name, geom_id
USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''),
CASE WHEN ST_GeometryType(geom) = 'ST_Point' THEN
st_buffer(geom::geography, 10)::geometry(geometry, 4326)
ELSE geom
END;
raise notice 'Using boundary %', geom_id;
IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN
map_type := 'areaNormalized';
ELSIF normalize ILIKE 'denominator' THEN

13
src/pg/sql/42_observatory_exploration.sql
View File

@@ -416,7 +416,8 @@ $$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
bounds Geometry(Geometry, 4326) DEFAULT NULL,
filter_geom_ids TEXT[] DEFAULT NULL
filter_geom_ids TEXT[] DEFAULT NULL,
desired_num_geoms INTEGER DEFAULT 3000
) RETURNS TABLE (
score NUMERIC,
numtiles BIGINT,
@@ -432,9 +433,9 @@ BEGIN
RETURN QUERY
EXECUTE format($string$
SELECT
(1 / (abs(numgeoms - 3000)
* (1 / Coalesce(NullIf(notnull_percent, 0), 1))
* (1 / Coalesce(NullIf(percentfill, 0), 0.0001))
(1 / (abs(numgeoms - $3)
--* (1 / Coalesce(NullIf(notnull_percent, 0), 1))
--* (1 / Coalesce(NullIf(percentfill, 0), 0.0001))
))::Numeric
AS score, *
FROM (
@@ -460,7 +461,7 @@ BEGIN
END)::Numeric AS notnull_percent
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 2, True)).sum
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0)
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0) * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(cdb_observatory.FIRST(tile), 0, 0)) * cdb_observatory.FIRST(pixels))
END)::Numeric AS numgeoms
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 3, True)).mean
@@ -481,7 +482,7 @@ BEGIN
GROUP BY a.column_id, a.table_id
ORDER BY a.column_id, a.table_id
) foo
$string$) USING bounds, filter_geom_ids;
$string$) USING bounds, filter_geom_ids, desired_num_geoms;
RETURN;
END
$$ LANGUAGE plpgsql;

12
src/pg/test/expected/42_observatory_exploration_test.out
View File

@@ -174,6 +174,18 @@ t
_obs_geometryscores_numgeoms_2500km_buffer
t
(1 row)
_obs_geometryscores_500km_buffer_50_geoms
t
(1 row)
_obs_geometryscores_500km_buffer_500_geoms
t
(1 row)
_obs_geometryscores_500km_buffer_2500_geoms
t
(1 row)
_obs_geometryscores_500km_buffer_25000_geoms
t
(1 row)
_total_pop_in_legacy_builder_metadata
t
(1 row)

48
src/pg/test/sql/42_observatory_exploration_test.sql
View File

@@ -352,7 +352,7 @@ AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014;
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5']
'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
@@ -361,7 +361,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.county', 'us.census.tiger.zcta5']
'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
@@ -369,8 +369,8 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.county',
'us.census.tiger.zcta5', 'us.census.tiger.block_group']
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5',
'us.census.tiger.county', 'us.census.tiger.block_group']
AS _obs_geometryscores_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
@@ -396,7 +396,7 @@ SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
= '{ "us.census.tiger.block_group" : 3960, "us.census.tiger.census_tract" : 1444, "us.census.tiger.zcta5" : 178, "us.census.tiger.county" : 4 }'
= '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }'
AS _obs_geometryscores_numgeoms_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
@@ -404,7 +404,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 3960, "us.census.tiger.census_tract" : 1444, "us.census.tiger.zcta5" : 178, "us.census.tiger.county" : 4 }'
'{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
AS _obs_geometryscores_numgeoms_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
@@ -435,6 +435,42 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_50_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 50);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC)
= ARRAY['us.census.tiger.zcta5', 'us.census.tiger.county',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_500_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 500);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5',
'us.census.tiger.county', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_2500_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_500km_buffer_25000_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000);
--
-- OBS_LegacyBuilderMetadata tests
--