cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use obs_column_table_tile_raster with simpler bands for faster performance

Browse Source
This commit is contained in:
John Krauss
2016-12-12 21:25:59 +00:00
parent 82137d5679
commit d4e6e7ac95
6 changed files with 504 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pg/sql/42_observatory_exploration.sql
+47 -56
View File
@@ -412,11 +412,6 @@ BEGIN
END
$$ LANGUAGE plpgsql;
--create table observatory.obs_column_table_tile_simple as select table_id, column_id, tile_id, st_band(tile, 2) as tile from observatory.obs_column_table_tile ;
--create table observatory.obs_column_table_tile_simpler as select table_id,
--column_id, tile_id, st_reclass(tile, '0-65536:0-65536', '16BUI') as tile
--from observatory.obs_column_table_tile_simple ;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
bounds Geometry(Geometry, 4326) DEFAULT NULL,
@@ -440,66 +435,62 @@ BEGIN
bounds := ST_Envelope(bounds);
END IF;
RETURN QUERY
EXECUTE format($string$
SELECT
((100.0 / (1+abs(log(1 + $3) - log(1 + numgeoms)))) * percentfill)::Numeric
AS score, *
FROM (
WITH clipped_geom AS (
SELECT column_id, table_id
--, CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, 2, $1, True)
, CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True)
ELSE tile END clipped_tile
--, tile as clipped_tile
, tile
FROM observatory.obs_column_table_tile
WHERE ($1 IS NULL OR ST_Intersects($1, tile))
AND (column_id = ANY($2) OR cardinality($2) = 0)
), clipped_geom_countagg AS (
SELECT column_id, table_id
, ST_CountAgg(clipped_tile, 2, True)::Numeric notnull_pixels
, ST_CountAgg(clipped_tile, 2, False)::Numeric pixels
FROM clipped_geom
GROUP BY column_id, table_id
), clipped_geom_reagg AS (
SELECT COUNT(*)::BIGINT cnt, a.column_id,
cdb_observatory.FIRST(pixels) first_pixel,
cdb_observatory.FIRST(notnull_pixels) first_notnull_pixel,
cdb_observatory.FIRST(tile) first_tile,
(ST_SummaryStatsAgg(clipped_tile, 2, True)).sum sum_geoms,
(ST_SummaryStatsAgg(clipped_tile, 3, True)).mean mean_fill
FROM clipped_geom_countagg a, clipped_geom b
WHERE a.table_id = b.table_id
AND a.column_id = b.column_id
GROUP BY a.column_id, a.table_id
) SELECT
EXECUTE $string$
WITH clipped_geom AS (
SELECT column_id, table_id
, CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True) -- -20
ELSE tile END clipped_tile
, tile
FROM observatory.obs_column_table_tile_simple
WHERE ($1 IS NULL OR ST_Intersects($1, tile))
AND (column_id = ANY($2) OR cardinality($2) = 0)
), clipped_geom_countagg AS (
SELECT column_id, table_id
, ST_CountAgg(clipped_tile, 1, True)::Numeric notnull_pixels -- -10
, ST_CountAgg(clipped_tile, 1, False)::Numeric pixels -- -10
FROM clipped_geom
GROUP BY column_id, table_id
), clipped_geom_reagg AS (
SELECT COUNT(*)::BIGINT cnt, a.column_id,
cdb_observatory.FIRST(pixels) first_pixel,
cdb_observatory.FIRST(notnull_pixels) first_notnull_pixel,
cdb_observatory.FIRST(tile) first_tile,
(ST_SummaryStatsAgg(clipped_tile, 1, True)).sum::Numeric sum_geoms, -- ND
(ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND
FROM clipped_geom_countagg a, clipped_geom b
WHERE a.table_id = b.table_id
AND a.column_id = b.column_id
GROUP BY a.column_id, a.table_id
), final AS (
SELECT
cnt, column_id
, (CASE WHEN first_notnull_pixel > 0
THEN first_notnull_pixel / first_pixel
ELSE 1
END)::Numeric AS notnull_percent
END)::Numeric
AS notnull_percent
, (CASE WHEN first_notnull_pixel > 0
THEN sum_geoms
ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1)), 0)
* (ST_Area($1) / ST_Area(ST_PixelAsPolygon(first_tile, 0, 0)) * first_pixel)
END)::Numeric AS numgeoms
ELSE COALESCE(ST_Value(first_tile, 1, ST_PointOnSurface($1)), 0)
* (ST_Area($1) / ST_Area(ST_PixelAsPolygon(first_tile, 0, 0))
* first_pixel) -- -20
END)::Numeric
AS numgeoms
, (CASE WHEN first_notnull_pixel > 0
THEN mean_fill
ELSE COALESCE(ST_Value(first_tile, 3, ST_PointOnSurface($1)), 0)
END)::Numeric AS percentfill
, null::numeric estnumgeoms, null::numeric meanmediansize
--, ((ST_Area(ST_Transform($1, 3857)) / 1000000) / NullIf(
-- CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
-- THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
-- ELSE Coalesce(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
-- END, 0))::Numeric AS estnumgeoms
--, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
-- THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
-- ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
-- END)::Numeric AS meanmediansize
ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2
END)::Numeric
AS percentfill
, null::numeric estnumgeoms
, null::numeric meanmediansize
FROM clipped_geom_reagg
) foo
$string$) USING bounds, filter_geom_ids, desired_num_geoms;
) SELECT
((100.0 / (1+abs(log(1 + $3) - log(1 + numgeoms::Numeric)))) * percentfill)::Numeric
AS score, *
FROM final
$string$ USING bounds, filter_geom_ids, desired_num_geoms;
RETURN;
END
$$ LANGUAGE plpgsql IMMUTABLE;
src/pg/test/fixtures/drop_fixtures.sql
Vendored
+1
View File
@@ -13,6 +13,7 @@ DROP TABLE IF EXISTS observatory.obs_meta_denom;
DROP TABLE IF EXISTS observatory.obs_meta_geom;
DROP TABLE IF EXISTS observatory.obs_meta_timespan;
DROP TABLE IF EXISTS observatory.obs_column_table_tile;
DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple;
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
src/pg/test/fixtures/load_fixtures.sql
Vendored
+371 -249
View File
File diff suppressed because one or more lines are too long
src/pg/test/sql/42_observatory_exploration_test.sql
+5 -5
View File
@@ -412,7 +412,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 13 }'
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }'
AS _obs_geometryscores_numgeoms_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
@@ -420,7 +420,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 48415, "us.census.tiger.census_tract" : 15776, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 295 }'
'{ "us.census.tiger.block_group" : 48420, "us.census.tiger.census_tract" : 15774, "us.census.tiger.zcta5" : 6533, "us.census.tiger.county" : 304 }'
AS _obs_geometryscores_numgeoms_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
@@ -428,7 +428,7 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 165489, "us.census.tiger.census_tract" : 55152, "us.census.tiger.zcta5" : 26500, "us.census.tiger.county" : 2551 }'
'{ "us.census.tiger.block_group" : 165475, "us.census.tiger.census_tract" : 55128, "us.census.tiger.zcta5" : 26499, "us.census.tiger.county" : 2622 }'
AS _obs_geometryscores_numgeoms_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
@@ -436,8 +436,8 @@ SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.block_group']
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_50_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),