cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #212 from CartoDB/develop

Browse Source 
Release 1.1.2
This commit is contained in:
Mario de Frutos
2016-11-11 17:09:56 +01:00
committed by GitHub
parent c91fcab28c 7e04c38c3a
commit 60b723de92
12 changed files with 3514 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS.md
+15
View File
@@ -1,3 +1,18 @@
1.1.2 (2016-11-09)
__Improvements__
* Update public `OBS_GetMeasure` to use highest ranked boundary, aiming for 500
geoms. ([#190](https://github.com/CartoDB/observatory-extension/issues/190))
* Update test generation to capture our raster tiles
* Standardize the way we generate our test points for `autotest.py`
* Add points for epa and eurostat
* Should support database dump generated 20161109
__API Changes (Internal)__
* Add internal `_OBS_GetGeometryScores`
1.1.1 (2016-10-14) 1.1.1 (2016-10-14)
__Improvements__ __Improvements__
release/observatory--1.1.2.sql
+2606
View File
File diff suppressed because one or more lines are too long
release/observatory.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB Observatory backend extension' comment = 'CartoDB Observatory backend extension'
default_version = '1.1.1' default_version = '1.1.2'
requires = 'postgis, postgres_fdw' requires = 'postgis, postgres_fdw'
superuser = true superuser = true
schema = cdb_observatory schema = cdb_observatory
scripts/generate_fixtures.py
+2 -2
View File
@@ -61,7 +61,7 @@ def get_tablename_query(column_id, boundary_id, timespan):
METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag', METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta', 'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta',
'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom', 'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom',
'obs_meta_timespan', ] 'obs_meta_timespan', 'obs_column_table_tile', ]
FIXTURES = [ FIXTURES = [
('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'), ('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
@@ -290,7 +290,7 @@ def main():
"column_id IN ('{}', '{}')".format(numer_id, geom_id) "column_id IN ('{}', '{}')".format(numer_id, geom_id)
for numer_id, geom_id, timespan in FIXTURES for numer_id, geom_id, timespan in FIXTURES
]) ])
elif tablename == 'obs_column_table': elif tablename in ('obs_column_table', 'obs_column_table_tile', ):
where = 'WHERE column_id IN ({numer_ids}) ' \ where = 'WHERE column_id IN ({numer_ids}) ' \
'OR column_id IN ({geom_ids}) ' \ 'OR column_id IN ({geom_ids}) ' \
'OR table_id IN ({table_ids}) '.format( 'OR table_id IN ({table_ids}) '.format(
src/pg/observatory.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB Observatory backend extension' comment = 'CartoDB Observatory backend extension'
default_version = '1.1.1' default_version = '1.1.2'
requires = 'postgis, postgres_fdw' requires = 'postgis, postgres_fdw'
superuser = true superuser = true
schema = cdb_observatory schema = cdb_observatory
src/pg/sql/41_observatory_augmentation.sql
+32 -17
View File
@@ -360,6 +360,7 @@ DECLARE
geom_colname TEXT; geom_colname TEXT;
geom_geomref_colname TEXT; geom_geomref_colname TEXT;
geom_tablename TEXT; geom_tablename TEXT;
geom_id TEXT;
result NUMERIC; result NUMERIC;
sql TEXT; sql TEXT;
numer_name TEXT; numer_name TEXT;
@@ -370,23 +371,6 @@ BEGIN
geom := ST_SnapToGrid(geom, 0.000001); geom := ST_SnapToGrid(geom, 0.000001);
EXECUTE
$query$
SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name
FROM observatory.obs_meta
WHERE (geom_id = $1 OR ($1 = ''))
AND numer_id = $2
AND (numer_timespan = $3 OR ($3 = ''))
ORDER BY geom_weight DESC, numer_timespan DESC
LIMIT 1
$query$
INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name
USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, '');
IF ST_GeometryType(geom) = 'ST_Point' THEN IF ST_GeometryType(geom) = 'ST_Point' THEN
geom_type := 'point'; geom_type := 'point';
ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN
@@ -397,6 +381,37 @@ BEGIN
ST_GeometryType(geom); ST_GeometryType(geom);
END IF; END IF;
EXECUTE
$query$
WITH meta AS (SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename,
numer_name, geom_id
FROM observatory.obs_meta
WHERE (geom_id = $1 OR ($1 = ''))
AND numer_id = $2
AND (numer_timespan = $3 OR ($3 = ''))),
scores AS (SELECT *
FROM cdb_observatory._OBS_GetGeometryScores($4,
(SELECT Array_Agg(geom_id) FROM meta), 500))
SELECT meta.*
FROM meta, scores
WHERE meta.geom_id = scores.geom_id
ORDER BY score DESC
LIMIT 1
$query$
INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name, geom_id
USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''),
CASE WHEN ST_GeometryType(geom) = 'ST_Point' THEN
st_buffer(geom::geography, 10)::geometry(geometry, 4326)
ELSE geom
END;
raise notice 'Using boundary %', geom_id;
IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN
map_type := 'areaNormalized'; map_type := 'areaNormalized';
ELSIF normalize ILIKE 'denominator' THEN ELSIF normalize ILIKE 'denominator' THEN
src/pg/sql/42_observatory_exploration.sql
+102 -13
View File
@@ -254,7 +254,14 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableGeometries(
geom_source TEXT, geom_source TEXT,
valid_numer BOOLEAN, valid_numer BOOLEAN,
valid_denom BOOLEAN, valid_denom BOOLEAN,
valid_timespan BOOLEAN valid_timespan BOOLEAN,
score NUMERIC,
numtiles BIGINT,
notnull_percent NUMERIC,
numgeoms NUMERIC,
percentfill NUMERIC,
estnumgeoms NUMERIC,
meanmediansize NUMERIC
) AS $$ ) AS $$
DECLARE DECLARE
geom_clause TEXT; geom_clause TEXT;
@@ -271,18 +278,27 @@ BEGIN
RETURN QUERY RETURN QUERY
EXECUTE EXECUTE
format($string$ format($string$
SELECT geom_id::TEXT, WITH available_geoms AS (
geom_name::TEXT, SELECT geom_id::TEXT,
geom_description::TEXT, geom_name::TEXT,
geom_weight::NUMERIC, geom_description::TEXT,
NULL::TEXT geom_aggregate, geom_weight::NUMERIC,
NULL::TEXT license, NULL::TEXT geom_aggregate,
NULL::TEXT source, NULL::TEXT license,
$1 = ANY(numers) valid_numer, NULL::TEXT source,
$2 = ANY(denoms) valid_denom, $1 = ANY(numers) valid_numer,
$3 = ANY(timespans) valid_timespan $2 = ANY(denoms) valid_denom,
FROM observatory.obs_meta_geom $3 = ANY(timespans) valid_timespan
WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0) FROM observatory.obs_meta_geom
WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0)
), scores AS (
SELECT * FROM cdb_observatory._OBS_GetGeometryScores($5,
(SELECT ARRAY_AGG(geom_id) FROM available_geoms)
)
) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms,
percentfill, estnumgeoms, meanmediansize
FROM available_geoms, scores
WHERE available_geoms.geom_id = scores.geom_id
$string$, geom_clause) $string$, geom_clause)
USING numer_id, denom_id, timespan, filter_tags, bounds; USING numer_id, denom_id, timespan, filter_tags, bounds;
RETURN; RETURN;
@@ -397,3 +413,76 @@ BEGIN
RETURN; RETURN;
END END
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
bounds Geometry(Geometry, 4326) DEFAULT NULL,
filter_geom_ids TEXT[] DEFAULT NULL,
desired_num_geoms INTEGER DEFAULT 3000
) RETURNS TABLE (
score NUMERIC,
numtiles BIGINT,
geom_id TEXT,
notnull_percent NUMERIC,
numgeoms NUMERIC,
percentfill NUMERIC,
estnumgeoms NUMERIC,
meanmediansize NUMERIC
) AS $$
BEGIN
filter_geom_ids := COALESCE(filter_geom_ids, (ARRAY[])::TEXT[]);
RETURN QUERY
EXECUTE format($string$
SELECT
(1 / (abs(numgeoms - $3)
--* (1 / Coalesce(NullIf(notnull_percent, 0), 1))
--* (1 / Coalesce(NullIf(percentfill, 0), 0.0001))
))::Numeric
AS score, *
FROM (
WITH clipped_geom AS (
SELECT column_id, table_id
, CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True)
ELSE tile END clipped_tile
, tile
FROM observatory.obs_column_table_tile
WHERE ($1 IS NULL OR ST_Intersects($1, tile))
AND (column_id = ANY($2) OR cardinality($2) = 0)
), clipped_geom_countagg AS (
SELECT column_id, table_id
, ST_CountAgg(clipped_tile, 2, True)::Numeric notnull_pixels
, ST_CountAgg(clipped_tile, 2, False)::Numeric pixels
FROM clipped_geom
GROUP BY column_id, table_id
) SELECT
count(*)::BIGINT, a.column_id
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN cdb_observatory.FIRST(notnull_pixels) / cdb_observatory.FIRST(pixels)
ELSE 1
END)::Numeric AS notnull_percent
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 2, True)).sum
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0) * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(cdb_observatory.FIRST(tile), 0, 0)) * cdb_observatory.FIRST(pixels))
END)::Numeric AS numgeoms
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 3, True)).mean
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 3, ST_PointOnSurface($1)), 0)
END)::Numeric AS percentfill
, ((ST_Area(ST_Transform($1, 3857)) / 1000000) / NullIf(
CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
ELSE Coalesce(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
END, 0))::Numeric AS estnumgeoms
, (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
END)::Numeric AS meanmediansize
FROM clipped_geom_countagg a, clipped_geom b
WHERE a.table_id = b.table_id
AND a.column_id = b.column_id
GROUP BY a.column_id, a.table_id
ORDER BY a.column_id, a.table_id
) foo
$string$) USING bounds, filter_geom_ids, desired_num_geoms;
RETURN;
END
$$ LANGUAGE plpgsql;
src/pg/test/expected/42_observatory_exploration_test.out
+42
View File
@@ -144,6 +144,48 @@ t
_obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014 _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014
t t
(1 row) (1 row)
_obs_geometryscores_500m_buffer
t
(1 row)
_obs_geometryscores_5km_buffer
t
(1 row)
_obs_geometryscores_50km_buffer
t
(1 row)
_obs_geometryscores_500km_buffer
t
(1 row)
_obs_geometryscores_2500km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_500m_buffer
t
(1 row)
_obs_geometryscores_numgeoms_5km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_50km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_500km_buffer
t
(1 row)
_obs_geometryscores_numgeoms_2500km_buffer
t
(1 row)
_obs_geometryscores_500km_buffer_50_geoms
t
(1 row)
_obs_geometryscores_500km_buffer_500_geoms
t
(1 row)
_obs_geometryscores_500km_buffer_2500_geoms
t
(1 row)
_obs_geometryscores_500km_buffer_25000_geoms
t
(1 row)
_total_pop_in_legacy_builder_metadata _total_pop_in_legacy_builder_metadata
t t
(1 row) (1 row)
src/pg/test/fixtures/drop_fixtures.sql
Vendored
+1
View File
@@ -12,6 +12,7 @@ DROP TABLE IF EXISTS observatory.obs_meta_numer;
DROP TABLE IF EXISTS observatory.obs_meta_denom; DROP TABLE IF EXISTS observatory.obs_meta_denom;
DROP TABLE IF EXISTS observatory.obs_meta_geom; DROP TABLE IF EXISTS observatory.obs_meta_geom;
DROP TABLE IF EXISTS observatory.obs_meta_timespan; DROP TABLE IF EXISTS observatory.obs_meta_timespan;
DROP TABLE IF EXISTS observatory.obs_column_table_tile;
DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963; DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308; DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74; DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
src/pg/test/fixtures/load_fixtures.sql
Vendored
+547 -224
View File
File diff suppressed because one or more lines are too long
src/pg/test/sql/42_observatory_exploration_test.sql
+127 -1
View File
@@ -346,7 +346,134 @@ FROM cdb_observatory.OBS_GetAvailableTimespans(
) WHERE valid_denom = True) ) WHERE valid_denom = True)
AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014; AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014;
--
-- _OBS_GetGeometryScores tests
--
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5',
'us.census.tiger.county', 'us.census.tiger.block_group']
AS _obs_geometryscores_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY[ 'us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
= '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }'
AS _obs_geometryscores_numgeoms_500m_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
AS _obs_geometryscores_numgeoms_5km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 13 }'
AS _obs_geometryscores_numgeoms_50km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 48415, "us.census.tiger.census_tract" : 15776, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 295 }'
AS _obs_geometryscores_numgeoms_500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
'{ "us.census.tiger.block_group" : 165489, "us.census.tiger.census_tract" : 55152, "us.census.tiger.zcta5" : 26500, "us.census.tiger.county" : 2551 }'
AS _obs_geometryscores_numgeoms_2500km_buffer
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_50_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 50);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC)
= ARRAY['us.census.tiger.zcta5', 'us.census.tiger.county',
'us.census.tiger.census_tract', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_500_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 500);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5',
'us.census.tiger.county', 'us.census.tiger.block_group']
AS _obs_geometryscores_500km_buffer_2500_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500);
SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county']
AS _obs_geometryscores_500km_buffer_25000_geoms
FROM cdb_observatory._OBS_GetGeometryScores(
ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000);
--
-- OBS_LegacyBuilderMetadata tests -- OBS_LegacyBuilderMetadata tests
--
SELECT 'us.census.acs.B01003001' IN (SELECT SELECT 'us.census.acs.B01003001' IN (SELECT
(jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id (jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
@@ -367,4 +494,3 @@ SELECT 'us.census.acs.B19013001' NOT IN (SELECT
(jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id (jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
FROM cdb_observatory.OBS_LegacyBuilderMetadata('sum') FROM cdb_observatory.OBS_LegacyBuilderMetadata('sum')
) AS _median_income_not_in_legacy_builder_metadata_sums; ) AS _median_income_not_in_legacy_builder_metadata_sums;
src/python/test/autotest.py
+38 -18
View File
@@ -86,34 +86,32 @@ SKIP_COLUMNS = set([
# return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)' # return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
def default_point(column_id): def default_lonlat(column_id):
''' '''
Returns default test point for the column_id. Returns default test point for the column_id.
''' '''
if column_id == 'whosonfirst.wof_disputed_geom': if column_id == 'whosonfirst.wof_disputed_geom':
return 'ST_SetSRID(ST_MakePoint(76.57, 33.78), 4326)' return (76.57, 33.78)
elif column_id == 'whosonfirst.wof_marinearea_geom': elif column_id == 'whosonfirst.wof_marinearea_geom':
return 'ST_SetSRID(ST_MakePoint(-68.47, 43.33), 4326)' return (-68.47, 43.33)
elif column_id in ('us.census.tiger.school_district_elementary', elif column_id in ('us.census.tiger.school_district_elementary',
'us.census.tiger.school_district_secondary', 'us.census.tiger.school_district_secondary',
'us.census.tiger.school_district_elementary_clipped', 'us.census.tiger.school_district_elementary_clipped',
'us.census.tiger.school_district_secondary_clipped'): 'us.census.tiger.school_district_secondary_clipped'):
return 'ST_SetSRID(ST_MakePoint(-73.7067, 40.7025), 4326)' return (40.7025, -73.7067)
elif column_id.startswith('uk'): elif column_id.startswith('uk'):
if 'WA' in column_id: if 'WA' in column_id:
return 'ST_SetSRID(ST_MakePoint(-3.184833526611328, 51.46844551219723), 4326)' return (51.46844551219723, -3.184833526611328)
else: else:
return 'ST_SetSRID(ST_MakePoint(-0.08883476257324219, 51.51461834694225), 4326)' return (51.51461834694225, -0.08883476257324219)
elif column_id.startswith('es'): elif column_id.startswith('es'):
return 'ST_SetSRID(ST_MakePoint(-2.51141249535454, 42.8226119029222), 4326)' return (42.8226119029222, -2.51141249535454)
elif column_id.startswith('us.zillow'): elif column_id.startswith('us.zillow'):
return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)' return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('mx.'): elif column_id.startswith('mx.'):
return 'ST_SetSRID(ST_MakePoint(-99.17019367218018, 19.41347699386547), 4326)' return (19.41347699386547, -99.17019367218018)
elif column_id.startswith('ca.'):
return 'ST_SetSRID(ST_MakePoint(-79.39716339111328, 43.65694347778308), 4326)'
elif column_id.startswith('th.'): elif column_id.startswith('th.'):
return 'ST_SetSRID(ST_MakePoint(100.49263000488281, 13.725377712079784), 4326)' return (13.725377712079784, 100.49263000488281)
# cols for French Guyana only # cols for French Guyana only
elif column_id in ('fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR' elif column_id in ('fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
, 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC' , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
@@ -127,13 +125,35 @@ def default_point(column_id):
, 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE' , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
, 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC' , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
, 'fr.insee.P12_ACTOCC15P_ILT45D'): , 'fr.insee.P12_ACTOCC15P_ILT45D'):
return 'ST_SetSRID(ST_MakePoint(-52.32908248901367, 4.938408371206558), 4326)' return (4.938408371206558, -52.32908248901367)
elif column_id.startswith('fr'): elif column_id.startswith('fr.'):
return 'ST_SetSRID(ST_MakePoint(2.3613739013671875, 48.860875144709475), 4326)' return (48.860875144709475, 2.3613739013671875)
elif column_id.startswith('ca'): elif column_id.startswith('ca.'):
return 'ST_SetSRID(ST_MakePoint(-79.37965393066406, 43.65594991256823), 4326)' return (43.65594991256823, -79.37965393066406)
elif column_id.startswith('us.census.'):
return (40.7, -73.9)
elif column_id.startswith('us.dma.'):
return (40.7, -73.9)
elif column_id.startswith('us.ihme.'):
return (40.7, -73.9)
elif column_id.startswith('us.bls.'):
return (40.7, -73.9)
elif column_id.startswith('us.qcew.'):
return (40.7, -73.9)
elif column_id.startswith('whosonfirst.'):
return (40.7, -73.9)
elif column_id.startswith('us.epa.'):
return (40.7, -73.9)
elif column_id.startswith('eu.'):
return (52.52207036136366, 13.40606689453125)
else: else:
return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)' raise Exception('No catalog point set for {}'.format(column_id))
def default_point(column_id):
lat, lng = default_lonlat(column_id)
return 'ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326)'.format(
lat=lat, lng=lng)
def default_area(column_id): def default_area(column_id):