Allow for target_geoms and target_area override on column-by-column basis

2017-03-21 17:26:02 +00:00
parent 217ca2d84d
commit 61552adba4
8 changed files with 1208 additions and 941 deletions
@@ -261,3 +261,31 @@ t|t
 ary_type|obs_getdata_api_geomrefs_args_string_return
 t|t
 (1 row)
+setseed
+
+(1 row)
+bg_sample|bg_max_error|bg_avg_error|bg_min_error
+1|t|t|t
+2|t|t|t
+3|t|t|t
+5|t|t|t
+10|t|t|t
+25|t|t|t
+50|t|t|t
+100|t|t|t
+2085|t|t|t
+(9 rows)
+tract_sample|tract_max_error|tract_avg_error|tract_min_error
+1|t|t|t
+2|t|t|t
+3|t|t|t
+5|t|t|t
+10|t|t|t
+25|t|t|t
+50|t|t|t
+100|t|t|t
+761|t|t|t
+(9 rows)
+no_bg_point_error
+t
+(1 row)
@@ -159,21 +159,36 @@ t
 _obs_geometryscores_2500km_buffer
 t
 (1 row)
-_obs_geometryscores_numgeoms_500m_buffer
-t
-(1 row)
-_obs_geometryscores_numgeoms_5km_buffer
-t
-(1 row)
-_obs_geometryscores_numgeoms_50km_buffer
-t
-(1 row)
-_obs_geometryscores_numgeoms_500km_buffer
-t
-(1 row)
-_obs_geometryscores_numgeoms_2500km_buffer
-t
-(1 row)
+column_id|_obs_geometryscores_numgeoms_500m_buffer
+us.census.tiger.block_group|2
+us.census.tiger.census_tract|1
+us.census.tiger.zcta5|0
+us.census.tiger.county|0
+(4 rows)
+column_id|_obs_geometryscores_numgeoms_5km_buffer
+us.census.tiger.block_group|244
+us.census.tiger.census_tract|78
+us.census.tiger.zcta5|9
+us.census.tiger.county|0
+(4 rows)
+column_id|_obs_geometryscores_numgeoms_50km_buffer
+us.census.tiger.block_group|10817
+us.census.tiger.census_tract|3396
+us.census.tiger.zcta5|484
+us.census.tiger.county|11
+(4 rows)
+column_id|_obs_geometryscores_numgeoms_500km_buffer
+us.census.tiger.block_group|48567
+us.census.tiger.census_tract|15823
+us.census.tiger.zcta5|6466
+us.census.tiger.county|295
+(4 rows)
+column_id|_obs_geometryscores_numgeoms_2500km_buffer
+us.census.tiger.block_group|165852
+us.census.tiger.census_tract|55283
+us.census.tiger.zcta5|27046
+us.census.tiger.county|2551
+(4 rows)
 _obs_geometryscores_500km_buffer_50_geoms
 t
 (1 row)
@@ -186,6 +201,12 @@ t
 _obs_geometryscores_500km_buffer_25000_geoms
 t
 (1 row)
+testarea_uses_tract
+t
+(1 row)
+points_use_bg
+t
+(1 row)
 _total_pop_in_legacy_builder_metadata
 t
 (1 row)
@@ -798,3 +798,146 @@ SELECT json_typeof(data->0->'value') = 'array' ary_type,
 AS OBS_GetData_API_geomrefs_args_string_return
 FROM cdb_observatory.obs_getdata(array['36047'],
      '[{"numer_type": "text", "numer_colname": "obs_getboundarybyid", "api_method": "obs_getboundarybyid", "api_args": ["us.census.tiger.county"]}]');
+
+-- Ensure consistent results below.
+select setseed(0);
+
+-- Check that random assortment of block groups in Brooklyn return accurate data
+WITH _geoms AS (
+  SELECT
+    (data->0->>'value')::geometry the_geom,
+    data->0->>'geomref' geom_ref,
+    (data->1->>'value')::numeric total_pop
+  FROM cdb_observatory.OBS_GetData(
+    array[(st_buffer(cdb_observatory._testpoint(), 0.2), 1)::geomval],
+    (SELECT cdb_observatory.OBS_GetMeta(ST_MakeEnvelope(-179, 89, 179, -89, 4326),
+      '[{"geom_id": "us.census.tiger.block_group"},
+        {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group", "normalization": "predenom"}]')),
+    FALSE
+  )
+  WHERE data->0->>'geomref' LIKE '36047%'
+  ORDER BY RANDOM()
+), geoms AS (
+  SELECT *, row_number() OVER () cartodb_id FROM _geoms
+), samples AS (
+  SELECT COUNT(*) cnt, unnest(ARRAY[1, 2, 3, 5, 10, 25, 50, 100, COUNT(*)]) sample FROM geoms
+), filtered AS (
+  SELECT * FROM geoms, samples WHERE cartodb_id % (cnt / sample) = 0
+), summary AS (
+  SELECT sample, ST_SetSRID(ST_Extent(the_geom), 4326) extent,
+    COUNT(*)::INT cnt,
+    ARRAY_AGG((the_geom, cartodb_id)::geomval) geomvals,
+    SUM(ST_Area(the_geom))::Numeric sumarea
+  FROM filtered
+  GROUP BY sample
+), meta AS (
+  SELECT sample, cdb_observatory.OBS_GetMeta(extent,
+    ('[{"numer_id": "us.census.acs.B01003001", "normalization": "predenom", "target_area": ' || sumarea || '}]')::JSON,
+    1, 1, cnt) meta
+  FROM summary
+  GROUP BY sample, extent, cnt, sumarea
+), results AS (
+  SELECT summary.sample, id, meta->0->>'geom_id' geom_id, (data->0->>'value')::Numeric as val
+  FROM summary, meta, LATERAL cdb_observatory.OBS_GetData(geomvals, meta) data
+  WHERE summary.sample = meta.sample
+) SELECT sample bg_sample
+ , MAX(100 * abs((geoms.total_pop - val) / Coalesce(NullIf(total_pop, 0), NULL)))::Numeric(10, 2) < 10 bg_max_error
+ , AVG(100 * abs((geoms.total_pop - val) / Coalesce(NullIf(total_pop, 0), NULL)))::Numeric(10, 2) < 10 bg_avg_error
+ , MIN(100 * abs((geoms.total_pop - val) / Coalesce(NullIf(total_pop, 0), NULL)))::Numeric(10, 2) < 10 bg_min_error
+FROM geoms, results
+WHERE cartodb_id = id
+GROUP BY sample
+ORDER BY sample
+;
+
+-- Check that random assortment of tracts in Brooklyn return accurate data
+WITH _geoms AS (
+  SELECT
+    (data->0->>'value')::geometry the_geom,
+    data->0->>'geomref' geom_ref,
+    (data->1->>'value')::numeric total_pop
+  FROM cdb_observatory.OBS_GetData(
+    array[(st_buffer(cdb_observatory._testpoint(), 0.2), 1)::geomval],
+    (SELECT cdb_observatory.OBS_GetMeta(ST_MakeEnvelope(-179, 89, 179, -89, 4326),
+      '[{"geom_id": "us.census.tiger.census_tract"},
+        {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.census_tract", "normalization": "predenom"}]')),
+    FALSE
+  )
+  WHERE data->0->>'geomref' LIKE '36047%'
+  ORDER BY RANDOM()
+), geoms AS (
+  SELECT *, row_number() OVER () cartodb_id FROM _geoms
+), samples AS (
+  SELECT COUNT(*) cnt, unnest(ARRAY[1, 2, 3, 5, 10, 25, 50, 100, COUNT(*)]) sample FROM geoms
+), filtered AS (
+  SELECT * FROM geoms, samples WHERE cartodb_id % (cnt / sample) = 0
+), summary AS (
+  SELECT sample, ST_SetSRID(ST_Extent(the_geom), 4326) extent,
+    COUNT(*)::INT cnt,
+    ARRAY_AGG((the_geom, cartodb_id)::geomval) geomvals,
+    SUM(ST_Area(the_geom))::Numeric sumarea
+  FROM filtered
+  GROUP BY sample
+), meta AS (
+  SELECT sample, cdb_observatory.OBS_GetMeta(extent,
+    ('[{"numer_id": "us.census.acs.B01003001", "normalization": "predenom", "target_area": ' || sumarea || '}]')::JSON,
+    1, 1, cnt) meta
+  FROM summary
+  GROUP BY sample, extent, cnt, sumarea
+), results AS (
+  SELECT summary.sample, id, meta->0->>'geom_id' geom_id, (data->0->>'value')::Numeric as val
+  FROM summary, meta, LATERAL cdb_observatory.OBS_GetData(geomvals, meta) data
+  WHERE summary.sample = meta.sample
+) SELECT sample tract_sample
+ , MAX(100 * abs((geoms.total_pop - val) / Coalesce(NullIf(total_pop, 0), NULL)))::Numeric(10, 2) < 10 tract_max_error
+ , AVG(100 * abs((geoms.total_pop - val) / Coalesce(NullIf(total_pop, 0), NULL)))::Numeric(10, 2) < 10 tract_avg_error
+ , MIN(100 * abs((geoms.total_pop - val) / Coalesce(NullIf(total_pop, 0), NULL)))::Numeric(10, 2) < 10 tract_min_error
+FROM geoms, results
+WHERE cartodb_id = id
+GROUP BY sample
+ORDER BY sample
+;
+
+-- Check that random assortment of block group points in Brooklyn return accurate data
+WITH _geoms AS (
+  SELECT
+    ST_PointOnSurface((data->0->>'value')::geometry) the_geom,
+    data->0->>'geomref' geom_ref,
+    (data->1->>'value')::numeric total_pop
+  FROM cdb_observatory.OBS_GetData(
+    array[(st_buffer(cdb_observatory._testpoint(), 0.2), 1)::geomval],
+    (SELECT cdb_observatory.OBS_GetMeta(ST_MakeEnvelope(-179, 89, 179, -89, 4326),
+      '[{"geom_id": "us.census.tiger.block_group"},
+        {"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.block_group", "normalization": "predenom"}]')),
+    FALSE
+  )
+  WHERE data->0->>'geomref' LIKE '36047%'
+), geoms AS (
+  SELECT *, row_number() OVER () cartodb_id FROM _geoms
+), samples AS (
+  SELECT COUNT(*) cnt, unnest(ARRAY[1, 2, 3, 5, 10, 25, 50, 100, COUNT(*)]) sample FROM geoms
+), filtered AS (
+  SELECT * FROM geoms, samples WHERE cartodb_id % (cnt / sample) = 0
+), summary AS (
+  SELECT sample, ST_SetSRID(ST_Extent(the_geom), 4326) extent,
+    COUNT(*)::INT cnt,
+    ARRAY_AGG((the_geom, cartodb_id)::geomval) geomvals,
+    SUM(ST_Area(the_geom))::Numeric sumarea
+  FROM filtered
+  GROUP BY sample
+), meta AS (
+  SELECT sample, cdb_observatory.OBS_GetMeta(extent,
+    ('[{"numer_id": "us.census.acs.B01003001", "normalization": "predenom", "target_area": ' || sumarea || '}]')::JSON,
+    1, 1, cnt) meta
+  FROM summary
+  GROUP BY sample, extent, cnt, sumarea
+), results AS (
+  SELECT summary.sample, id, meta->0->>'geom_id' geom_id, (data->0->>'value')::Numeric as val
+  FROM summary, meta, LATERAL cdb_observatory.OBS_GetData(geomvals, meta) data
+  WHERE summary.sample = meta.sample
+) SELECT
+ BOOL_AND(abs((geoms.total_pop - val) /
+      Coalesce(NullIf(total_pop, 0), 1)) = 0) is True no_bg_point_error
+FROM geoms, results
+WHERE cartodb_id = id
+;
@@ -360,9 +360,9 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
        'us.census.tiger.county', 'us.census.tiger.zcta5'])
      WHERE table_id LIKE '%2015%';

-SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
-       ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
-             'us.census.tiger.zcta5', 'us.census.tiger.county']
+SELECT ARRAY_AGG(column_id ORDER BY score DESC)
+       = ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+             'us.census.tiger.county', 'us.census.tiger.zcta5']
       AS _obs_geometryscores_5km_buffer
       FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
@@ -390,60 +390,55 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
        'us.census.tiger.zcta5', 'us.census.tiger.county'])
      WHERE table_id LIKE '%2015%';

-SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
-       ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
-             'us.census.tiger.census_tract', 'us.census.tiger.block_group']
+SELECT ARRAY_AGG(column_id ORDER BY score DESC)
+       = ARRAY['us.census.tiger.county', 'us.census.tiger.census_tract',
+             'us.census.tiger.zcta5', 'us.census.tiger.block_group']
      AS _obs_geometryscores_2500km_buffer
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
-  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
-        'us.census.tiger.zcta5', 'us.census.tiger.county'])
+  ARRAY['us.census.tiger.county', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.block_group'])
      WHERE table_id LIKE '%2015%';

-SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text
-      = '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }'
-      AS _obs_geometryscores_numgeoms_500m_buffer
+SELECT column_id, numgeoms::int AS _obs_geometryscores_numgeoms_500m_buffer
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
        'us.census.tiger.zcta5', 'us.census.tiger.county'])
-      WHERE table_id LIKE '%2015%';
+      WHERE table_id LIKE '%2015%'
+      ORDER BY numgeoms DESC;

-SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
-      '{ "us.census.tiger.block_group" : 880, "us.census.tiger.census_tract" : 310, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
-      AS _obs_geometryscores_numgeoms_5km_buffer
+SELECT column_id, numgeoms::int AS _obs_geometryscores_numgeoms_5km_buffer
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
        'us.census.tiger.zcta5', 'us.census.tiger.county'])
-      WHERE table_id LIKE '%2015%';
+      WHERE table_id LIKE '%2015%'
+      ORDER BY numgeoms DESC;

-SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
-      '{ "us.census.tiger.block_group" : 11531, "us.census.tiger.census_tract" : 3601, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 14 }'
-      AS _obs_geometryscores_numgeoms_50km_buffer
+SELECT column_id, numgeoms::int AS _obs_geometryscores_numgeoms_50km_buffer
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
        'us.census.tiger.zcta5', 'us.census.tiger.county'])
-      WHERE table_id LIKE '%2015%';
+      WHERE table_id LIKE '%2015%'
+      ORDER BY numgeoms DESC;

-SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
-      '{ "us.census.tiger.block_group" : 48917, "us.census.tiger.census_tract" : 15969, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 314 }'
-      AS _obs_geometryscores_numgeoms_500km_buffer
+SELECT column_id, numgeoms::int AS _obs_geometryscores_numgeoms_500km_buffer
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
        'us.census.tiger.zcta5', 'us.census.tiger.county'])
-      WHERE table_id LIKE '%2015%';
+      WHERE table_id LIKE '%2015%'
+      ORDER BY numgeoms DESC;

-SELECT JSON_Object_Agg(column_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
-      '{ "us.census.tiger.block_group" : 169191, "us.census.tiger.census_tract" : 56469, "us.census.tiger.zcta5" : 26525, "us.census.tiger.county" : 2753 }'
-      AS _obs_geometryscores_numgeoms_2500km_buffer
+SELECT column_id, numgeoms::int AS _obs_geometryscores_numgeoms_2500km_buffer
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
        'us.census.tiger.zcta5', 'us.census.tiger.county'])
-      WHERE table_id LIKE '%2015%';
+      WHERE table_id LIKE '%2015%'
+      ORDER BY numgeoms DESC;

 SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
       ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
@@ -475,9 +470,9 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
        'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500)
      WHERE table_id LIKE '%2015%';

-SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
-       ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
-             'us.census.tiger.zcta5', 'us.census.tiger.county']
+SELECT ARRAY_AGG(column_id ORDER BY score DESC)
+       = ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+               'us.census.tiger.county', 'us.census.tiger.zcta5']
      AS _obs_geometryscores_500km_buffer_25000_geoms
      FROM cdb_observatory._OBS_GetGeometryScores(
  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
@@ -485,6 +480,44 @@ SELECT ARRAY_AGG(column_id ORDER BY score DESC) =
        'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000)
      WHERE table_id LIKE '%2015%';

+-- Check that one small geom approximates tract data
+WITH geoms AS (SELECT cdb_observatory._testarea() the_geom),
+summary AS (SELECT ST_SetSRID(ST_Extent(the_geom), 4326) extent,
+                   COUNT(*)::INT cnt,
+                   SUM(ST_Area(the_geom))::Numeric sumarea
+            FROM geoms)
+SELECT column_id = 'us.census.tiger.census_tract' testarea_uses_tract
+FROM summary, LATERAL (
+  SELECT *
+  FROM cdb_observatory._OBS_GetGeometryScores(extent,
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county'],
+        cnt, sumarea)) foo
+ORDER BY score DESC LIMIT 1;
+
+-- Check that randomly distributed points always use smallest geometry if we
+-- order by numgeoms desc
+WITH geoms as (SELECT UNNEST(ARRAY[
+    cdb_observatory._testpoint(),
+    st_translate(cdb_observatory._testpoint(), -0.003, 0),
+    st_translate(cdb_observatory._testpoint(), -0.006, 0)
+]) the_geom),
+summary as (SELECT
+  ST_SetSRID(ST_Extent(the_geom), 4326) extent,
+  SUM(ST_Area(the_geom))::Numeric area,
+  COUNT(*)::INTEGER cnt
+  FROM geoms
+)
+SELECT column_id = 'us.census.tiger.block_group' points_use_bg
+      FROM summary, LATERAL (
+        SELECT * FROM cdb_observatory._OBS_GetGeometryScores(
+          extent,
+          ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+                'us.census.tiger.zcta5', 'us.census.tiger.county'],
+        cnt, area)) foo
+      WHERE table_id LIKE '%2015%'
+      ORDER BY numgeoms DESC LIMIT 1;
+
 --
 -- OBS_LegacyBuilderMetadata tests
 --