Merge pull request #212 from CartoDB/develop

Release 1.1.2
Release 1.1.2 artifact
2016-11-11 17:09:56 +01:00 · 2016-11-11 17:08:04 +01:00 · 2016-11-11 17:05:28 +01:00 · 2016-11-09 22:11:20 +00:00 · 2016-11-09 16:58:25 -05:00 · 2016-11-09 16:47:14 -05:00
12 changed files with 3514 additions and 277 deletions
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,18 @@
+1.1.2 (2016-11-09)
+
+__Improvements__
+
+* Update public `OBS_GetMeasure` to use highest ranked boundary, aiming for 500
+  geoms. ([#190](https://github.com/CartoDB/observatory-extension/issues/190))
+* Update test generation to capture our raster tiles
+* Standardize the way we generate our test points for `autotest.py`
+* Add points for epa and eurostat
+* Should support database dump generated 20161109
+
+__API Changes (Internal)__
+
+* Add internal `_OBS_GetGeometryScores`
+
 1.1.1 (2016-10-14)

 __Improvements__
--- a/release/observatory--1.1.2.sql
+++ b/release/observatory--1.1.2.sql
--- a/release/observatory.control
+++ b/release/observatory.control
@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.1.1'
+default_version = '1.1.2'
 requires = 'postgis, postgres_fdw'
 superuser = true
 schema = cdb_observatory
--- a/scripts/generate_fixtures.py
+++ b/scripts/generate_fixtures.py
@@ -61,7 +61,7 @@ def get_tablename_query(column_id, boundary_id, timespan):
 METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
                   'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta',
                   'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom',
-                   'obs_meta_timespan', ]
+                   'obs_meta_timespan', 'obs_column_table_tile', ]

 FIXTURES = [
    ('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),
@@ -290,7 +290,7 @@ def main():
                "column_id IN ('{}', '{}')".format(numer_id, geom_id)
                for numer_id, geom_id, timespan in FIXTURES
            ])
-        elif tablename == 'obs_column_table':
+        elif tablename in ('obs_column_table', 'obs_column_table_tile', ):
            where = 'WHERE column_id IN ({numer_ids}) ' \
                    'OR column_id IN ({geom_ids}) ' \
                    'OR table_id IN ({table_ids}) '.format(
--- a/src/pg/observatory.control
+++ b/src/pg/observatory.control
@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.1.1'
+default_version = '1.1.2'
 requires = 'postgis, postgres_fdw'
 superuser = true
 schema = cdb_observatory
--- a/src/pg/sql/41_observatory_augmentation.sql
+++ b/src/pg/sql/41_observatory_augmentation.sql
@@ -360,6 +360,7 @@ DECLARE
  geom_colname TEXT;
  geom_geomref_colname TEXT;
  geom_tablename TEXT;
+  geom_id TEXT;
  result NUMERIC;
  sql TEXT;
  numer_name TEXT;
@@ -370,23 +371,6 @@ BEGIN

  geom := ST_SnapToGrid(geom, 0.000001);

-  EXECUTE
-     $query$
-     SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
-            denom_colname, denom_geomref_colname, denom_tablename,
-            geom_colname, geom_geomref_colname, geom_tablename, numer_name
-             FROM observatory.obs_meta
-             WHERE (geom_id = $1 OR ($1 = ''))
-               AND numer_id = $2
-               AND (numer_timespan = $3 OR ($3 = ''))
-             ORDER BY geom_weight DESC, numer_timespan DESC
-             LIMIT 1
-     $query$
-    INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
-         denom_colname, denom_geomref_colname, denom_tablename,
-         geom_colname, geom_geomref_colname, geom_tablename, numer_name
-    USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, '');
-
  IF ST_GeometryType(geom) = 'ST_Point' THEN
    geom_type := 'point';
  ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN
@@ -397,6 +381,37 @@ BEGIN
                    ST_GeometryType(geom);
  END IF;

+
+  EXECUTE
+     $query$
+      WITH meta AS (SELECT numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
+                    denom_colname, denom_geomref_colname, denom_tablename,
+                    geom_colname, geom_geomref_colname, geom_tablename,
+                    numer_name, geom_id
+                     FROM observatory.obs_meta
+                     WHERE (geom_id = $1 OR ($1 = ''))
+                       AND numer_id = $2
+                       AND (numer_timespan = $3 OR ($3 = ''))),
+           scores AS (SELECT *
+                      FROM cdb_observatory._OBS_GetGeometryScores($4,
+                           (SELECT Array_Agg(geom_id) FROM meta), 500))
+     SELECT meta.*
+     FROM meta, scores
+     WHERE meta.geom_id = scores.geom_id
+     ORDER BY score DESC
+     LIMIT 1
+    $query$
+    INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
+         denom_colname, denom_geomref_colname, denom_tablename,
+         geom_colname, geom_geomref_colname, geom_tablename, numer_name, geom_id
+    USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''),
+      CASE WHEN ST_GeometryType(geom) = 'ST_Point' THEN
+                st_buffer(geom::geography, 10)::geometry(geometry, 4326)
+           ELSE geom
+      END;
+
+  raise notice 'Using boundary %', geom_id;
+
  IF normalize ILIKE 'area' AND numer_aggregate ILIKE 'sum' THEN
    map_type := 'areaNormalized';
  ELSIF normalize ILIKE 'denominator' THEN
--- a/src/pg/sql/42_observatory_exploration.sql
+++ b/src/pg/sql/42_observatory_exploration.sql
@@ -254,7 +254,14 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableGeometries(
  geom_source TEXT,
  valid_numer BOOLEAN,
  valid_denom BOOLEAN,
-  valid_timespan BOOLEAN
+  valid_timespan BOOLEAN,
+  score NUMERIC,
+  numtiles BIGINT,
+  notnull_percent NUMERIC,
+  numgeoms NUMERIC,
+  percentfill NUMERIC,
+  estnumgeoms NUMERIC,
+  meanmediansize NUMERIC
 ) AS $$
 DECLARE
  geom_clause TEXT;
@@ -271,18 +278,27 @@ BEGIN
  RETURN QUERY
  EXECUTE
  format($string$
-    SELECT geom_id::TEXT,
-           geom_name::TEXT,
-           geom_description::TEXT,
-           geom_weight::NUMERIC,
-           NULL::TEXT geom_aggregate,
-           NULL::TEXT license,
-           NULL::TEXT source,
-    $1 = ANY(numers) valid_numer,
-    $2 = ANY(denoms) valid_denom,
-    $3 = ANY(timespans) valid_timespan
-    FROM observatory.obs_meta_geom
-    WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0)
+    WITH available_geoms AS (
+      SELECT geom_id::TEXT,
+             geom_name::TEXT,
+             geom_description::TEXT,
+             geom_weight::NUMERIC,
+             NULL::TEXT geom_aggregate,
+             NULL::TEXT license,
+             NULL::TEXT source,
+      $1 = ANY(numers) valid_numer,
+      $2 = ANY(denoms) valid_denom,
+      $3 = ANY(timespans) valid_timespan
+      FROM observatory.obs_meta_geom
+      WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0)
+    ), scores AS (
+      SELECT * FROM cdb_observatory._OBS_GetGeometryScores($5,
+        (SELECT ARRAY_AGG(geom_id) FROM available_geoms)
+      )
+    ) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms,
+             percentfill, estnumgeoms, meanmediansize
+      FROM available_geoms, scores
+      WHERE available_geoms.geom_id = scores.geom_id
  $string$, geom_clause)
  USING numer_id, denom_id, timespan, filter_tags, bounds;
  RETURN;
@@ -397,3 +413,76 @@ BEGIN
  RETURN;
 END
 $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores(
+  bounds Geometry(Geometry, 4326) DEFAULT NULL,
+  filter_geom_ids TEXT[] DEFAULT NULL,
+  desired_num_geoms INTEGER DEFAULT 3000
+) RETURNS TABLE (
+  score NUMERIC,
+  numtiles BIGINT,
+  geom_id TEXT,
+  notnull_percent NUMERIC,
+  numgeoms NUMERIC,
+  percentfill NUMERIC,
+  estnumgeoms NUMERIC,
+  meanmediansize NUMERIC
+) AS $$
+BEGIN
+  filter_geom_ids := COALESCE(filter_geom_ids, (ARRAY[])::TEXT[]);
+  RETURN QUERY
+  EXECUTE format($string$
+    SELECT
+      (1 / (abs(numgeoms - $3)
+        --* (1 / Coalesce(NullIf(notnull_percent, 0), 1))
+        --* (1 / Coalesce(NullIf(percentfill, 0), 0.0001))
+      ))::Numeric
+      AS score, *
+    FROM (
+      WITH clipped_geom AS (
+        SELECT column_id, table_id
+          , CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True)
+                 ELSE tile END clipped_tile
+          , tile
+        FROM observatory.obs_column_table_tile
+        WHERE ($1 IS NULL OR ST_Intersects($1, tile))
+          AND (column_id = ANY($2) OR cardinality($2) = 0)
+      ), clipped_geom_countagg AS (
+        SELECT column_id, table_id
+          , ST_CountAgg(clipped_tile, 2, True)::Numeric notnull_pixels
+          , ST_CountAgg(clipped_tile, 2, False)::Numeric pixels
+        FROM clipped_geom
+        GROUP BY column_id, table_id
+      ) SELECT
+        count(*)::BIGINT, a.column_id
+        , (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
+                THEN cdb_observatory.FIRST(notnull_pixels) / cdb_observatory.FIRST(pixels)
+                ELSE 1
+          END)::Numeric AS notnull_percent
+        , (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
+                THEN (ST_SummaryStatsAgg(clipped_tile, 2, True)).sum
+                ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 2, ST_PointOnSurface($1)), 0) * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(cdb_observatory.FIRST(tile), 0, 0)) * cdb_observatory.FIRST(pixels))
+          END)::Numeric AS numgeoms
+        , (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
+                THEN (ST_SummaryStatsAgg(clipped_tile, 3, True)).mean
+                ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 3, ST_PointOnSurface($1)), 0)
+          END)::Numeric AS percentfill
+        , ((ST_Area(ST_Transform($1, 3857)) / 1000000) / NullIf(
+            CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
+                 THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
+                 ELSE Coalesce(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
+            END, 0))::Numeric AS estnumgeoms
+        , (CASE WHEN cdb_observatory.FIRST(notnull_pixels) > 0
+                THEN (ST_SummaryStatsAgg(clipped_tile, 1, True)).mean
+                ELSE COALESCE(ST_Value(cdb_observatory.FIRST(tile), 1, ST_PointOnSurface($1)), 0)
+          END)::Numeric AS meanmediansize
+      FROM clipped_geom_countagg a, clipped_geom b
+      WHERE a.table_id = b.table_id
+        AND a.column_id = b.column_id
+      GROUP BY a.column_id, a.table_id
+      ORDER BY a.column_id, a.table_id
+    ) foo
+  $string$) USING bounds, filter_geom_ids, desired_num_geoms;
+  RETURN;
+END
+$$ LANGUAGE plpgsql;
--- a/src/pg/test/expected/42_observatory_exploration_test.out
+++ b/src/pg/test/expected/42_observatory_exploration_test.out
@@ -144,6 +144,48 @@ t
 _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014
 t
 (1 row)
+_obs_geometryscores_500m_buffer
+t
+(1 row)
+_obs_geometryscores_5km_buffer
+t
+(1 row)
+_obs_geometryscores_50km_buffer
+t
+(1 row)
+_obs_geometryscores_500km_buffer
+t
+(1 row)
+_obs_geometryscores_2500km_buffer
+t
+(1 row)
+_obs_geometryscores_numgeoms_500m_buffer
+t
+(1 row)
+_obs_geometryscores_numgeoms_5km_buffer
+t
+(1 row)
+_obs_geometryscores_numgeoms_50km_buffer
+t
+(1 row)
+_obs_geometryscores_numgeoms_500km_buffer
+t
+(1 row)
+_obs_geometryscores_numgeoms_2500km_buffer
+t
+(1 row)
+_obs_geometryscores_500km_buffer_50_geoms
+t
+(1 row)
+_obs_geometryscores_500km_buffer_500_geoms
+t
+(1 row)
+_obs_geometryscores_500km_buffer_2500_geoms
+t
+(1 row)
+_obs_geometryscores_500km_buffer_25000_geoms
+t
+(1 row)
 _total_pop_in_legacy_builder_metadata
 t
 (1 row)
--- a/src/pg/test/fixtures/drop_fixtures.sql
+++ b/src/pg/test/fixtures/drop_fixtures.sql
@@ -12,6 +12,7 @@ DROP TABLE IF EXISTS observatory.obs_meta_numer;
 DROP TABLE IF EXISTS observatory.obs_meta_denom;
 DROP TABLE IF EXISTS observatory.obs_meta_geom;
 DROP TABLE IF EXISTS observatory.obs_meta_timespan;
+DROP TABLE IF EXISTS observatory.obs_column_table_tile;
 DROP TABLE IF EXISTS observatory.obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963;
 DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
 DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
--- a/src/pg/test/fixtures/load_fixtures.sql
+++ b/src/pg/test/fixtures/load_fixtures.sql
--- a/src/pg/test/sql/42_observatory_exploration_test.sql
+++ b/src/pg/test/sql/42_observatory_exploration_test.sql
@@ -346,7 +346,134 @@ FROM cdb_observatory.OBS_GetAvailableTimespans(
 ) WHERE valid_denom = True)
 AS _obs_getavailablegeometries_foobarbaz_denom_not_in_2010_2014;

+--
+-- _OBS_GetGeometryScores tests
+--
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+             'us.census.tiger.zcta5', 'us.census.tiger.county']
+       AS _obs_geometryscores_500m_buffer
+       FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+             'us.census.tiger.zcta5', 'us.census.tiger.county']
+       AS _obs_geometryscores_5km_buffer
+       FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5',
+             'us.census.tiger.county', 'us.census.tiger.block_group']
+       AS _obs_geometryscores_50km_buffer
+       FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY[ 'us.census.tiger.county', 'us.census.tiger.zcta5',
+             'us.census.tiger.census_tract', 'us.census.tiger.block_group']
+      AS _obs_geometryscores_500km_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
+             'us.census.tiger.census_tract', 'us.census.tiger.block_group']
+      AS _obs_geometryscores_2500km_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text
+      = '{ "us.census.tiger.block_group" : 9, "us.census.tiger.census_tract" : 3, "us.census.tiger.zcta5" : 0, "us.census.tiger.county" : 0 }'
+      AS _obs_geometryscores_numgeoms_500m_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
+      '{ "us.census.tiger.block_group" : 899, "us.census.tiger.census_tract" : 328, "us.census.tiger.zcta5" : 45, "us.census.tiger.county" : 1 }'
+      AS _obs_geometryscores_numgeoms_5km_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 5000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
+      '{ "us.census.tiger.block_group" : 12112, "us.census.tiger.census_tract" : 3792, "us.census.tiger.zcta5" : 550, "us.census.tiger.county" : 13 }'
+      AS _obs_geometryscores_numgeoms_50km_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
+      '{ "us.census.tiger.block_group" : 48415, "us.census.tiger.census_tract" : 15776, "us.census.tiger.zcta5" : 6534, "us.census.tiger.county" : 295 }'
+      AS _obs_geometryscores_numgeoms_500km_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 500000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT JSON_Object_Agg(geom_id, numgeoms::int ORDER BY numgeoms DESC)::Text =
+      '{ "us.census.tiger.block_group" : 165489, "us.census.tiger.census_tract" : 55152, "us.census.tiger.zcta5" : 26500, "us.census.tiger.county" : 2551 }'
+      AS _obs_geometryscores_numgeoms_2500km_buffer
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 2500000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county']);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.county', 'us.census.tiger.zcta5',
+             'us.census.tiger.census_tract', 'us.census.tiger.block_group']
+      AS _obs_geometryscores_500km_buffer_50_geoms
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county'], 50);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC)
+      = ARRAY['us.census.tiger.zcta5', 'us.census.tiger.county',
+              'us.census.tiger.census_tract', 'us.census.tiger.block_group']
+      AS _obs_geometryscores_500km_buffer_500_geoms
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county'], 500);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.census_tract', 'us.census.tiger.zcta5',
+             'us.census.tiger.county', 'us.census.tiger.block_group']
+      AS _obs_geometryscores_500km_buffer_2500_geoms
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county'], 2500);
+
+SELECT ARRAY_AGG(geom_id ORDER BY score DESC) =
+       ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+             'us.census.tiger.zcta5', 'us.census.tiger.county']
+      AS _obs_geometryscores_500km_buffer_25000_geoms
+      FROM cdb_observatory._OBS_GetGeometryScores(
+  ST_Buffer(ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)::Geography, 50000)::Geometry(Geometry, 4326),
+  ARRAY['us.census.tiger.block_group', 'us.census.tiger.census_tract',
+        'us.census.tiger.zcta5', 'us.census.tiger.county'], 25000);
+
+--
 -- OBS_LegacyBuilderMetadata tests
+--

 SELECT 'us.census.acs.B01003001' IN (SELECT
  (jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
@@ -367,4 +494,3 @@ SELECT 'us.census.acs.B19013001' NOT IN (SELECT
  (jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
  FROM cdb_observatory.OBS_LegacyBuilderMetadata('sum')
 ) AS _median_income_not_in_legacy_builder_metadata_sums;
-
--- a/src/python/test/autotest.py
+++ b/src/python/test/autotest.py
@@ -86,34 +86,32 @@ SKIP_COLUMNS = set([
 #        return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'


-def default_point(column_id):
+def default_lonlat(column_id):
    '''
    Returns default test point for the column_id.
    '''
    if column_id == 'whosonfirst.wof_disputed_geom':
-        return 'ST_SetSRID(ST_MakePoint(76.57, 33.78), 4326)'
+        return (76.57, 33.78)
    elif column_id == 'whosonfirst.wof_marinearea_geom':
-        return 'ST_SetSRID(ST_MakePoint(-68.47, 43.33), 4326)'
+        return (-68.47, 43.33)
    elif column_id in ('us.census.tiger.school_district_elementary',
                       'us.census.tiger.school_district_secondary',
                       'us.census.tiger.school_district_elementary_clipped',
                       'us.census.tiger.school_district_secondary_clipped'):
-        return 'ST_SetSRID(ST_MakePoint(-73.7067, 40.7025), 4326)'
+        return (40.7025, -73.7067)
    elif column_id.startswith('uk'):
        if 'WA' in column_id:
-            return 'ST_SetSRID(ST_MakePoint(-3.184833526611328, 51.46844551219723), 4326)'
+            return (51.46844551219723, -3.184833526611328)
        else:
-            return 'ST_SetSRID(ST_MakePoint(-0.08883476257324219, 51.51461834694225), 4326)'
+            return (51.51461834694225, -0.08883476257324219)
    elif column_id.startswith('es'):
-        return 'ST_SetSRID(ST_MakePoint(-2.51141249535454, 42.8226119029222), 4326)'
+        return (42.8226119029222, -2.51141249535454)
    elif column_id.startswith('us.zillow'):
-        return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)'
+        return (28.3305906291771, -81.3544048197256)
    elif column_id.startswith('mx.'):
-        return 'ST_SetSRID(ST_MakePoint(-99.17019367218018, 19.41347699386547), 4326)'
-    elif column_id.startswith('ca.'):
-        return 'ST_SetSRID(ST_MakePoint(-79.39716339111328, 43.65694347778308), 4326)'
+        return (19.41347699386547, -99.17019367218018)
    elif column_id.startswith('th.'):
-        return 'ST_SetSRID(ST_MakePoint(100.49263000488281, 13.725377712079784), 4326)'
+        return (13.725377712079784, 100.49263000488281)
    # cols for French Guyana only
    elif column_id in ('fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
                       , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
@@ -127,13 +125,35 @@ def default_point(column_id):
                       , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
                       , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
                       , 'fr.insee.P12_ACTOCC15P_ILT45D'):
-        return 'ST_SetSRID(ST_MakePoint(-52.32908248901367, 4.938408371206558), 4326)'
-    elif column_id.startswith('fr'):
-        return 'ST_SetSRID(ST_MakePoint(2.3613739013671875, 48.860875144709475), 4326)'
-    elif column_id.startswith('ca'):
-        return 'ST_SetSRID(ST_MakePoint(-79.37965393066406, 43.65594991256823), 4326)'
+        return (4.938408371206558, -52.32908248901367)
+    elif column_id.startswith('fr.'):
+        return (48.860875144709475, 2.3613739013671875)
+    elif column_id.startswith('ca.'):
+        return (43.65594991256823, -79.37965393066406)
+    elif column_id.startswith('us.census.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('us.dma.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('us.ihme.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('us.bls.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('us.qcew.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('whosonfirst.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('us.epa.'):
+        return (40.7, -73.9)
+    elif column_id.startswith('eu.'):
+        return (52.52207036136366, 13.40606689453125)
    else:
-        return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
+        raise Exception('No catalog point set for {}'.format(column_id))
+
+
+def default_point(column_id):
+    lat, lng = default_lonlat(column_id)
+    return 'ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326)'.format(
+        lat=lat, lng=lng)


 def default_area(column_id):
Author	SHA1	Message	Date
Mario de Frutos	60b723de92	Merge pull request #212 from CartoDB/develop Release 1.1.2	2016-11-11 17:09:56 +01:00
Mario de Frutos	7e04c38c3a	Release 1.1.2 artifact	2016-11-11 17:08:04 +01:00
Mario de Frutos	45dea25ec0	Merge pull request #211 from CartoDB/release-v-1.1.2 Release v 1.1.2	2016-11-11 17:05:28 +01:00
John Krauss	39836ea321	update NEWS.md	2016-11-09 22:11:20 +00:00
john krauss	17d343a756	Merge pull request #209 from CartoDB/use-rasters Use rasters	2016-11-09 16:58:25 -05:00
john krauss	c7c8a6676a	Merge pull request #210 from CartoDB/eu-epa-testpoints add test points for EU and EPA, make it easier to work with meta.py	2016-11-09 16:47:14 -05:00
John Krauss	be4b5abbfa	use highest ranked geom for obs_getmeasure, simplify scoring	2016-11-07 23:57:33 +00:00
John Krauss	8dad88a6b3	fix minor bug in _obs_getgeometryscores with FIRST, add tests	2016-11-07 21:26:44 +00:00
John Krauss	7e6489f2a1	add test points for EU and EPA, make it easier to work with meta.py	2016-11-07 16:50:00 +00:00
John Krauss	9fdca9161c	minor stylistic fix	2016-11-04 15:32:25 +00:00
John Krauss	785a5eed29	obs_getgeometryscores and usage by obs_getavailablegeometries	2016-11-02 21:11:38 +00:00