Merge pull request #93 from CartoDB/develop

Version 0.0.3

release/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '0.0.2'
+default_version = '0.0.3'
 requires = 'postgis'
 superuser = true
 schema = cdb_observatory

scripts/autotest.py

@@ -56,7 +56,7 @@ def default_point(column_id):
                        'us.census.tiger.school_district_secondary_clipped'):
         return 'CDB_LatLng(40.7025, -73.7067)'
     elif column_id.startswith('es.ine'):
-        return 'CDB_LatLng(40.39, -3.7)'
+        return 'CDB_LatLng(42.8226119029222, -2.51141249535454)'
     elif column_id.startswith('us.zillow'):
         return 'CDB_LatLng(28.3305906291771, -81.3544048197256)'
     else:

scripts/generate_fixtures.py

@@ -46,6 +46,7 @@ fixtures = [
     ('us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014'),
     ('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2014-01'),
     ('us.zillow.AllHomes_Zhvi', 'us.census.tiger.zcta5', '2016-03'),
+    ('whosonfirst.wof_country_geom', 'whosonfirst.wof_country_geom', '2016'),
     ('us.census.tiger.zcta5_clipped', 'us.census.tiger.zcta5_clipped', '2014'),
     ('us.census.tiger.block_group_clipped', 'us.census.tiger.block_group_clipped', '2014'),
 ]
@@ -76,10 +77,15 @@ with open('src/pg/test/fixtures/load_fixtures.sql', 'w') as outfile:
 
         for tablename, colname, boundary_id in unique_tables:
             if 'zcta5' in boundary_id:
-                where = '11%'
+                where = '\'11%\''
+                compare = 'LIKE'
+            elif 'whosonfirst' in boundary_id:
+                where = '(\'85632785\',\'85633051\',\'85633111\',\'85633147\',\'85633253\',\'85633267\')'
+                compare = 'IN'
             else:
-                where = '36047%'
-            print ' '.join([select_star(tablename), "WHERE {} LIKE '{}'".format(colname, where)])
-            cdb.dump(' '.join([select_star(tablename), "WHERE {} LIKE '{}'".format(colname, where)]),
-                    tablename, outfile, schema='observatory')
+                where = '\'36047%\''
+                compare = 'LIKE'
+            print ' '.join([select_star(tablename), "WHERE {}::text {} {}".format(colname, compare, where)])
+            cdb.dump(' '.join([select_star(tablename), "WHERE {}::text {} {}".format(colname, compare, where)]),
+                     tablename, outfile, schema='observatory')
             dropfiles.write('DROP TABLE IF EXISTS observatory.{};\n'.format(tablename))

src/pg/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '0.0.2'
+default_version = '0.0.3'
 requires = 'postgis'
 superuser = true
 schema = cdb_observatory

src/pg/sql/41_observatory_augmentation.sql

@@ -520,9 +520,37 @@ DECLARE
   q_sum text;
   q text;
   i NUMERIC;
+  data_geoid_colname text;
+  geom_geoid_colname text;
 BEGIN
 
-  q_select := 'SELECT geoid, ';
+  -- TODO we're assuming our geom_table has only one geom_ref column
+  --      we *really* should pass in both geom_table_name and boundary_id
+  -- TODO tablename should not be passed here (use boundary_id)
+  EXECUTE
+    format('SELECT ct.colname
+              FROM observatory.obs_column_to_column c2c,
+                   observatory.obs_column_table ct,
+                   observatory.obs_table t
+             WHERE c2c.reltype = ''geom_ref''
+               AND ct.column_id = c2c.source_id
+               AND ct.table_id = t.id
+               AND t.tablename = %L'
+     , (data_table_info)[1]->>'tablename')
+  INTO data_geoid_colname;
+  EXECUTE
+    format('SELECT ct.colname
+              FROM observatory.obs_column_to_column c2c,
+                   observatory.obs_column_table ct,
+                   observatory.obs_table t
+             WHERE c2c.reltype = ''geom_ref''
+               AND ct.column_id = c2c.source_id
+               AND ct.table_id = t.id
+               AND t.tablename = %L'
+   , geom_table_name)
+  INTO geom_geoid_colname;
+
+  q_select := format('SELECT %I, ', data_geoid_colname);
   q_sum    := 'SELECT Array[';
 
   FOR i IN 1..array_upper(data_table_info, 1)
@@ -543,22 +571,22 @@ BEGIN
     END IF;
  END LOOP;
 
-  q = format('
+  q := format('
     WITH _overlaps As (
       SELECT ST_Area(
         ST_Intersection($1, a.the_geom)
       ) / ST_Area(a.the_geom) As overlap_fraction,
-      geoid
+      %I
       FROM observatory.%I As a
       WHERE $1 && a.the_geom
     ),
     values As (
-    ', geom_table_name);
+    ', geom_geoid_colname, geom_table_name);
 
   q := q || q_select || format('FROM observatory.%I ', ((data_table_info)[1]->>'tablename'));
 
-  q := q || ' ) ' || q_sum || ' ]::numeric[] FROM _overlaps, values
-  WHERE values.geoid = _overlaps.geoid';
+  q := format(q || ' ) ' || q_sum || ' ]::numeric[] FROM _overlaps, values
+  WHERE values.%I = _overlaps.%I', geom_geoid_colname, geom_geoid_colname);
 
   EXECUTE
     q

src/pg/sql/44_observatory_geometries.sql

@@ -161,9 +161,9 @@ BEGIN
 
   RAISE NOTICE 'target_table: %, geoid_colname: %', target_table, geoid_colname;
 
-  -- return name of geometry id column
+  -- return geometry id column value
   EXECUTE format(
-    'SELECT %I
+    'SELECT %I::text
      FROM observatory.%I
      WHERE ST_Intersects($1, the_geom)
      LIMIT 1', geoid_colname, target_table)
@@ -282,7 +282,7 @@ BEGIN
   -- return first boundary in intersections
   RETURN QUERY
   EXECUTE format(
-    'SELECT %I, %I
+    'SELECT %I, %I::text
      FROM observatory.%I
      WHERE ST_%s($1, the_geom)
      ', geom_colname, geoid_colname, target_table, overlap_type)
@@ -409,7 +409,7 @@ BEGIN
     RAISE EXCEPTION 'Overlap type ''%'' is not an accepted type (choose intersects, within, or contains)', overlap_type;
   ELSIF ST_GeometryType(geom) NOT IN ('ST_Polygon', 'ST_MultiPolygon')
   THEN
-      RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom);
+    RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom);
   END IF;
 
   SELECT * INTO geoid_colname, target_table, geom_colname
@@ -428,7 +428,7 @@ BEGIN
   -- return first boundary in intersections
   RETURN QUERY
   EXECUTE format(
-    'SELECT ST_PointOnSurface(%I) As %s, %I
+    'SELECT ST_PointOnSurface(%I) As %s, %I::text
      FROM observatory.%I
      WHERE ST_%s($1, the_geom)
      ', geom_colname, geom_colname, geoid_colname, target_table, overlap_type)
@@ -565,6 +565,7 @@ BEGIN
           geom_c.type ILIKE 'geometry' AND
           geom_c.id = '%s'
     $string$, boundary_id, boundary_id);
+  RETURN;
     --  AND geom_t.timespan = '%s' <-- put in requested year
     -- TODO: filter by clipped vs. not so appropriate tablename are unique
     --       so the limit 1 can be removed

src/pg/test/expected/44_observatory_geometries_test.out

@@ -57,6 +57,9 @@ t
 obs_getboundariesbygeometry_tracts_around_null_island
 t
 (1 row)
+obs_getboundariesbygeometry_wof
+t
+(1 row)
 obs_getboundariesbypointandradius_around_cartodb
 t
 (1 row)
@@ -90,3 +93,6 @@ t
 geoid_name_matches|table_name_matches|geom_name_matches
 t|t|t
 (1 row)
+geoid_name_matches|table_name_matches|geom_name_matches
+t|t|t
+(1 row)

src/pg/test/fixtures/drop_fixtures.sql

@@ -13,6 +13,7 @@ DROP TABLE IF EXISTS observatory.obs_7615e8622a68bfc5fe37c69c9880edfb40250103;
 DROP TABLE IF EXISTS observatory.obs_1babf5a26a1ecda5fb74963e88408f71d0364b81;
 DROP TABLE IF EXISTS observatory.obs_8764a6b439a4f8714f54d4b3a157bc5e36519066;
 DROP TABLE IF EXISTS observatory.obs_b393b5b88c6adda634b2071a8005b03c551b609a;
+DROP TABLE IF EXISTS observatory.obs_1ea93bbc109c87c676b3270789dacf7a1430db6c;
 DROP TABLE IF EXISTS observatory.obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7;
 DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
 DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;

src/pg/test/sql/44_observatory_geometries_test.sql

@@ -171,6 +171,17 @@ FROM (
   ORDER BY geom_refs ASC
 ) As m(the_geom, geom_refs);
 
+-- who's on first boundaries
+SELECT
+  array_agg(geom_refs) = Array['85632785','85633051','85633111','85633147','85633253','85633267'] As OBS_GetBoundariesByGeometry_wof
+FROM (
+  SELECT *
+  FROM cdb_observatory.OBS_GetBoundariesByGeometry(
+    ST_MakeEnvelope(-4.66, 40.43, 14.48, 51.99, 4326),
+    'whosonfirst.wof_country_geom')
+  ORDER BY geom_refs ASC
+) As m(the_geom, geom_refs);
+
 -- OBS_GetBoundariesByPointAndRadius
 
 -- check that all census tracts intersecting with the geometry are returned
@@ -320,6 +331,7 @@ FROM (
 ) As m(the_geom, geom_refs);
 
 -- _OBS_GetGeometryMetadata
+-- get metadata for census tracts
 
 SELECT
   geoid_colname = 'geoid' As geoid_name_matches,
@@ -328,4 +340,11 @@ SELECT
 FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract')
      As m(geoid_colname, target_table, geom_colname);
 
+-- get metadata for boundaries with clipped geometries
+ SELECT
+   geoid_colname = 'geoid' As geoid_name_matches,
+   target_table = 'obs_fcd4e4f5610f6764973ef8c0c215b2e80bec8963' As table_name_matches,
+   geom_colname = 'the_geom' As geom_name_matches
+ FROM cdb_observatory._OBS_GetGeometryMetadata('us.census.tiger.census_tract_clipped') As m(geoid_colname, target_table, geom_colname);
+
 \i test/fixtures/drop_fixtures.sql