Merge pull request #314 from CartoDB/develop

Release 1.8.0

.travis.yml

@@ -0,0 +1,43 @@
+language: c
+dist: precise
+
+env:
+  global:
+    - PAGER=cat
+
+before_install:
+  - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
+  - sudo add-apt-repository -y ppa:cartodb/gis
+  - sudo add-apt-repository -y ppa:cartodb/gis-testing
+  - sudo apt-get update
+
+  # Install postgres db and build deps
+  - sudo /etc/init.d/postgresql stop # stop travis default instance
+  - sudo apt-get -y remove --purge postgresql-9.1
+  - sudo apt-get -y remove --purge postgresql-9.2
+  - sudo apt-get -y remove --purge postgresql-9.3
+  - sudo apt-get -y remove --purge postgresql-9.4
+  - sudo apt-get -y remove --purge postgresql-9.5
+  - sudo rm -rf /var/lib/postgresql/
+  - sudo rm -rf /var/log/postgresql/
+  - sudo rm -rf /etc/postgresql/
+  - sudo apt-get -y remove --purge postgis-2.2
+  - sudo apt-get -y autoremove
+
+  - sudo apt-get -y install postgresql-9.5=9.5.2-3cdb3
+  - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb3
+  - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb3
+  - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
+  - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
+
+  # configure it to accept local connections from postgres
+  - echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" \
+    | sudo tee /etc/postgresql/9.5/main/pg_hba.conf
+  - sudo /etc/init.d/postgresql restart 9.5
+
+install:
+  - sudo make install
+
+script:
+  - cd src/pg
+  - make test || { cat src/pg/test/regression.diffs; false; }

NEWS.md

@@ -1,4 +1,28 @@
+1.8.0 (2017-10-18)
+------------------
+
+__Improvements__
+
+* Add `number_geometries` field to `OBS_GetAvailableGeometries` in order to provide the number of geometries from the source data to be used in the score calculation ([#313](https://github.com/CartoDB/observatory-extension/issues/313))
+
+1.7.0 (2017-08-18)
+------------------
+
+__Improvements__
+
+* Add Travis support to execute the extension tests ([#183](https://github.com/CartoDB/observatory-extension/issues/183))
+
+__API Changes__
+
+* Add new function `OBS_MetadataValidation` ([#303](https://github.com/CartoDB/observatory-extension/pull/303))
+
+__Bugfixes__
+
+* Fixed parentheses for obs_getdata with ids
+* Fixed failing tests due changes in the data dump for some TIGER geometries
+
 1.6.0 (2017-07-20)
+------------------
 
 __Improvements__
 
@@ -17,6 +41,7 @@ __Bugfixes__
 
 
 1.5.1 (2017-05-16)
+------------------
 
 __Improvements__
 
@@ -26,6 +51,7 @@ __Improvements__
   ([#285](https://github.com/CartoDB/observatory-extension/pull/285))
 
 1.5.0 (2017-04-24)
+------------------
 
 __API Changes__
 
@@ -39,6 +65,7 @@ __API Changes__
   ([#282](https://github.com/CartoDB/observatory-extension/pull/282))
 
 1.4.0 (2017-03-21)
+------------------
 
 __API Changes__
 
@@ -59,16 +86,19 @@ __Improvements__
   boundary selection
 
 1.3.5 (2017-03-15)
+------------------
 
 No changes.  Artifact to allow for data update.
 
 1.3.4 (2017-03-10)
+------------------
 
 __Bugfixes__
 
 * Remove erroneously committed `RAISE NOTICE` in `OBS_GetData`
 
 1.3.3 (2017-03-10)
+------------------
 
 __Bugfixes__
 
@@ -91,6 +121,7 @@ __Improvements__
   ([#267](https://github.com/CartoDB/observatory-extension/pull/267))
 
 1.3.2 (2017-03-02)
+------------------
 
 __Bugfixes__
 
@@ -98,6 +129,7 @@ __Bugfixes__
   This fixes issues with Camshaft.
 
 1.3.1 (2017-02-16)
+------------------
 
 __Improvements__
 
@@ -108,6 +140,7 @@ __Improvements__
   called for measures for polygons
 
 1.3.0 (2017-01-17)
+------------------
 
 __API Changes__
 
@@ -132,9 +165,8 @@ __Bugfixes__
 * Remove unnecessary dependency on `postgres_fdw`
 * `OBS_GetData()` now aggregates measures with mixed geoms correctly
 
-__API Changes__
-
 1.2.1 (2017-01-17)
+------------------
 
 __Improvements__
 
@@ -142,6 +174,7 @@ __Improvements__
   ([#243](https://github.com/CartoDB/observatory-extension/pull/233))
 
 1.2.0 (2016-12-28)
+------------------
 
 __API Changes__
 
@@ -162,6 +195,7 @@ __Improvements__
 * Return both `table_id` and `column_id` from `_OBS_GetGeometryScores`
 
 1.1.7 (2016-12-15)
+------------------
 
 __Improvements__
 
@@ -174,6 +208,7 @@ __Improvements__
 * Yields a ~50% improvement in performance for `_OBSGetGeomeryScores`.
 
 1.1.6 (2016-12-08)
+------------------
 
 __Bugfixes__
 
@@ -200,6 +235,7 @@ __Improvements__
   - Add ability to persist results to JSON for graph visualization later
 
 1.1.5 (2016-11-29)
+------------------
 
 __Bugfixes__
 
@@ -207,6 +243,7 @@ __Bugfixes__
   a geometry where it does not exist ([#220](https://github.com/CartoDB/observatory-extension/issues/220)).
 
 1.1.4 (2016-11-21)
+------------------
 
 __Bugfixes__
 
@@ -214,10 +251,12 @@ __Bugfixes__
   `OBS_GetLegacyMetadata` ([#216](https://github.com/CartoDB/observatory-extension/issues/216)).
 
 1.1.3 (2016-11-15)
+------------------
 
 * Temporarily ignore EU data for the sake of testing
 
 1.1.2 (2016-11-09)
+------------------
 
 __Improvements__
 
@@ -233,12 +272,14 @@ __API Changes (Internal)__
 * Add internal `_OBS_GetGeometryScores`
 
 1.1.1 (2016-10-14)
+------------------
 
 __Improvements__
 
 * Test points for Canada and France ([#204](https://github.com/CartoDB/observatory-extension/issues/120))
 
 1.1.0 (2016-10-04)
+------------------
 
 __Bugfixes__
 
@@ -261,6 +302,7 @@ __API Changes__
   is also referred to here ([CartoDB/design#68](https://github.com/CartoDB/design/issues/68)).
 
 1.0.7 (2016-09-20)
+------------------
 
 __Bugfixes__
 
@@ -272,6 +314,7 @@ __Improvements__
 * Automatic tests work for Canada and Thailand
 
 1.0.6 (2016-09-08)
+------------------
 
 __Improvements__
 
@@ -279,6 +322,7 @@ __Improvements__
   framework logic from the observatory measure functions.
 
 1.0.5 (2016-08-12)
+------------------
 
 __Improvements__
 
@@ -286,6 +330,7 @@ __Improvements__
   any HTTP SQL API.
 
 1.0.4 (2016-07-26)
+------------------
 
 __Bugfixes__
 
@@ -294,6 +339,7 @@ __Bugfixes__
   ([#173](https://github.com/CartoDB/observatory-extension/issues/173))
 
 1.0.3 (2016-07-25)
+------------------
 
 __Bugfixes__
 

doc/boundary_functions.md

@@ -2,7 +2,7 @@
 
 Use the following functions to retrieve [Boundary](https://carto.com/docs/carto-engine/data/overview/#boundary-data) data. Data ranges from small areas (e.g. US Census Block Groups) to large areas (e.g. Countries). You can access boundaries by point location lookup, bounding box lookup, direct ID access and several other methods described below.
 
-You can [access](https://carto.com/docs/carto-engine/data/accessing) boundaries through CARTO Builder. The same methods will work if you are using the CARTO Engine to develop your application. We [encourage you](http://docs/carto-engine/data/accessing/#best-practices) to use table modifying methods (UPDATE and INSERT) over dynamic methods (SELECT).
+You can [access](https://carto.com/docs/carto-engine/data/accessing) boundaries through CARTO Builder. The same methods will work if you are using the CARTO Engine to develop your application. We [encourage you](https://carto.com/docs/carto-engine/data/accessing/#best-practices) to use table modifying methods (UPDATE and INSERT) over dynamic methods (SELECT).
 
 ## OBS_GetBoundariesByGeometry(geom geometry, geometry_id text)
 
@@ -123,7 +123,7 @@ SET the_geom = OBS_GetBoundary(the_geom, 'us.census.tiger.block_group')
 
 ## OBS_GetBoundaryId(point_geometry, boundary_id)
 
-The ```OBS_GetBoundaryId(point_geometry, boundary_id)``` returns a unique geometry_id for the boundary geometry that contains a given point geometry. See the [Boundary ID Glossary](http://docs/carto-engine/data/glossary/#boundary-ids). The method can be combined with ```OBS_GetBoundaryById(geometry_id)``` to create a point aggregation workflow.
+The ```OBS_GetBoundaryId(point_geometry, boundary_id)``` returns a unique geometry_id for the boundary geometry that contains a given point geometry. See the [Boundary ID Glossary](https://carto.com/docs/carto-engine/data/glossary/#boundary-ids). The method can be combined with ```OBS_GetBoundaryById(geometry_id)``` to create a point aggregation workflow.
 
 #### Arguments
 

doc/discovery_functions.md

@@ -228,7 +228,7 @@ SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(
 WHERE valid_timespan IS True;
 ```
 
-## OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan)
+## OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan, number_geometries)
 
 Return available geometries within a boundary and with the specified
 `filter_tags`.
@@ -242,6 +242,7 @@ filter_tags | Text[] | a list of filters.  Only geometries for which all of thes
 numer_id | Text | the ID of a numerator to check whether the geometry is valid against.  Will not reduce length of returned table, but will change values for `valid_numer` (optional)
 denom_id | Text | the ID of a denominator to check whether the geometry is valid against.  Will not reduce length of returned table, but will change values for `valid_denom` (optional)
 timespan | Text | the ID of a timespan to check whether the geometry is valid against.  Will not reduce length of returned table, but will change values for `valid_timespan` (optional)
+number_geometries | Integer | Number of geometries of the source data in order to calculate more accurately the score value to know which geometry fits better with the provided extent. (optional)
 
 #### Returns
 

doc/measures_functions.md

@@ -108,7 +108,7 @@ The ```OBS_GetMeasure(polygon, measure_id)``` function returns any Data Observat
 Name |Description
 --- | ---
 polygon_geometry | a WGS84 polygon geometry (the_geom)
-measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf))  
+measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf))
 normalize | for measures that are **sums** (e.g. population) the default normalization is 'none' and response comes back as a raw value. Other options are 'denominator', which will use the denominator specified in the [Data Catalog](https://cartodb.github.io/bigmetadata/index.html) (optional)
 boundary_id | source of geometries to pull measure from (e.g., 'us.census.tiger.census_tract')
 time_span | time span of interest (e.g., 2010 - 2014)
@@ -143,7 +143,7 @@ The ```OBS_GetMeasureById(geom_ref, measure_id, boundary_id)``` function returns
 Name |Description
 --- | ---
 geom_ref | a geometry reference (e.g., a US Census geoid)
-measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf))  
+measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf))
 boundary_id | source of geometries to pull measure from (e.g., 'us.census.tiger.census_tract')
 time_span (optional) | time span of interest (e.g., 2010 - 2014). If `NULL` is passed, the measure from the most recent data will be used.
 
@@ -215,7 +215,7 @@ extent | A geometry of the extent of the input geometries
 metadata | A JSON array composed of metadata input objects.  Each indicates one desired measure for an output column, and optionally additional parameters about that column
 num_timespan_options | How many historical time periods to include.  Defaults to 1
 num_score_options | How many alternative boundary levels to include.  Defaults to 1
-target_geoms | Target number of geometries.  Boundaries with close to this many objects within `extent` will be ranked highest. 
+target_geoms | Target number of geometries.  Boundaries with close to this many objects within `extent` will be ranked highest.
 
 The schema of the metadata input objects are as follows:
 
@@ -321,6 +321,55 @@ SELECT OBS_GetMeta(
 ) FROM tablename
 ```
 
+## OBS_MetadataValidation(extent geometry, geometry_type text, metadata json, target_geoms)
+
+The ```OBS_MetadataValidation``` function performs a validation check over the known issues using the extent, type of geometry, and metadata that is being used in the ```OBS_GetMeta``` function.
+
+#### Arguments
+
+Name | Description
+---- | -----------
+extent | A geometry of the extent of the input geometries
+geometry_type | The geometry type of the source data
+metadata | A JSON array composed of metadata input objects. Each indicates one desired measure for an output column, and optional additional parameters about that column
+target_geoms | Target number of geometries. Boundaries with close to this many objects within `extent` will be ranked highest
+
+The schema of the metadata input objects are as follows:
+
+Metadata Input Key | Description
+--- | -----------
+numer_id | The identifier for the desired measurement. If left blank, a `geom_id` is specified and the column returns a geometry, instead of a measurement
+geom_id | Identifier for a desired geographic boundary level used to calculate measures. If undefined, this is automatically assigned. If defined, `numer_id` is blank and the column returns a geometry, instead of a measurement
+normalization | The desired normalization. One of 'area', 'prenormalized', or 'denominated'.  'Area' will normalize the measure per square kilometer, 'prenormalized' will return the original value, and 'denominated' will normalize by a denominator. If the metadata object specifies a geometry, this is ignored
+denom_id | When `normalization` is 'denominated', this is the identifier for a desired normalization column. This is automatically assigned. If the metadata object specifies a geometry, this is ignored
+numer_timespan | The desired timespan for the measurement. If left unspecified, it defaults to the most recent timespan available
+geom_timespan | The desired timespan for the geometry. If left unspecified, it defaults to the timespan matching `numer_timespan`
+target_area | Instead of aiming to have `target_geoms` in the area of the geometry passed as `extent`, fill this area. Unit is square degrees WGS84. Set this to `0` if you want to use the smallest source geometry for this element of metadata. For example, if you are passing in points
+target_geoms | Override global `target_geoms` for this element of metadata
+max_timespan_rank | Only include timespans of this recency (For example, `1` is only the most recent timespan). There is no limit by default
+max_score_rank | Only include boundaries of this relevance (for example, `1` is the most relevant boundary). The default is `1`
+
+#### Returns
+
+Key | Description
+--- | -----------
+valid | A boolean field that represents if the validation was successful or not
+errors | A text array with all possible errors
+
+#### Examples
+
+Validate metadata with two additional columns of US census data; using a boundary relevant for the geometry provided and the latest timespan. Limited to the most recent column, and the most relevant, based on the extent and density of input geometries in `tablename`.
+
+```SQL
+SELECT OBS_MetadataValidation(
+  ST_SetSRID(ST_Extent(the_geom), 4326),
+  ST_GeometryType(the_geom),
+  '[{"numer_id": "us.census.acs.B01003001"}, {"numer_id": "us.census.acs.B01001002"}]',
+  COUNT(*)::INTEGER
+) FROM tablename
+GROUP BY ST_GeometryType(the_geom)
+```
+
 ## OBS_GetData(geomvals array[geomval], metadata json)
 
 The ```OBS_GetData(geomvals, metadata)``` function returns a measure and/or
@@ -465,7 +514,7 @@ WITH meta AS (
     '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]'
 ) meta FROM tablename)
 SELECT id AS fips, (data->0->>'value')::Numeric AS pop_density
-FROM OBS_GetData((SELECT ARRAY_AGG((fips) FROM tablename),
+FROM OBS_GetData((SELECT ARRAY_AGG(fips) FROM tablename),
                  (SELECT meta FROM meta))
 ```
 
@@ -481,7 +530,7 @@ WITH meta AS (
 ) meta FROM tablename),
 data as (
   SELECT id AS fips, (data->0->>'value') AS pop_density
-  FROM OBS_GetData((SELECT ARRAY_AGG((fips) FROM tablename),
+  FROM OBS_GetData((SELECT ARRAY_AGG(fips) FROM tablename),
                    (SELECT meta FROM meta)))
 UPDATE tablename
 SET pop_density = data.pop_density

release/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.6.0'
+default_version = '1.8.0'
 requires = 'postgis'
 superuser = true
 schema = cdb_observatory

scripts/generate_fixtures.py

@@ -52,8 +52,8 @@ def get_tablename_query(column_id, boundary_id, timespan):
 METADATA_TABLES = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
                    'obs_tag', 'obs_column_to_column', 'obs_dump_version', 'obs_meta',
                    'obs_meta_numer', 'obs_meta_denom', 'obs_meta_geom',
-                   'obs_meta_timespan', 'obs_column_table_tile',
-                   'obs_column_table_tile_simple']
+                   'obs_meta_timespan', 'obs_meta_geom_numer_timespan',
+                   'obs_column_table_tile', 'obs_column_table_tile_simple']
 
 FIXTURES = [
     ('us.census.acs.B01003001_quantile', 'us.census.tiger.census_tract', '2010 - 2014'),

scripts/requirements.txt

@@ -1,3 +1,4 @@
 requests
 nose
 nose_parameterized
+psycopg2

src/pg/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.6.0'
+default_version = '1.8.0'
 requires = 'postgis'
 superuser = true
 schema = cdb_observatory

src/pg/sql/41_observatory_augmentation.sql

@@ -1076,3 +1076,46 @@ BEGIN
   RETURN result;
 END;
 $$ LANGUAGE plpgsql STABLE;
+
+-- MetadataValidation checks the metadata parameters and the geometry type
+-- of the data in order to find possible wrong cases
+CREATE OR REPLACE FUNCTION cdb_observatory.obs_metadatavalidation(
+  geometry_extent geometry(Geometry, 4326),
+  geometry_type text,
+  params JSON,
+  target_geoms INTEGER DEFAULT NULL
+)
+RETURNS TABLE(valid boolean, errors text[]) AS $$
+DECLARE
+  meta json;
+  errors text[];
+BEGIN
+  errors := (ARRAY[])::TEXT[];
+  IF geometry_type IN ('ST_Polygon', 'ST_MultiPolygon') THEN
+      FOR meta IN EXECUTE 'SELECT json_array_elements(cdb_observatory.OBS_GetMeta($1, $2, 1, 1, $3))' USING geometry_extent, params, target_geoms
+      LOOP
+          IF (meta->>'normalization' = 'denominated' AND meta->>'denom_id' is NULL) THEN
+              errors := array_append(errors, 'Normalizated measure should have a numerator and a denominator. Please review the provided options.');
+          END IF;
+          IF (meta->>'numer_aggregate' IS NULL) THEN
+              errors := array_append(errors, 'For polygon geometries, aggregation is mandatory. Please review the provided options');
+          END IF;
+          IF (meta->>'numer_aggregate' IN ('median', 'average') AND meta->>'denom_id' IS NULL) THEN
+              errors := array_append(errors, 'Median or average aggregation for polygons requires a denominator to provide weights. Please review the provided options');
+          END IF;
+          IF (meta->>'numer_aggregate' IN ('median', 'average') AND meta->>'normalization' NOT LIKE 'pre%') THEN
+              errors := array_append(errors, format('Median or average aggregation only supports prenormalized normalization, %s passed. Please review the provided options', meta->>'normalization'));
+          END IF;
+      END LOOP;
+
+      IF CARDINALITY(errors) > 0 THEN
+          RETURN QUERY EXECUTE 'SELECT FALSE, $1' USING errors;
+      ELSE
+          RETURN QUERY SELECT TRUE, ARRAY[]::TEXT[];
+      END IF;
+  ELSE
+    RETURN QUERY SELECT TRUE, ARRAY[]::TEXT[];
+  END IF;
+  RETURN;
+END;
+$$ LANGUAGE plpgsql STABLE;

src/pg/sql/42_observatory_exploration.sql

@@ -323,7 +323,8 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableGeometries(
   filter_tags TEXT[] DEFAULT NULL,
   numer_id TEXT DEFAULT NULL,
   denom_id TEXT DEFAULT NULL,
-  timespan TEXT DEFAULT NULL
+  timespan TEXT DEFAULT NULL,
+  number_geoms INTEGER DEFAULT NULL
 ) RETURNS TABLE (
   geom_id TEXT,
   geom_name TEXT,
@@ -390,15 +391,16 @@ BEGIN
       FROM observatory.obs_meta_geom o
       WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0)
     ), scores AS (
-      SELECT * FROM cdb_observatory._OBS_GetGeometryScores($5,
-        (SELECT ARRAY_AGG(geom_id) FROM available_geoms)
+      SELECT * FROM cdb_observatory._OBS_GetGeometryScores(bounds => $5,
+        filter_geom_ids => (SELECT ARRAY_AGG(geom_id) FROM available_geoms),
+        desired_num_geoms => $6::integer
       )
     ) SELECT DISTINCT ON (geom_id) available_geoms.*, score, numtiles, notnull_percent, numgeoms,
              percentfill, estnumgeoms, meanmediansize
       FROM available_geoms, scores
       WHERE available_geoms.geom_id = scores.column_id
   $string$, geom_clause)
-  USING numer_id, denom_id, timespan, filter_tags, bounds;
+  USING numer_id, denom_id, timespan, filter_tags, bounds, number_geoms;
   RETURN;
 END
 $$ LANGUAGE plpgsql;

src/pg/test/expected/41_observatory_augmentation_test.out

@@ -304,3 +304,12 @@ tract_sample|tract_max_error|tract_avg_error|tract_min_error
 no_bg_point_error
 t
 (1 row)
+valid|errors
+t|{}
+(1 row)
+valid|errors
+f|{"Median or average aggregation only supports prenormalized normalization, denominated passed. Please review the provided options"}
+(1 row)
+valid|errors
+f|{"Normalizated measure should have a numerator and a denominator. Please review the provided options."}
+(1 row)

src/pg/test/fixtures/drop_fixtures.sql

@@ -12,6 +12,7 @@ DROP TABLE IF EXISTS observatory.obs_meta_numer;
 DROP TABLE IF EXISTS observatory.obs_meta_denom;
 DROP TABLE IF EXISTS observatory.obs_meta_geom;
 DROP TABLE IF EXISTS observatory.obs_meta_timespan;
+DROP TABLE IF EXISTS observatory.obs_meta_geom_numer_timespan;
 DROP TABLE IF EXISTS observatory.obs_column_table_tile;
 DROP TABLE IF EXISTS observatory.obs_column_table_tile_simple;
 DROP TABLE IF EXISTS observatory.obs_78fb6c1d6ff6505225175922c2c389ce48d7632c;

src/pg/test/sql/41_observatory_augmentation_test.sql

@@ -973,3 +973,9 @@ WITH _geoms AS (
 FROM geoms, results
 WHERE cartodb_id = id
 ;
+
+-- OBS_MetadataValidation
+
+SELECT * FROM cdb_observatory.OBS_MetadataValidation(NULL, 'ST_Polygon', '[{"numer_id": "us.census.acs.B01003001","denom_id": null,"normalization": "prenormalized","geom_id": null,"numer_timespan": "2010 - 2014"}]'::json, 500);
+SELECT * FROM cdb_observatory.OBS_MetadataValidation(NULL, 'ST_Polygon', '[{"numer_id": "us.census.acs.B25058001","denom_id": null,"normalization": "denominated","geom_id": null,"numer_timespan": "2010 - 2014"}]'::json, 500);
+SELECT * FROM cdb_observatory.OBS_MetadataValidation(NULL, 'ST_Polygon', '[{"numer_id": "us.census.acs.B15003001","denom_id": null,"normalization": "denominated","geom_id": null,"numer_timespan": "2010 - 2014"}]'::json, 500);

src/python/test/autotest.py

@@ -1,5 +1,4 @@
 from nose.tools import assert_equal, assert_is_not_none
-from nose.plugins.skip import SkipTest
 from nose_parameterized import parameterized
 
 from itertools import izip_longest
@@ -55,84 +54,50 @@ SKIP_COLUMNS = set([
     u'us.census.tiger.mtfcc',
     u'whosonfirst.wof_county_name',
     u'whosonfirst.wof_region_name',
-    'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
-    , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
-    , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
-    , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
-    , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
-    , 'fr.insee.P12_ACTOCC15P_ILT45D'
-    , 'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
-    , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
-    , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
-    , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
-    , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
-    , 'fr.insee.P12_ACTOCC15P_ILT45D'
-    , 'uk.ons.LC3202WA0007'
-    , 'uk.ons.LC3202WA0010'
-    , 'uk.ons.LC3202WA0004'
-    , 'uk.ons.LC3204WA0004'
-    , 'uk.ons.LC3204WA0007'
-    , 'uk.ons.LC3204WA0010'
-    , 'br.geo.subdistritos_name'
+    u'fr.insee.P12_RP_CHOS',
+    u'fr.insee.P12_RP_HABFOR',
+    u'fr.insee.P12_RP_EAUCH',
+    u'fr.insee.P12_RP_BDWC',
+    u'fr.insee.P12_RP_MIDUR',
+    u'fr.insee.P12_RP_CLIM',
+    u'fr.insee.P12_RP_MIBOIS',
+    u'fr.insee.P12_RP_CASE',
+    u'fr.insee.P12_RP_TTEGOU',
+    u'fr.insee.P12_RP_ELEC',
+    u'fr.insee.P12_ACTOCC15P_ILT45D',
+    u'fr.insee.P12_RP_CHOS',
+    u'fr.insee.P12_RP_HABFOR',
+    u'fr.insee.P12_RP_EAUCH',
+    u'fr.insee.P12_RP_BDWC',
+    u'fr.insee.P12_RP_MIDUR',
+    u'fr.insee.P12_RP_CLIM',
+    u'fr.insee.P12_RP_MIBOIS',
+    u'fr.insee.P12_RP_CASE',
+    u'fr.insee.P12_RP_TTEGOU',
+    u'fr.insee.P12_RP_ELEC',
+    u'fr.insee.P12_ACTOCC15P_ILT45D',
+    u'uk.ons.LC3202WA0007',
+    u'uk.ons.LC3202WA0010',
+    u'uk.ons.LC3202WA0004',
+    u'uk.ons.LC3204WA0004',
+    u'uk.ons.LC3204WA0007',
+    u'uk.ons.LC3204WA0010',
+    u'br.geo.subdistritos_name'
 ])
 
 MEASURE_COLUMNS = query('''
-SELECT ARRAY_AGG(DISTINCT numer_id) numer_ids,
+SELECT cdb_observatory.FIRST(distinct numer_id) numer_ids,
        numer_aggregate,
-       denom_reltype,
-       section_tags
+       denom_reltype
 FROM observatory.obs_meta
 WHERE numer_weight > 0
   AND numer_id NOT IN ('{skip}')
+  AND numer_id NOT LIKE 'eu.%' --Skipping Eurostat
   AND section_tags IS NOT NULL
   AND subsection_tags IS NOT NULL
-GROUP BY numer_aggregate, section_tags, denom_reltype
+GROUP BY numer_id, numer_aggregate, denom_reltype
 '''.format(skip="', '".join(SKIP_COLUMNS))).fetchall()
 
-#CATEGORY_COLUMNS = query('''
-#SELECT distinct numer_id
-#FROM observatory.obs_meta
-#WHERE numer_type ILIKE 'text'
-#AND numer_weight > 0
-#''').fetchall()
-#
-#BOUNDARY_COLUMNS = query('''
-#SELECT id FROM observatory.obs_column
-#WHERE type ILIKE 'geometry'
-#AND weight > 0
-#''').fetchall()
-#
-#US_CENSUS_MEASURE_COLUMNS = query('''
-#SELECT distinct numer_name
-#FROM observatory.obs_meta
-#WHERE numer_type ILIKE 'numeric'
-#AND 'us.census.acs' = ANY (subsection_tags)
-#AND numer_weight > 0
-#''').fetchall()
-
-
-#def default_geometry_id(column_id):
-#    '''
-#    Returns default test point for the column_id.
-#    '''
-#    if column_id == 'whosonfirst.wof_disputed_geom':
-#        return 'ST_SetSRID(ST_MakePoint(76.57, 33.78), 4326)'
-#    elif column_id == 'whosonfirst.wof_marinearea_geom':
-#        return 'ST_SetSRID(ST_MakePoint(-68.47, 43.33), 4326)'
-#    elif column_id in ('us.census.tiger.school_district_elementary',
-#                       'us.census.tiger.school_district_secondary',
-#                       'us.census.tiger.school_district_elementary_clipped',
-#                       'us.census.tiger.school_district_secondary_clipped'):
-#        return 'ST_SetSRID(ST_MakePoint(-73.7067, 40.7025), 4326)'
-#    elif column_id.startswith('es.ine'):
-#        return 'ST_SetSRID(ST_MakePoint(-2.51141249535454, 42.8226119029222), 4326)'
-#    elif column_id.startswith('us.zillow'):
-#        return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)'
-#    elif column_id.startswith('ca.'):
-#        return ''
-#    else:
-#        return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
-
 
 def default_lonlat(column_id):
     '''
@@ -142,11 +107,6 @@ def default_lonlat(column_id):
         return (76.57, 33.78)
     elif column_id == 'whosonfirst.wof_marinearea_geom':
         return (-68.47, 43.33)
-    elif column_id in ('us.census.tiger.school_district_elementary',
-                       'us.census.tiger.school_district_secondary',
-                       'us.census.tiger.school_district_elementary_clipped',
-                       'us.census.tiger.school_district_secondary_clipped'):
-        return (40.7025, -73.7067)
     elif column_id.startswith('uk'):
         if 'WA' in column_id:
             return (51.46844551219723, -3.184833526611328)
@@ -158,30 +118,19 @@ def default_lonlat(column_id):
         return (28.3305906291771, -81.3544048197256)
     elif column_id.startswith('mx.'):
         return (19.41347699386547, -99.17019367218018)
-    elif column_id.startswith('th.'):
-        return (13.725377712079784, 100.49263000488281)
-    # cols for French Guyana only
-    #elif column_id in ('fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
-    #                   , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
-    #                   , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
-    #                   , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
-    #                   , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
-    #                   , 'fr.insee.P12_ACTOCC15P_ILT45D'
-    #                   , 'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
-    #                   , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
-    #                   , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
-    #                   , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
-    #                   , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
-    #                   , 'fr.insee.P12_ACTOCC15P_ILT45D'):
-    #    return (4.938408371206558, -52.32908248901367)
     elif column_id.startswith('fr.'):
         return (48.860875144709475, 2.3613739013671875)
     elif column_id.startswith('ca.'):
         return (43.65594991256823, -79.37965393066406)
+    elif column_id in ('us.census.tiger.school_district_elementary',
+                       'us.census.tiger.school_district_secondary',
+                       'us.census.tiger.school_district_elementary_clipped',
+                       'us.census.tiger.school_district_secondary_clipped',
+                       'us.census.tiger.school_district_elementary_geoname',
+                       'us.census.tiger.school_district_secondary_geoname'):
+        return (40.7025, -73.7067)
     elif column_id.startswith('us.census.'):
         return (28.3305906291771, -81.3544048197256)
-    elif column_id.startswith('us.dma.'):
-        return (28.3305906291771, -81.3544048197256)
     elif column_id.startswith('us.ihme.'):
         return (28.3305906291771, -81.3544048197256)
     elif column_id.startswith('us.bls.'):
@@ -192,8 +141,6 @@ def default_lonlat(column_id):
         return (28.3305906291771, -81.3544048197256)
     elif column_id.startswith('us.epa.'):
         return (28.3305906291771, -81.3544048197256)
-    elif column_id.startswith('eu.'):
-        raise SkipTest('No tests for Eurostat!')
     elif column_id.startswith('br.'):
         return (-23.53, -46.63)
     elif column_id.startswith('au.'):
@@ -202,56 +149,65 @@ def default_lonlat(column_id):
         raise Exception('No catalog point set for {}'.format(column_id))
 
 
-def default_point(column_id):
-    lat, lng = default_lonlat(column_id)
+def default_point(test_point):
+    lat, lng = test_point
     return 'ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326)'.format(
         lat=lat, lng=lng)
 
 
-def default_area(column_id):
+def default_area(test_point):
     '''
     Returns default test area for the column_id
     '''
-    point = default_point(column_id)
+    point = default_point(test_point)
     area = 'ST_Transform(ST_Buffer(ST_Transform({point}, 3857), 250), 4326)'.format(
         point=point)
     return area
 
-#@parameterized(US_CENSUS_MEASURE_COLUMNS)
-#def test_get_us_census_measure_points(name):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetUSCensusMeasure({point}, '{name}')
-#                 '''.format(name=name.replace("'", "''"),
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            point=default_point('')))
-#    rows = resp.fetchall()
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
+
+def filter_points():
+    return MEASURE_COLUMNS
 
 
-def grouped_measure_columns():
-    for numer_ids, numer_aggregate, denom_reltype, section_tags in MEASURE_COLUMNS:
+def filter_areas():
+    filtered = []
+    for numer_ids, numer_aggregate, denom_reltype in MEASURE_COLUMNS:
+        if numer_aggregate is None or numer_aggregate.lower() not in ('sum', 'median', 'average'):
+            continue
+        if numer_aggregate.lower() in ('median', 'average') \
+                and (denom_reltype is None or denom_reltype.lower() != 'universe'):
+            continue
+        filtered.append((numer_ids, numer_aggregate, denom_reltype))
+
+    return filtered
+
+
+def grouped_measure_columns(filtered_columns):
+    groupbypoint = dict()
+    for row in filtered_columns:
+        numer_ids = row[0]
+        point = default_lonlat(numer_ids)
+        if point in groupbypoint:
+            groupbypoint[point].append(numer_ids)
+        else:
+            groupbypoint[point] = [numer_ids]
+
+    for point, numer_ids in groupbypoint.iteritems():
         for colgroup in grouper(numer_ids, 50):
-            yield [c for c in colgroup if c], numer_aggregate, denom_reltype, section_tags
+            yield point, [c for c in colgroup if c]
 
 
-@parameterized(grouped_measure_columns())
-def test_get_measure_points(numer_ids, numer_aggregate, denom_reltype, section_tags):
-    _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_point(numer_ids[0]))
+@parameterized(grouped_measure_columns(filter_points()))
+def test_get_measure_points(point, numer_ids):
+    _test_measures(numer_ids, default_point(point))
 
 
-@parameterized(grouped_measure_columns())
-def test_get_measure_areas(numer_ids, numer_aggregate, denom_reltype, section_tags):
-    if numer_aggregate is None or numer_aggregate.lower() not in ('sum', 'median', 'average'):
-        return
-    if numer_aggregate.lower() in ('median', 'average') \
-       and (denom_reltype is None \
-            or denom_reltype.lower() != 'universe'):
-        return
-    _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_area(numer_ids[0]))
+@parameterized(grouped_measure_columns(filter_areas()))
+def test_get_measure_areas(point, numer_ids):
+    _test_measures(numer_ids, default_area(point))
 
 
-def _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, geom):
+def _test_measures(numer_ids, geom):
     in_params = []
     for numer_id in numer_ids:
         in_params.append({
@@ -284,90 +240,3 @@ def _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, geom
     assert_equal(len(vals), len(in_params))
     for i, val in enumerate(vals):
         assert_is_not_none(val, 'NULL for {}'.format(in_params[i]['numer_id']))
-
-
-#@parameterized(CATEGORY_COLUMNS)
-#def test_get_category_areas(column_id):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetCategory({area}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            area=default_area(column_id)))
-#    assert_equal(resp.status_code, 200)
-#    rows = resp.json()['rows']
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-
-#@parameterized(CATEGORY_COLUMNS)
-#def test_get_category_points(column_id):
-#    if column_id in SKIP_COLUMNS:
-#        raise SkipTest('Column {} should be skipped'.format(column_id))
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetCategory({point}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            point=default_point(column_id)))
-#    rows = resp.fetchall()
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-
-#@parameterized(BOUNDARY_COLUMNS)
-#def test_get_boundaries_by_geometry(column_id):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetBoundariesByGeometry({area}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            area=default_area(column_id)))
-#    assert_equal(resp.status_code, 200)
-#    rows = resp.json()['rows']
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-
-#@parameterized(BOUNDARY_COLUMNS)
-#def test_get_points_by_geometry(column_id):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetPointsByGeometry({area}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            area=default_area(column_id)))
-#    assert_equal(resp.status_code, 200)
-#    rows = resp.json()['rows']
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-
-#@parameterized(BOUNDARY_COLUMNS)
-#def test_get_boundary_points(column_id):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetBoundary({point}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            point=default_point(column_id)))
-#    assert_equal(resp.status_code, 200)
-#    rows = resp.json()['rows']
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-
-#@parameterized(BOUNDARY_COLUMNS)
-#def test_get_boundary_id(column_id):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetBoundaryId({point}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            point=default_point(column_id)))
-#    assert_equal(resp.status_code, 200)
-#    rows = resp.json()['rows']
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-
-#@parameterized(BOUNDARY_COLUMNS)
-#def test_get_boundary_by_id(column_id):
-#    resp = query('''
-#SELECT * FROM {schema}OBS_GetBoundaryById({geometry_id}, '{column_id}')
-#                 '''.format(column_id=column_id,
-#                            schema='cdb_observatory.' if USE_SCHEMA else '',
-#                            geometry_id=default_geometry_id(column_id)))
-#    assert_equal(resp.status_code, 200)
-#    rows = resp.json()['rows']
-#    assert_equal(1, len(rows))
-#    assert_is_not_none(rows[0][0])
-

src/python/test/perftest.py

@@ -44,33 +44,7 @@ for q in (
                                  -73.81885528564453,41.745696344339564, 4326),
                  'us.census.tiger.county_clipped')) foo
            ORDER BY ST_NPoints(the_geom) DESC
-           LIMIT 50;''',
-        'DROP TABLE IF EXISTS obs_perftest_country_simple',
-        '''CREATE TABLE obs_perftest_country_simple (cartodb_id SERIAL PRIMARY KEY,
-           geom GEOMETRY,
-           name TEXT) ''',
-        '''INSERT INTO obs_perftest_country_simple (geom, name)
-           SELECT the_geom geom,
-                  geom_refs AS name
-           FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
-                 st_makeenvelope(-179,-89, 179,89, 4326),
-                 'whosonfirst.wof_country_geom')) foo
-           ORDER BY ST_NPoints(the_geom) ASC
-           LIMIT 50;''',
-        'DROP TABLE IF EXISTS obs_perftest_country_complex',
-        '''CREATE TABLE obs_perftest_country_complex (cartodb_id SERIAL PRIMARY KEY,
-           geom GEOMETRY,
-           name TEXT) ''',
-        '''INSERT INTO obs_perftest_country_complex (geom, name)
-           SELECT the_geom geom,
-                  geom_refs AS name
-           FROM (SELECT * FROM {schema}OBS_GetBoundariesByGeometry(
-                 st_makeenvelope(-179,-89, 179,89, 4326),
-                 'whosonfirst.wof_country_geom')) foo
-           ORDER BY ST_NPoints(the_geom) DESC
-           LIMIT 50;''',
-        #'''SET statement_timeout = 5000;'''
-):
+           LIMIT 50;'''):
     q_formatted = q.format(
         schema='cdb_observatory.' if USE_SCHEMA else '',
     )
@@ -118,15 +92,7 @@ def record(params, results):
 
     ('complex', '_OBS_GetGeometryScores', 'NULL', 1),
     ('complex', '_OBS_GetGeometryScores', 'NULL', 500),
-    ('complex', '_OBS_GetGeometryScores', 'NULL', 3000),
-
-    ('country_simple', '_OBS_GetGeometryScores', 'NULL', 1),
-    ('country_simple', '_OBS_GetGeometryScores', 'NULL', 500),
-    ('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000),
-
-    ('country_complex', '_OBS_GetGeometryScores', 'NULL', 1),
-    ('country_complex', '_OBS_GetGeometryScores', 'NULL', 500),
-    ('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000),
+    ('complex', '_OBS_GetGeometryScores', 'NULL', 3000)
 ])
 def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms):
     print api_method, geom_complexity, filters, target_geoms