cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into builder-api-func

Browse Source
This commit is contained in:
John Krauss
2016-09-29 20:23:08 +00:00
parent 18cfdc60d0 f4113eaea3
commit 48221fc358
13 changed files with 4628 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/PULL_REQUEST_TEMPLATE.md
+17
View File
@@ -0,0 +1,17 @@
## Request for a new Data observatory extension deploy
I'd like to request a new data observatory extension deploy: dump + extension
## Dump database id to be deployed
Please put here the dump id to be deployed: <dump_id>
## Data Observatory extension PRs included.
*Please update the NEWS.md*
Add down here the PR links to be added and deployed:
-
// @CartoDB/dataservices
Makefile
+1 -1
View File
@@ -18,7 +18,7 @@ test: ## Run the tests for the development version of the extension
$(MAKE) -C $(EXT_DIR) test
# Generate a new release into release
release: ## Generate a new release of the extension. Only for telease manager
release: ## Generate a new release of the extension. Only for release manager
$(MAKE) -C $(EXT_DIR) release
# Install the current release.
NEWS.md
+18
View File
@@ -1,3 +1,21 @@
1.0.7 (2016-09-20)
__Bugfixes__
* `NULL` geometries or geometry IDs no longer result in an exception from any
augmentation functions ([#178](https://github.com/CartoDB/observatory-extension/issues/178))
__Improvements__
* Automatic tests work for Canada and Thailand
1.0.6 (2016-09-08)
__Improvements__
* New function structure for Table-level functions which allows to separate the
framework logic from the observatory measure functions.
1.0.5 (2016-08-12)
__Improvements__
release/observatory--1.0.6.sql
+2195
View File
File diff suppressed because one or more lines are too long
release/observatory--1.0.7.sql
+2208
View File
File diff suppressed because one or more lines are too long
release/observatory.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB Observatory backend extension'
default_version = '1.0.5'
default_version = '1.0.7'
requires = 'postgis, postgres_fdw'
superuser = true
schema = cdb_observatory
src/pg/observatory.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB Observatory backend extension'
default_version = '1.0.5'
default_version = '1.0.7'
requires = 'postgis, postgres_fdw'
superuser = true
schema = cdb_observatory
src/pg/sql/41_observatory_augmentation.sql
+14 -1
View File
@@ -179,7 +179,10 @@ BEGIN
--raise notice 'Cannot find data table for boundary ID %, column_ids %, and time_span %', geometry_level, column_ids, time_span;
END IF;
IF ST_GeometryType(geom) = 'ST_Point'
IF geom IS NULL
THEN
results := NULL;
ELSIF ST_GeometryType(geom) = 'ST_Point'
THEN
--raise notice 'geom_table_name %, data_table_info %', geom_table_name, data_table_info::json[];
results := cdb_observatory._OBS_GetPoints(geom,
@@ -361,6 +364,10 @@ DECLARE
sql TEXT;
numer_name TEXT;
BEGIN
IF geom IS NULL THEN
RETURN NULL;
END IF;
geom := ST_SnapToGrid(geom, 0.000001);
EXECUTE
@@ -525,6 +532,9 @@ DECLARE
measure_val NUMERIC;
data_geoid_colname TEXT;
BEGIN
IF geom_ref IS NULL THEN
RETURN NULL;
END IF;
EXECUTE
$query$
@@ -573,6 +583,9 @@ DECLARE
category_val TEXT;
category_share NUMERIC;
BEGIN
IF geom IS NULL THEN
RETURN NULL;
END IF;
EXECUTE
$query$
src/pg/sql/50_table_level_functions.sql → src/pg/sql/50_table_level_framework.sql
+15 -52
View File
@@ -24,9 +24,12 @@ BEGIN
EXCEPTION
WHEN others THEN
-- Disconnect user imported table. Delete schema and FDW server.
EXECUTE 'DROP FOREIGN TABLE IF EXISTS ' || fdw_import_schema || '.' || table_name;
EXECUTE 'DROP SCHEMA IF EXISTS ' || fdw_import_schema || ' CASCADE';
EXECUTE 'DROP SERVER IF EXISTS ' || fdw_server || ' CASCADE;';
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".' || table_name;
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || fdw_import_schema || '".cdb_tablemetadata';
EXECUTE 'DROP SCHEMA IF EXISTS "' || fdw_import_schema || '"';
EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || fdw_server || '"';
EXECUTE 'DROP SERVER IF EXISTS "' || fdw_server || '"';
RETURN (null, null, null);
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
@@ -37,27 +40,9 @@ AS $$
DECLARE
colnames text[];
coltypes text[];
requested_measures text[];
measure text;
BEGIN
-- Simple mock, there should be real logic in here.
IF $3 NOT ILIKE 'GetMeasure' OR $3 IS NULL THEN
RAISE 'This function is not supported yet: %', $3;
END IF;
SELECT translate($4::json->>'tag_name','[]', '{}')::text[] INTO requested_measures;
FOREACH measure IN ARRAY requested_measures
LOOP
IF NOT measure ILIKE ANY (Array['total_pop', 'pop_16_over']::text[]) THEN
RAISE 'This measure is not supported yet: %', measure;
END IF;
SELECT array_append(colnames, measure) INTO colnames;
SELECT array_append(coltypes, 'double precision'::text) INTO coltypes;
END LOOP;
EXECUTE FORMAT('SELECT r.colnames::text[], r.coltypes::text[] FROM cdb_observatory._%sResultMetadata(%L::json) r', function_name, params::text)
INTO colnames, coltypes;
RETURN (colnames::text[], coltypes::text[]);
END;
@@ -68,41 +53,17 @@ RETURNS SETOF record
AS $$
DECLARE
data_query text;
tag_name text[];
tag text;
tags_list text;
tags_query text;
rec RECORD;
BEGIN
SELECT translate($6::json->>'tag_name','[]', '{}')::text[] INTO tag_name;
SELECT array_to_string(tag_name, ',') INTO tags_list;
tags_query := '';
FOREACH tag IN ARRAY tag_name
LOOP
SELECT tags_query || ' sum(' || tag || '/fraction)::double precision as ' || tag || ', ' INTO tags_query;
END LOOP;
-- Simple mock, there should be real logic in here.
data_query := '(WITH _areas AS(SELECT ST_Area(a.the_geom::geography)'
|| '/ (1000 * 1000) as fraction, a.geoid, b.cartodb_id FROM '
|| 'observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308 as a, '
|| table_schema || '.' || table_name || ' AS b '
|| 'WHERE b.the_geom && a.the_geom ), values AS (SELECT geoid, '
|| tags_list
|| ' FROM observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd ) '
|| 'SELECT '
|| tags_query
|| ' cartodb_id::int FROM _areas, values '
|| 'WHERE values.geoid = _areas.geoid GROUP BY cartodb_id);';
EXECUTE FORMAT('SELECT cdb_observatory._%sQuery(%L, %L, %L::json)', function_name, table_schema, table_name, params::text)
INTO data_query;
FOR rec IN EXECUTE data_query
LOOP
RETURN NEXT rec;
END LOOP;
RETURN;
RETURN;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
@@ -112,8 +73,10 @@ RETURNS boolean
AS $$
BEGIN
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".' || table_name;
EXECUTE 'DROP SCHEMA IF EXISTS ' || table_schema || ' CASCADE';
EXECUTE 'DROP SERVER IF EXISTS ' || servername || ' CASCADE;';
EXECUTE 'DROP FOREIGN TABLE IF EXISTS "' || table_schema || '".cdb_tablemetadata';
EXECUTE 'DROP SCHEMA IF EXISTS "' || table_schema || '"';
EXECUTE 'DROP USER MAPPING IF EXISTS FOR public SERVER "' || servername || '"';
EXECUTE 'DROP SERVER IF EXISTS "' || servername || '"';
RETURN true;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
src/pg/sql/51_table_level_functions.sql
+79
View File
@@ -0,0 +1,79 @@
--
--
-- OBS_GetMeasure
--
--
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureResultMetadata(params json)
RETURNS cdb_observatory.ds_return_metadata
AS $$
DECLARE
colnames text[]; -- Array to store the name of the measures to be returned
coltypes text[]; -- Array to store the type of the measures to be returned
requested_measures text[];
measure_id text;
BEGIN
-- By definition, all the measure results for the OBS_GetMeasure API are numeric values
SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO requested_measures;
FOREACH measure_id IN ARRAY requested_measures
LOOP
SELECT array_append(colnames, measure_id) INTO colnames;
SELECT array_append(coltypes, 'numeric'::text) INTO coltypes;
END LOOP;
RETURN (colnames::text[], coltypes::text[]);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetMeasureQuery(table_schema text, table_name text, params json)
RETURNS text
AS $$
DECLARE
data_query text;
measure_ids_arr text[];
measure_id text;
measures_list text;
measures_query text;
normalize text;
boundary_id text;
time_span text;
geom_table_name text;
data_table_name text;
BEGIN
measures_query := '';
-- SELECT table_name from obs_meta WHERE boundary_id = {bound} AND [...] INTO geom_table_name
geom_table_name := 'observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308';
-- SELECT table_name from obs_meta WHERE time_span = {time} AND [...] INTO data_table_name
data_table_name := 'observatory.obs_1a098da56badf5f32e336002b0a81708c40d29cd';
-- Get measure_ids array from JSON
SELECT ARRAY(SELECT json_array_elements_text(params->'measure_id'))::text[] INTO measure_ids_arr;
-- Get a comma-separated list of measures ("total_pop, over_16_pop") to be used in SELECTs
SELECT array_to_string(measure_ids_arr, ',') INTO measures_list;
FOREACH measure_id IN ARRAY measure_ids_arr
LOOP
-- Build query to compute each value and normalize
-- Assumes the default normalization method, the normalize parameter given in the JSON
-- should be checked in order to build the final query
SELECT measures_query || ' sum(' || measure_id || '/fraction)::numeric as ' || measure_id || ', ' INTO measures_query;
END LOOP;
-- Data query should select the measures and the cartodb_id of the user table, in that order.
data_query := '(WITH _areas AS(SELECT ST_Area(a.the_geom::geography)'
|| '/ (1000 * 1000) as fraction, a.geoid, b.cartodb_id FROM '
|| geom_table_name || ' as a, '
|| table_schema || '.' || table_name || ' AS b '
|| 'WHERE b.the_geom && a.the_geom ), values AS (SELECT geoid, '
|| measures_list
|| ' FROM ' || data_table_name || ' ) '
|| 'SELECT '
|| measures_query
|| ' cartodb_id::int FROM _areas, values '
|| 'WHERE values.geoid = _areas.geoid GROUP BY cartodb_id);';
RETURN data_query;
END;
$$ LANGUAGE plpgsql;
src/pg/test/expected/41_observatory_augmentation_test.out
+18
View File
@@ -66,12 +66,18 @@ t
obs_getmeasure_bad_geometry
t
(1 row)
obs_getmeasure_null
t
(1 row)
obs_getcategory_point
t
(1 row)
obs_getcategory_polygon
t
(1 row)
obs_getcategory_null
t
(1 row)
obs_getpopulation
t
(1 row)
@@ -81,6 +87,9 @@ t
obs_getpopulation_polygon_null_test
t
(1 row)
obs_getpopulation_polygon_null_geom_test
t
(1 row)
obs_getuscensusmeasure_point_male_pop
t
(1 row)
@@ -90,12 +99,18 @@ t
obs_getuscensusmeasure_null
t
(1 row)
obs_getuscensusmeasure_null_geom
t
(1 row)
obs_getuscensuscategory_point
t
(1 row)
obs_getuscensuscategory_polygon
t
(1 row)
obs_getuscensuscategory_null
t
(1 row)
obs_getmeasurebyid_cartodb_census_tract
t
(1 row)
@@ -108,3 +123,6 @@ t
obs_getmeasurebyid_nulls
t
(1 row)
obs_getmeasurebyid_null_id
t
(1 row)
src/pg/test/sql/41_observatory_augmentation_test.sql
+33
View File
@@ -203,6 +203,11 @@ SELECT abs(cdb_observatory.OBS_GetMeasure(
cdb_observatory._ProblemTestArea(),
'us.census.acs.B01003001') - 96230.2929825897) / 96230.2929825897 < 0.001 As OBS_GetMeasure_bad_geometry;
-- OBS_GetMeasure with NULL Input
SELECT cdb_observatory.OBS_GetMeasure(
NULL,
'us.census.acs.B01003001') IS NULL As OBS_GetMeasure_null;
-- Point-based OBS_GetCategory
SELECT cdb_observatory.OBS_GetCategory(
cdb_observatory._TestPoint(), 'us.census.spielman_singleton_segments.X10') = 'Wealthy, urban without Kids' As OBS_GetCategory_point;
@@ -211,6 +216,10 @@ SELECT cdb_observatory.OBS_GetCategory(
SELECT cdb_observatory.OBS_GetCategory(
cdb_observatory._TestArea(), 'us.census.spielman_singleton_segments.X10') = 'Wealthy, urban without Kids' As obs_getcategory_polygon;
-- NULL Input OBS_GetCategory
SELECT cdb_observatory.OBS_GetCategory(
NULL, 'us.census.spielman_singleton_segments.X10') IS NULL As obs_getcategory_null;
-- Point-based OBS_GetPopulation, default normalization (area)
SELECT (abs(OBS_GetPopulation - 10923.093200390833950) / 10923.093200390833950) < 0.001 As OBS_GetPopulation FROM
cdb_observatory.OBS_GetPopulation(
@@ -231,6 +240,13 @@ FROM
cdb_observatory._TestArea(), NULL
) As m(obs_getpopulation_polygon_null);
-- Null input OBS_GetPopulation
SELECT obs_getpopulation_polygon_null_geom IS NULL As obs_getpopulation_polygon_null_geom_test
FROM
cdb_observatory.OBS_GetPopulation(
NULL, NULL
) As m(obs_getpopulation_polygon_null_geom);
-- Point-based OBS_GetUSCensusMeasure, default normalization (area)
SELECT (abs(cdb_observatory.obs_getuscensusmeasure(
cdb_observatory._testpoint(), 'male population') - 6789.5647735060920500) / 6789.5647735060920500) < 0.001 As obs_getuscensusmeasure_point_male_pop;
@@ -244,6 +260,11 @@ SELECT (abs(cdb_observatory.obs_getuscensusmeasure(
SELECT (abs(cdb_observatory.obs_getuscensusmeasure(
cdb_observatory._testarea(), 'male population', NULL) - 6043.63061042765) / 6043.63061042765) < 0.001 As obs_getuscensusmeasure_null;
-- Poly-based OBS_GetUSCensusMeasure, Null input geom
SELECT cdb_observatory.obs_getuscensusmeasure(
NULL, 'male population', NULL) IS NULL As obs_getuscensusmeasure_null_geom;
-- Point-based OBS_GetUSCensusCategory
SELECT cdb_observatory.OBS_GetUSCensusCategory(
cdb_observatory._testpoint(), 'Spielman-Singleton Segments: 10 Clusters') = 'Wealthy, urban without Kids' As OBS_GetUSCensusCategory_point;
@@ -252,6 +273,10 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory(
SELECT cdb_observatory.OBS_GetUSCensusCategory(
cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Wealthy, urban without Kids' As OBS_GetUSCensusCategory_polygon;
-- Null-input OBS_GetUSCensusCategory
SELECT cdb_observatory.OBS_GetUSCensusCategory(
NULL, 'Spielman-Singleton Segments: 10 Clusters') IS NULL As OBS_GetUSCensusCategory_null;
-- OBS_GetMeasureById tests
-- typical query
@@ -285,3 +310,11 @@ SELECT cdb_observatory.OBS_GetMeasureById(
'us.census.tiger.block_group',
'2010 - 2014'
) IS NULL As OBS_GetMeasureById_nulls;
-- NULL input id
SELECT cdb_observatory.OBS_GetMeasureById(
NULL,
'us.census.acs.B01003001',
'us.census.tiger.block_group',
'2010 - 2014'
) IS NULL As OBS_GetMeasureById_null_id;
src/python/test/autotest.py
+28 -20
View File
@@ -60,27 +60,30 @@ SKIP_COLUMNS = set([
u'mx.inegi_columns.POB36',
u'mx.inegi_columns.POB33',
u'mx.inegi_columns.POB58',
u'mx.inegi_columns.DISC4',
])
def default_geometry_id(column_id):
'''
Returns default test point for the column_id.
'''
if column_id == 'whosonfirst.wof_disputed_geom':
return 'ST_SetSRID(ST_MakePoint(76.57, 33.78), 4326)'
elif column_id == 'whosonfirst.wof_marinearea_geom':
return 'ST_SetSRID(ST_MakePoint(-68.47, 43.33), 4326)'
elif column_id in ('us.census.tiger.school_district_elementary',
'us.census.tiger.school_district_secondary',
'us.census.tiger.school_district_elementary_clipped',
'us.census.tiger.school_district_secondary_clipped'):
return 'ST_SetSRID(ST_MakePoint(-73.7067, 40.7025), 4326)'
elif column_id.startswith('es.ine'):
return 'ST_SetSRID(ST_MakePoint(-2.51141249535454, 42.8226119029222), 4326)'
elif column_id.startswith('us.zillow'):
return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)'
else:
return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
#def default_geometry_id(column_id):
# '''
# Returns default test point for the column_id.
# '''
# if column_id == 'whosonfirst.wof_disputed_geom':
# return 'ST_SetSRID(ST_MakePoint(76.57, 33.78), 4326)'
# elif column_id == 'whosonfirst.wof_marinearea_geom':
# return 'ST_SetSRID(ST_MakePoint(-68.47, 43.33), 4326)'
# elif column_id in ('us.census.tiger.school_district_elementary',
# 'us.census.tiger.school_district_secondary',
# 'us.census.tiger.school_district_elementary_clipped',
# 'us.census.tiger.school_district_secondary_clipped'):
# return 'ST_SetSRID(ST_MakePoint(-73.7067, 40.7025), 4326)'
# elif column_id.startswith('es.ine'):
# return 'ST_SetSRID(ST_MakePoint(-2.51141249535454, 42.8226119029222), 4326)'
# elif column_id.startswith('us.zillow'):
# return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)'
# elif column_id.startswith('ca.'):
# return ''
# else:
# return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
def default_point(column_id):
@@ -107,6 +110,11 @@ def default_point(column_id):
return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)'
elif column_id.startswith('mx.'):
return 'ST_SetSRID(ST_MakePoint(-99.17019367218018, 19.41347699386547), 4326)'
elif column_id.startswith('ca.'):
raise SkipTest('Skipping Canada until validation of data complete')
return 'ST_SetSRID(ST_MakePoint(-79.39716339111328, 43.65694347778308), 4326)'
elif column_id.startswith('th.'):
return 'ST_SetSRID(ST_MakePoint(100.49263000488281, 13.725377712079784), 4326)'
else:
return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
@@ -116,7 +124,7 @@ def default_area(column_id):
Returns default test area for the column_id
'''
point = default_point(column_id)
area = 'ST_Transform(ST_Buffer(ST_Transform({point}, 3857), 1000), 4326)'.format(
area = 'ST_Transform(ST_Buffer(ST_Transform({point}, 3857), 250), 4326)'.format(
point=point)
return area