From d1f4e570ad38a6b51b1e98cf003249ad8fa76b85 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 27 Jun 2017 14:37:46 +0200 Subject: [PATCH 1/7] Return NULL for the affected value and not for all the measurements Right now we're doing INNER JOINS when we JOIN the _procgeoms and the data so we end up with NULL value instead of id, NULL value. We need to have the id available to make the JOIN at the end of the query and provide results like this: id | data ----+------------------------------------------------------------------ 1 | [{"value" : 858469},{"value" : 73.9397964478},{"value" : 69092}] 2 | [{"value" : 738774},{"value" : null},{"value" : 2235406}] --- src/pg/sql/41_observatory_augmentation.sql | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 3bcca1c..971cb82 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -760,16 +760,13 @@ BEGIN || ') val_' || colid, ', ') || ' FROM _procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' || - Coalesce(', ' || String_Agg(DISTINCT - Coalesce('observatory.' || numer_tablename, - 'LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' || + Coalesce(String_Agg(DISTINCT + Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname, + ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' || (SELECT STRING_AGG(REPLACE(val::text, '"', ''''), ', ') FROM (SELECT JSON_Array_Elements(api_args) as val) as vals), '') || ')) AS ' || api_method) - , ', '), '') || - Coalesce(' WHERE ' || String_Agg(DISTINCT - '_procgeoms.geomref = ' || numer_tablename || '.' || numer_geomref_colname, ' AND ' - ), '') || + , ' '), '') || CASE $3 WHEN True THEN E'\n GROUP BY _procgeoms.id ORDER BY _procgeoms.id ' ELSE E'\n GROUP BY _procgeoms.id, _procgeoms.geomref ORDER BY _procgeoms.id, _procgeoms.geomref' END From 24a736c72ed325981bd649470dc96305a7d3c1c9 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Jun 2017 13:33:07 +0200 Subject: [PATCH 2/7] Tests for the PR #298 --- .../expected/41_observatory_augmentation_test.out | 3 +++ src/pg/test/sql/41_observatory_augmentation_test.sql | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 5c7bced..a08f8ae 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -210,6 +210,9 @@ t|t|t id|data_polygon_measure_one_null|data_polygon_measure_two_null t|t|t (1 row) +id|data_polygon_measure_one_null|data_polygon_measure_two_null +t|t|t +(1 row) id|data_polygon_measure_one_predenom|data_polygon_measure_two_predenom t|t|t (1 row) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 18ffe99..39ad22e 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -596,6 +596,18 @@ SELECT id = 1 id, abs((data->1->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_two_null FROM data; +-- OBS_GetData/OBS_GetMeta by geom with two measures and one return null +WITH +meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(), + '[{"numer_id": "us.census.acs.B19013001_quantile"}, {"numer_id": "us.census.acs.B01001002"}]') meta), +data AS (SELECT * FROM cdb_observatory.OBS_GetData( + ARRAY[(cdb_observatory._TestArea(), 1)::geomval], + (SELECT meta FROM meta))) +SELECT id = 1 id, + (data->0->>'value') is NULL data_polygon_measure_one_null, + abs((data->1->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_two_null +FROM data; + -- OBS_GetData/OBS_GetMeta by geom with two standard measures predenom normalization WITH meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(), From 8612da57f7b0c975421a98bcfe6882a40f79a16d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Jun 2017 09:39:38 +0200 Subject: [PATCH 3/7] New private function _OBS_GetNumerators to be used in our UI The current OBS_GetAvailableNumerators is not designed with our UI in mind so it's causing a lot of troubles and we're doing so many hacks to fit our UI needs and the interface of the function so this function it's a better fit for our purposes. This function is private because, by now, we don't want to expose as a public function because could suffer changes in the near future. --- src/pg/sql/42_observatory_exploration.sql | 80 +++++++++++ .../42_observatory_exploration_test.out | 57 ++++++++ .../sql/42_observatory_exploration_test.sql | 136 ++++++++++++++++++ 3 files changed, 273 insertions(+) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index 4920167..a9d6c8d 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -181,6 +181,86 @@ BEGIN END $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetNumerators( + bounds GEOMETRY DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '' +) RETURNS TABLE ( + numer_id TEXT, + numer_name TEXT, + numer_description TEXT, + numer_weight NUMERIC, + numer_license TEXT, + numer_source TEXT, + numer_type TEXT, + numer_aggregate TEXT, + numer_extra JSONB, + numer_tags JSONB, + valid_denom BOOLEAN, + valid_geom BOOLEAN, + valid_timespan BOOLEAN +) AS $$ +DECLARE + where_clause_elements TEXT[]; + geom_clause TEXT; + where_clause TEXT; +BEGIN + where_clause_elements := (ARRAY[])::TEXT[]; + where_clause := ''; + + IF bounds IS NOT NULL THEN + where_clause_elements := array_append(where_clause_elements, format($data$ST_Intersects(the_geom, '%s'::geometry)$data$, bounds)); + END IF; + IF cardinality(section_tags) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, section_tags)); + END IF; + IF cardinality(subsection_tags) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, subsection_tags)); + END IF; + IF cardinality(other_tags) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, other_tags)); + END IF; + IF cardinality(ids) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_id IN (array_to_string('%s'::text[], ','))$data$, ids)); + END IF; + IF name IS NOT NULL AND name != '' THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_name ilike '%%%s%%'$data$, name)); + END IF; + IF cardinality(where_clause_elements) > 0 THEN + where_clause := format($clause$WHERE %s$clause$, array_to_string(where_clause_elements, ' AND ')); + END IF; + RAISE DEBUG '%', array_to_string(where_clause_elements, ' AND '); + + RETURN QUERY + EXECUTE + format($string$ + SELECT numer_id::TEXT, + numer_name::TEXT, + numer_description::TEXT, + numer_weight::NUMERIC, + NULL::TEXT license, + NULL::TEXT source, + numer_type numer_type, + numer_aggregate numer_aggregate, + numer_extra::JSONB numer_extra, + numer_tags numer_tags, + $1 = ANY(denoms) valid_denom, + $2 = ANY(geoms) valid_geom, + $3 = ANY(timespans) valid_timespan + FROM observatory.obs_meta_numer + %s + $string$, where_clause) + USING denom_id, geom_id, timespan; + RETURN; +END +$$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableDenominators( bounds GEOMETRY DEFAULT NULL, filter_tags TEXT[] DEFAULT NULL, diff --git a/src/pg/test/expected/42_observatory_exploration_test.out b/src/pg/test/expected/42_observatory_exploration_test.out index 31bdf95..7e9cb54 100644 --- a/src/pg/test/expected/42_observatory_exploration_test.out +++ b/src/pg/test/expected/42_observatory_exploration_test.out @@ -48,6 +48,63 @@ t _obs_getavailablenumerators_no_total_pop_1996 t (1 row) +_obs_getnumerators_usa_pop_in_all +t +(1 row) +_obs_getnumerators_usa_pop_in_nyc_point +t +(1 row) +_obs_getnumerators_usa_pop_in_usa_extents +t +(1 row) +_obs_getnumerators_no_usa_pop_not_in_zero_point +t +(1 row) +_obs_getnumerators_usa_pop_in_age_gender_subsection +t +(1 row) +_obs_getnumerators_no_pop_in_income_subsection +t +(1 row) +_obs_getnumerators_male_pop_denom_by_total_pop +t +(1 row) +_obs_getnumerators_no_income_denom_by_total_pop +t +(1 row) +_obs_getnumerators_zillow_at_zcta5 +t +(1 row) +_obs_getnumerators_no_zillow_at_block_group +t +(1 row) +_obs_getnumerators_total_pop_2010_2014 +t +(1 row) +_obs_getnumerators_no_total_pop_1996 +t +(1 row) +_obs_getnumerators_total_pop_by_name +t +(1 row) +_obs_getnumerators_total_pop_by_section +t +(1 row) +_obs_getnumerators_total_pop_not_in_canada +t +(1 row) +_obs_getnumerators_total_pop_by_subsection +t +(1 row) +_obs_getnumerators_total_pop_not_in_employment_subsection +t +(1 row) +_obs_getnumerators_total_pop_by_id +t +(1 row) +_obs_getnumerators_total_pop_not_with_other_id +t +(1 row) _obs_getavailabledenominators_usa_pop_in_all t (1 row) diff --git a/src/pg/test/sql/42_observatory_exploration_test.sql b/src/pg/test/sql/42_observatory_exploration_test.sql index 1f08e00..0bb2ab6 100644 --- a/src/pg/test/sql/42_observatory_exploration_test.sql +++ b/src/pg/test/sql/42_observatory_exploration_test.sql @@ -119,6 +119,142 @@ FROM cdb_observatory.OBS_GetAvailableNumerators( ) WHERE valid_timespan = True) AS _obs_getavailablenumerators_no_total_pop_1996; +-- +-- _OBS_GetNumerators tests +-- + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators()) +AS _obs_getnumerators_usa_pop_in_all; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +)) AS _obs_getnumerators_usa_pop_in_nyc_point; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakeEnvelope( + -169.8046875, 21.289374355860424, + -47.4609375, 72.0739114882038 + ), 4326), + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +)) AS _obs_getnumerators_usa_pop_in_usa_extents; + +SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(0, 0), 4326), + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL +)) AS _obs_getnumerators_no_usa_pop_not_in_zero_point; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + subsection_tags => ARRAY['subsection/tags.age_gender'] +)) +AS _obs_getnumerators_usa_pop_in_age_gender_subsection; + +SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + subsection_tags => ARRAY['subsection/tags.income'] +)) +AS _obs_getnumerators_no_pop_in_income_subsection; + +SELECT 'us.census.acs.B01001002' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + denom_id => 'us.census.acs.B01003001' +) WHERE valid_denom = True) +AS _obs_getnumerators_male_pop_denom_by_total_pop; + +SELECT 'us.census.acs.B19013001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + denom_id => 'us.census.acs.B01003001' +) WHERE valid_denom = True) +AS _obs_getnumerators_no_income_denom_by_total_pop; + +SELECT 'us.zillow.AllHomes_Zhvi' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + geom_id => 'us.census.tiger.zcta5' +) WHERE valid_geom = True) +AS _obs_getnumerators_zillow_at_zcta5; + +SELECT 'us.zillow.AllHomes_Zhvi' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + geom_id => 'us.census.tiger.block_group' +) WHERE valid_geom = True) +AS _obs_getnumerators_no_zillow_at_block_group; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + timespan => '2010 - 2014' +) WHERE valid_timespan = True) +AS _obs_getnumerators_total_pop_2010_2014; + +SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + timespan => '1996' +) WHERE valid_timespan = True) +AS _obs_getnumerators_no_total_pop_1996; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + name => 'tot' +)) +AS _obs_getnumerators_total_pop_by_name; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + section_tags => '{section/tags.united_states}' +)) +AS _obs_getnumerators_total_pop_by_section; + +SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + section_tags => '{section/tags.ca}' +)) +AS _obs_getnumerators_total_pop_not_in_canada; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + section_tags => '{section/tags.united_states}', + subsection_tags => '{subsection/tags.age_gender}' +)) +AS _obs_getnumerators_total_pop_by_subsection; + +SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + section_tags => '{section/tags.united_states}', + subsection_tags => '{subsection/tags.employment}' +)) +AS _obs_getnumerators_total_pop_not_in_employment_subsection; + +SELECT 'us.census.acs.B01003001' IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + ids => '{us.census.acs.B01003001}' +)) +AS _obs_getnumerators_total_pop_by_id; + +SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id +FROM cdb_observatory._OBS_GetNumerators( + ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326), + ids => '{us.census.acs.B01003002}' +)) +AS _obs_getnumerators_total_pop_not_with_other_id; + -- -- OBS_GetAvailableDenominators tests -- From 8b98b6b64aa31c890b3b3696f54ddee05e2a56e1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 29 Jun 2017 17:54:11 +0200 Subject: [PATCH 4/7] Bump version 1.6.0 --- NEWS.md | 3 +++ src/pg/observatory.control | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 74f251d..81ffa53 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +1.6.0 (2017-mm-dd) + + 1.5.1 (2017-05-16) __Improvements__ diff --git a/src/pg/observatory.control b/src/pg/observatory.control index dc9c991..ef9b995 100644 --- a/src/pg/observatory.control +++ b/src/pg/observatory.control @@ -1,5 +1,5 @@ comment = 'CartoDB Observatory backend extension' -default_version = '1.5.1' +default_version = '1.6.0' requires = 'postgis' superuser = true schema = cdb_observatory From adae37631e3641dc32a6b2f117350b9ddf890019 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 10 Jul 2017 13:22:11 +0200 Subject: [PATCH 5/7] Modified OBS_GetAvailableGeometries Now use the new meta ttable obs_meta_geom_numer_timspan to filter the geometries by geometries timepsan and/or numerator timespan (which is what we get when we use the obs_getavailabletimepspans) --- src/pg/sql/42_observatory_exploration.sql | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index a9d6c8d..bbd5e56 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -372,16 +372,28 @@ BEGIN geom_type::TEXT, geom_extra::JSONB, geom_tags::JSONB, - $1 = ANY(numers) valid_numer, - $2 = ANY(denoms) valid_denom, - $3 = ANY(timespans) valid_timespan - FROM observatory.obs_meta_geom + $1 = ANY(numers) valid_numer, + $2 = ANY(denoms) valid_denom, + CASE WHEN $3 IS NOT NULL AND $3 != '' THEN + -- Here we are looking for geometries with: a) geometry timespan or b) numerators linked to that geometries that fit in the + -- timespan passed. For example it look for geometries with timespan '2015 - 2015' or numerators linked to that geometry that has + -- '2015 - 2015' as one of the valid timespans. + -- If we pass a numerator_id, we filter by that numerator + CASE WHEN $1 IS NOT NULL AND $1 != '' THEN + EXISTS (SELECT 1 FROM observatory.obs_meta_geom_numer_timespan onu WHERE o.geom_id = onu.geom_id AND onu.numer_id = $1 AND ($3 = ANY(onu.timespans) OR $3 IN (select(unnest(o.timespans))))) + ELSE + EXISTS (SELECT 1 FROM observatory.obs_meta_geom_numer_timespan onu WHERE o.geom_id = onu.geom_id AND ($3 = ANY(onu.timespans) OR $3 IN (select(unnest(o.timespans))))) + END + ELSE + false + END as valid_timespan + FROM observatory.obs_meta_geom o WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0) ), scores AS ( SELECT * FROM cdb_observatory._OBS_GetGeometryScores($5, (SELECT ARRAY_AGG(geom_id) FROM available_geoms) ) - ) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms, + ) SELECT DISTINCT ON (geom_id) available_geoms.*, score, numtiles, notnull_percent, numgeoms, percentfill, estnumgeoms, meanmediansize FROM available_geoms, scores WHERE available_geoms.geom_id = scores.column_id From 8d52857f01a97adf2aa413935be6d02508ea5424 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 20 Jul 2017 10:50:32 +0200 Subject: [PATCH 6/7] Version 1.6.0 release artifact --- release/observatory--1.6.0.sql | 2400 ++++++++++++++++++++++++++++++++ release/observatory.control | 2 +- 2 files changed, 2401 insertions(+), 1 deletion(-) create mode 100644 release/observatory--1.6.0.sql diff --git a/release/observatory--1.6.0.sql b/release/observatory--1.6.0.sql new file mode 100644 index 0000000..6a95da0 --- /dev/null +++ b/release/observatory--1.6.0.sql @@ -0,0 +1,2400 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION observatory" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_observatory_version() +RETURNS text AS $$ + SELECT '1.6.0'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_observatory_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='observatory' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; + +-- Returns the table name with geoms for the given geometry_id +-- TODO probably needs to take in the column_id array to get the relevant +-- table where there is multiple sources for a column from multiple +-- geometries. +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GeomTable( + geom geometry(Geometry, 4326), + geometry_id text, + time_span text DEFAULT NULL +) + RETURNS TEXT +AS $$ +DECLARE + result text; +BEGIN + EXECUTE ' + SELECT tablename FROM observatory.OBS_table + WHERE id IN ( + SELECT table_id + FROM observatory.OBS_table tab, + observatory.OBS_column_table coltable, + observatory.OBS_column col + WHERE type ILIKE ''geometry'' + AND coltable.column_id = col.id + AND coltable.table_id = tab.id + AND col.id = $1 + AND CASE WHEN $3::TEXT IS NOT NULL THEN timespan ILIKE $3::TEXT ELSE TRUE END + ORDER BY timespan DESC LIMIT 1 + ) + ' + USING geometry_id, geom, time_span + INTO result; + + return result; + +END; +$$ LANGUAGE plpgsql; + + + +-- A function that gets the column data for multiple columns +-- Old: OBS_GetColumnData +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetColumnData( + geometry_id text, + column_ids text[], + timespan text +) +RETURNS SETOF JSON +AS $$ +BEGIN + + -- figure out highest-weight geometry_id/timespan pair for the first data column + -- TODO this should be done for each data column separately + IF geometry_id IS NULL OR timespan IS NULL THEN + EXECUTE ' + SELECT data_t.timespan timespan, geom_c.id boundary_id + FROM observatory.obs_table data_t, + observatory.obs_column_table data_ct, + observatory.obs_column data_c, + observatory.obs_column_table geoid_ct, + observatory.obs_column_to_column c2c, + observatory.obs_column geom_c + WHERE data_c.id = $2 + AND data_ct.column_id = data_c.id + AND data_ct.table_id = data_t.id + AND geoid_ct.table_id = data_t.id + AND geoid_ct.column_id = c2c.source_id + AND c2c.reltype = ''geom_ref'' + AND geom_c.id = c2c.target_id + AND CASE WHEN $3 IS NULL THEN True ELSE $3 = timespan END + AND CASE WHEN $1 IS NULL THEN True ELSE $1 = geom_c.id END + ORDER BY geom_c.weight DESC, + data_t.timespan DESC + LIMIT 1 + ' INTO timespan, geometry_id + USING geometry_id, (column_ids)[1], timespan; + END IF; + + RETURN QUERY + EXECUTE ' + WITH geomref AS ( + SELECT ct.table_id id + FROM observatory.OBS_column_to_column c2c, + observatory.OBS_column_table ct + WHERE c2c.reltype = ''geom_ref'' + AND c2c.target_id = $1 + AND c2c.source_id = ct.column_id + ), + column_ids as ( + select row_number() over () as no, a.column_id as column_id from (select unnest($2) as column_id) a + ) + SELECT row_to_json(a) from ( + select colname, + tablename, + aggregate, + name, + type, + c.description, + $1 AS boundary_id + FROM column_ids, observatory.OBS_column c, observatory.OBS_column_table ct, observatory.OBS_table t + WHERE column_ids.column_id = c.id + AND c.id = ct.column_id + AND t.id = ct.table_id + AND t.timespan = $3 + AND t.id in (SELECT id FROM geomref) + order by column_ids.no + ) a + ' + USING geometry_id, column_ids, timespan + RETURN; + +END; +$$ LANGUAGE plpgsql; + +--Test point cause Stuart always seems to make random points in the water +CREATE OR REPLACE FUNCTION cdb_observatory._TestPoint() + RETURNS geometry(Point, 4326) +AS $$ +BEGIN + -- new york city + RETURN ST_SetSRID(ST_Point( -73.936669, 40.704512), 4326); +END; +$$ LANGUAGE plpgsql; + +--Test polygon cause Stuart always seems to make random points in the water +-- TODO: remove as it's not used anywhere? +CREATE OR REPLACE FUNCTION cdb_observatory._TestArea() + RETURNS geometry(Geometry, 4326) +AS $$ +BEGIN + -- Buffer NYC point by 500 meters + RETURN ST_Buffer(cdb_observatory._TestPoint()::geography, 500)::geometry; + +END; +$$ LANGUAGE plpgsql; + +--Problematic test area that tends to cause errors +CREATE OR REPLACE FUNCTION cdb_observatory._ProblemTestArea() + RETURNS geometry(Geometry, 4326) +AS $$ +BEGIN + RETURN ST_Translate('0106000020E610000001000000010300000004000000A3030000A09400CAD92D5AC088FA8054CBD04340C74F1462DA2D5AC080F9946B63D043400CBFCDA8DA2D5AC048A9756963D04340BCA4A5B0DA2D5AC010733FD05FD0434054923667DA2D5AC0986276D25FD043406038BC25DB2D5AC0600F50FBDCCF43400453A577DB2D5AC0E834A8F6DCCF4340AC97C1A6DB2D5AC0A06DFAF5BBCF43401C80158CDC2D5AC0D8139CE81DCF43407008A7FBDC2D5AC058865A08EFCE4340DCD519AFDE2D5AC0982B0911EFCE43400C8832AFDE2D5AC070AED4FFEECE43406471BEBFDE2D5AC0B7B6E3FFEECE4340C01E93F2DE2D5AC0077C7A16CBCE4340B4EA55F2DE2D5AC0E0147D15CBCE4340385DF195DF2D5AC0585D07A158CE4340649353D3DF2D5AC010AF6FBD2CCE4340A4981805E02D5AC0889B302709CE434077676710E02D5AC0D8ECC62A01CE4340E7F37D10E02D5AC04047191B01CE4340A0F8F5CCDD2D5AC0B89BFF1501CE43404475C582DE2D5AC0B075C01A8CCD4340A8F865B4DE2D5AC090BC3DEC6BCD4340D89E52EBDE2D5AC038A7AD0C48CD4340971F1E40E12D5AC048E3BA1048CD43404463BE00FE2D5AC0D04FFE7C48CD4340DC6B6347FE2D5AC0C0DB48881BCD4340203BDB48FE2D5AC0C815D7981ACD43404CF94B49FE2D5AC0185A9C521ACD43401421F24CFE2D5AC0A0D6345C15CD43402F82F7CFFF2D5AC028761C9514CD43409FD10571152E5AC028DC1FED14CD43404C31B28A152E5AC050929DDA00CD4340444C4399152E5AC0F0E2E5D200CD4340E87E1A99152E5AC0B00DD8EC00CD4340F8D55ACD372E5AC070C9CECEEECC4340185D1ECB382E5AC0C0309E50EECC43406CDA9A473A2E5AC07FCC3DADEDCC43400053E4483B2E5AC0D003AE51EDCC434094CDD14D3C2E5AC018E78B04EDCC4340A0CEA9523D2E5AC0D0FCC1C5ECCC43402F1D2A5B3E2E5AC0988E0594ECCC434050DF93633F2E5AC018C4A770ECCC434050CC31EE402E5AC0488FAC57ECCC4340046D20FA412E5AC080815A58ECCC4340CC0A7184432E5AC0D8609D74ECCC43402C1E4F12452E5AC0E8E297B0ECCC4340C0FB8A98462E5AC0E085250CEDCC434058D39A1E482E5AC080FBCA88EDCC4340849AA8D84A2E5AC03849F9A4EECC43401CB8D1CD4C2E5AC078B0FB84EFCC434090E02BBF4E2E5AC02055DC71F0CC43404472BCAC502E5AC00052176DF1CC4340489D389A522E5AC0E09C4E75F2CC4340AC1EEA83542E5AC0E84A668AF3CC43405009D269562E5AC0B09CDCADF4CC43409024ED4B582E5AC098F3C1DCF5CC434054833C2A5A2E5AC0B8A2011AF7CC4340345EC3045C2E5AC010EEB862F8CC434067D235DF5D2E5AC01874D7B9F9CC4340E76C28B25F2E5AC090425A1CFBCC434004384E81612E5AC0B856CA8BFCCC4340447FAB4C632E5AC048A81808FECC434040E43A14652E5AC058655691FFCC43408C6F4AD4662E5AC08890FA2501CD4340A8DF4994682E5AC0F88527C602CD43405350C74C6A2E5AC0D0212A7304CD43405B177A016C2E5AC0977FA62B06CD4340247384F06E2E5AC06885D04B09CD4340D03C6AF36F2E5AC0902F206F0ACD4340A45FD598712E5AC0D891646F0CCD4340708FA911732E5AC0B8C3F4640ECD434044DF9286742E5AC080A29A7B10CD4340F45CE2D2752E5AC0F75A958612CD4340D487612C782E5AC0FF8FFDAD16CD4340ECF38E3C782E5AC0B8D443CC16CD434078184049782E5AC070F20BE416CD43401C81A16F742E5AC0B0A384AF1BCD4340EF78AFC7752E5AC01057146A1ECD434090475AD1762E5AC058D8C8BF20CD4340803AE7D2782E5AC070CE07BF25CD434030725C147D2E5AC040A74A7530CD4340D3D9C772812E5AC0D8B06C743BCD4340F8ABBF0C832E5AC0883E577C3FCD43400456A793842E5AC0F830465443CD434048E65526872E5AC080902ECE49CD43406CEF40BB882E5AC0D08441944DCD4340F82791DA8A2E5AC018AA351652CD4340FCA1D5D38B2E5AC0500DF6F153CD4340080BA67B8C2E5AC05091453255CD43407CE0BCFB8D2E5AC07093A3D057CD4340204B852F8E2E5AC0D06A022B58CD4340AB56181B8D2E5AC078A969685ACD4340AF292487922E5AC0C05E3D6862CD434088A3DCF7942E5AC0F036546B65CD434058B1590A952E5AC09055998265CD434048856440972E5AC0B86AB6E367CD4340E4948B549F2E5AC0A771539670CD4340A491D52E9F2E5AC06868F4F170CD4340AB776367A82E5AC0FF703DC87ACD4340E402822CB02E5AC027239D0A83CD4340440C2339B02E5AC0F8670D1883CD4340109761DEAE2E5AC0804BD43386CD4340107679BCAE2E5AC0102E387E86CD43403B3663ADAE2E5AC0A04C53A686CD43402090A065AE2E5AC02849636387CD434030B50E3CAE2E5AC0D0C3F7D587CD4340E4128012AE2E5AC0C0469B4888CD4340C88DA5ECAD2E5AC0381543BB88CD4340C8F5C9C6AD2E5AC03054C93089CD4340C73F7DA8AD2E5AC0F8AC239589CD4340EB4AEDA0AD2E5AC020934FA689CD434064F7189DAD2E5AC0903D6DBA89CD4340DFF5C77EAD2E5AC06096C71E8ACD434053559E5CAD2E5AC060BF41978ACD4340ACB9A449AD2E5AC0D01D33D98ACD4340A8C7753AAD2E5AC0972BB10F8BCD434090B5061CAD2E5AC0203738888BCD4340F45793FDAC2E5AC0E04197038CCD4340B81C43EEAC2E5AC0204D1E518CCD4340D088DAE2AC2E5AC06809017F8CCD4340E8931FC8AC2E5AC0C0247EFA8CCD43400CAF04B5AC2E5AC070625D568DCD4340C02D5EADAC2E5AC047A8CA788DCD43408C02AEA5AC2E5AC060B8C0A68DCD43404CC75D96AC2E5AC0A0C347F48DCD43407802587FAC2E5AC0F79AA7728ECD434084505368AC2E5AC0C72603F18ECD434000960055AC2E5AC028514E728FCD43400027B241AC2E5AC0000BBBF08FCD4340F0FA1932AC2E5AC0205B087290CD4340E0CE8122AC2E5AC008FF68F390CD4340A4F8A016AC2E5AC0B8C8CB7491CD434034EDC60EAC2E5AC008996CC291CD434098CC0807AC2E5AC060FB25F691CD434010730803AC2E5AC04040EC2992CD434074BDE5FEAB2E5AC0D081937792CD43400843E8F6AB2E5AC0D86D58D992CD4340D7110CEFAB2E5AC0E863FB2693CD4340609FD90BAB2E5AC00084664993CD4340C0F5C933AA2E5AC0F8BCFE6B93CD43405CA7CF43AA2E5AC028F2CC5493CD43405CAC16A8AA2E5AC0907AADC392CD4340EF233639AB2E5AC09FF2C2EC91CD43401842537EAB2E5AC088E2A08791CD4340A3C89280AB2E5AC098E0267391CD4340048BF7D4AB2E5AC0801FC6F28ECD4340B0CD6EDCAB2E5AC06F4867C38ECD4340985AD9E5AB2E5AC0E02BC58B8ECD43405478A1FDAB2E5AC0F0AD100F8ECD43406F17D027AC2E5AC030D62F2C8DCD43400CF4064EAC2E5AC0D81654778CCD4340E7C1AE6AAC2E5AC0582FFFEE8BCD43409C858D7DAC2E5AC0482B609D8BCD434044040DAAAB2E5AC0E037666285CD43402C6320BBAA2E5AC0B827C7587ECD43404C5CC802AA2E5AC0808BA0947DCD4340A4B6AA0BA92E5AC0C0EEA48D7CCD4340AC3D6997A52E5AC0CFA8A2E078CD434064F5840EA22E5AC0F002B51D75CD43405CDA4BDB9E2E5AC0C0EBEDB571CD434034B3E3DA9E2E5AC0306680B571CD4340D8390E689B2E5AC01F84207779CD4340E84871659B2E5AC0B00C047D79CD4340F4BCADB7972E5AC068BEBC007BCD4340545700B1972E5AC0501E7D037BCD43402FC174B0972E5AC028F1E8027BCD43407770A457962E5AC007C69A9479CD43406C6B1943952E5AC0F8E5BE027CCD4340841EAF42952E5AC0A06AAF037CCD43407035622E952E5AC0704F0A0C7CCD434048F38129922E5AC030833A4A7DCD43407C9B18D48F2E5AC0D02DB23F7ECD43408F2D5CCD8F2E5AC0781C6C427ECD434098C2A0058E2E5AC0A0DFE5F97ECD4340E027B9E98D2E5AC0A85A21057FCD4340CC0A5AE88D2E5AC008CBAA057FCD4340D40765898A2E5AC04823626180CD4340EC5E5F32882E5AC0D0FAC35281CD43400CC20E29872E5AC0E8EBA2BD81CD4340BF49F7F9832E5AC0C00D0F0683CD4340208FB9C8832E5AC040DAE51983CD434054496368802E5AC09831207684CD4340EC6112087D2E5AC0B8AE5CD285CD434024B9F0A7792E5AC088238A2E87CD4340A367C0A7792E5AC0109D9F2E87CD4340CC1ED6EB7B2E5AC0D07E55AF94CD43407C8769DF7C2E5AC0AFEFA95A9ACD4340FC12B3BC7D2E5AC010E444819FCD434054033DCF7D2E5AC020A9B6EF9FCD43409CEA25007E2E5AC0E0FA2C13A1CD4340046CD0C27E2E5AC0001A0B9BA5CD434068BCF25B7F2E5AC070F6962BA9CD434054765EB67F2E5AC0E0216846ABCD4340078E523C802E5AC098339564AECD434027F231A6802E5AC0100177DBB0CD4340581FF1B9802E5AC058DF1451B1CD4340F847C499812E5AC0304CC986B6CD4340039055D5812E5AC0E897AAE9B7CD4340EF9D27EA812E5AC0A0BAB965B8CD4340C06DB3FF812E5AC0E0FE1616BCCD4340A086E0FF812E5AC06095A11DBCCD4340A0CBCFFE812E5AC0583E48DFBCCD4340D40940ED812E5AC080471A6AC1CD4340AC2F6CDC812E5AC00095F5CBC2CD4340B3ADF2D9812E5AC08FBF9307C3CD4340A872B7D7812E5AC078688C34C3CD434094C24093812E5AC010173C96C8CD4340C07E7774812E5AC078A4C801CBCD43401C0B384E812E5AC0206A7203CECD43402C1C143C812E5AC038AC9E70CFCD4340EC6DC209812E5AC080D52C65D3CD434024BE4BF0802E5AC0D8FC8A65D5CD43408FA3B8C4802E5AC010DD5ED2D8CD4340EFB749B4802E5AC0E0E93B1DDACD43408464CAB2802E5AC0783B573BDACD4340B4F2DBD5802E5AC0CFF64E5DDDCD4340A83945E9802E5AC01FA27319DFCD4340F062D226812E5AC0A04DAE99E4CD4340BCC0C436812E5AC020DC5B06E6CD4340C0EBF54C812E5AC060EAE201E8CD43403030BFC1812E5AC038F76DE1E9CD43408064EA76822E5AC0885F4EC9ECCD43403C9ED6CD822E5AC0E881392EEECD43400F03F919842E5AC0C72FEB81F3CD4340D0BC7A7A842E5AC0380D3D0EF5CD4340D45BF0C0852E5AC0A0885E16F8CD4340A4E275E6862E5AC0C01B2FD0FACD4340BF29C9F3862E5AC07827DEEFFACD4340A420FDFE862E5AC0D09DFEF8FACD43401803E00F872E5AC0577DC006FBCD43403FB0091A872E5AC018340A0FFBCD4340642A2697882E5AC0D0579A45FCCD4340805B881F8D2E5AC0672FB7F5FFCD43401BA70D598D2E5AC01017342500CE4340745668A68F2E5AC09F27080502CE4340643ADBA68F2E5AC0207F640502CE4340A8406C6A922E5AC09063C00602CE4340C8B66BF3932E5AC0D81A880702CE4340D820273A952E5AC0A86F6D0902CE434098FB0D3B952E5AC098F3A58706CE4340406DA839952E5AC0506DE59C0DCE4340083C94F4922E5AC0408D72380CCE43400C6D999C922E5AC050D307020CCE434004AC12E8902E5AC078B261DB0FCE4340108D245D8D2E5AC0C84F85DA17CE4340705923ED8C2E5AC000785AD718CE4340C03B5AB18C2E5AC0107B535E19CE4340BFEA12B28C2E5AC0102A0C5F19CE4340FC9CCDD38E2E5AC0100A2FA01BCE4340F7826D5C902E5AC088CFFB3E1DCE43404C4F29F3922E5AC048AE25FB1FCE4340138FFC08962E5AC0D806923D23CE4340FB7DB1EA982E5AC06031C14826CE43406C4947CC9B2E5AC0084F1B5429CE43405438FCAD9E2E5AC0C8BC3F5F2CCE434008DE8F8FA12E5AC030009C6A2FCE434007F99085A22E5AC0D04A296E30CE434058D72767A52E5AC080FF8B7933CE434094863925A62E5AC0E8ECA34134CE4340F442B74DA72E5AC090280BD134CE4340F0A16B5CA82E5AC0103BFB5335CE4340409B80B2AB2E5AC05001F0EF36CE43403B584908AF2E5AC078771A8C38CE434060BC2E75B02E5AC07009B83B39CE4340D7E30085B02E5AC0F8CD534339CE4340680386CBBB2E5AC028F1BE3942CE4340C089E921BE2E5AC09FB6980246CE43405BA89F44BF2E5AC0609C50D947CE4340A85F6745BF2E5AC0E0E754D947CE43406C2EF645BF2E5AC088D53CDA47CE43401469B6E6BF2E5AC028C4F6DC47CE43409C8644E1C02E5AC0986435E147CE4340446A78E5C02E5AC0800813E147CE4340D396F9F5C02E5AC0E85494E047CE4340F0B9AF0AC12E5AC0E06AA0DD47CE4340F49DE215C12E5AC02020C4DA47CE4340305B411FC12E5AC0F8F15DD847CE43404BD8B525C12E5AC00807EDD547CE4340C8896633C12E5AC03853C4D047CE43407C5DBA3AC12E5AC0106A4DCD47CE4340A41D3A47C12E5AC0A88D56C747CE434058852049C12E5AC0288741C647CE4340BFE8AA5AC12E5AC0D01BA7BB47CE4340B4805B6DC12E5AC008CBDEAD47CE43401B8D727FC12E5AC0002AF79D47CE434084EAD090C12E5AC06083778C47CE43406C4D72A1C12E5AC04F49E37847CE4340FFC40EB1C12E5AC01827AA6347CE434033BA9DBFC12E5AC0C0B3D44C47CE4340C0219AC7C12E5AC000E2643E47CE4340E08032CDC12E5AC0786D3E3447CE4340E4B67ED9C12E5AC0A0F2811A47CE43409C98CEE4C12E5AC0A89B78FF46CE43406B72A3EEC12E5AC058EA46E346CE4340283B71F7C12E5AC028DA26C646CE4340F8FBC3FEC12E5AC00FD40FA846CE4340A31DE9FFC12E5AC078B423A246CE4340185A8A02C22E5AC00F5F959446CE43407CC5B904C22E5AC000B6488946CE434008EE8705C22E5AC0D7BFA78346CE434078E53909C22E5AC06834CD6946CE4340CC2DDE09C22E5AC080D2686346CE4340E8335F0CC22E5AC068200F4A46CE4340B04EDB0DC22E5AC0A017812D46CE43409067080EC22E5AC0C09F102A46CE43406C80350EC22E5AC028D8D30946CE434064E85A0CC22E5AC050540AE245CE434038A845F6C12E5AC078CB9C0844CE43401FD45DF0C12E5AC0A09CE08943CE434040B1BBC4C12E5AC0DFD68F1A41CE4340A09C55BDC12E5AC0A0BEDECA40CE4340A860DC8CC12E5AC07032FBC03ECE43406C1C158BC12E5AC02858D2AD3ECE4340FB318D89C12E5AC06092BA9F3ECE434088F76477C12E5AC0304E6FF83DCE434067ACF475C12E5AC0C0D627EB3DCE43404BC21431C12E5AC0E821F4823BCE434047C47484C02E5AC0D8A1FA7135CE43409442DDC3C22E5AC0B8285FAF34CE434040242EF3C22E5AC0E09C5B9F34CE43400CC8E82AC42E5AC0B02B43E23FCE43409C99D947C42E5AC0E8F1317B40CE4340183B1561C42E5AC0E7461A1441CE434004B7CD6BC42E5AC0E03FDE6441CE434017B78D76C42E5AC0D8C0DCAF41CE434020CA0E8CC42E5AC088C9984B42CE4340045988A1C42E5AC0F73B74E442CE4340705947B3C42E5AC098D2018343CE43409C7F08C5C42E5AC000D3AE1E44CE4340F84F12D3C42E5AC0A0AD59BA44CE4340D0D417E1C42E5AC0C061DA5845CE434078742EE8C42E5AC0F7CC4B9845CE4340C8461CEFC42E5AC0208761F745CE4340349B6DF2C42E5AC04061613F46CE434074D4AEF5C42E5AC067E7CE9546CE4340978B8CFCC42E5AC088FA660047CE43405B16FEFFC42E5AC08048643147CE4340ABDC9306C52E5AC0485DCDCF47CE43406F5BAD09C52E5AC0A0E7024348CE4340E4DB6B09C52E5AC047710E7148CE43401BDB430CC52E5AC0903A730F49CE43405F97650BC52E5AC00FB0C4AD49CE4340A853870AC52E5AC0FFD9114C4ACE43407C89BE09C52E5AC0088602DC4ACE434004AF1605C52E5AC03008AA884BCE43402074C5FCC42E5AC01804E6264CCE43407C279AF8C42E5AC06F2772774CCE4340D0B42DF8C42E5AC0D0BC2CC54CCE4340EB7F08F0C42E5AC040439A464DCE4340F83626ECC42E5AC0A88A57634DCE434044A61DE0C42E5AC010A486014ECE43408B9A04DCC42E5AC0289492464ECE43404C3B17D4C42E5AC0B097B39F4ECE4340638D5AC4C42E5AC0085EF73A4FCE4340F89399B4C42E5AC0604915D94FCE43405C7D24A1C42E5AC0F0BB457450CE4340C05C6699C42E5AC0481EFFA750CE4340A89FF189C42E5AC0F8E2710F51CE43404CE23D7EC42E5AC0F0A0257151CE4340340D8476C42E5AC08855A2AA51CE4340947BFE6AC42E5AC01898F0E651CE434007A1965BC42E5AC0C828BB4552CE43404FAB1E4CC42E5AC0B8830EB052CE4340B8216944C42E5AC0106E02DE52CE434004E90F1EC42E5AC020C1C7AC53CE43406C08ED86C12E5AC078758BB666CE4340AB819877C12E5AC0B83CF50667CE4340F712B473C12E5AC0E08A712967CE4340D8272E64C12E5AC0C882649C67CE43402016E95FC12E5AC04023CEB367CE43403896A858C12E5AC018A797DB67CE4340E0E6F050C12E5AC0E7BD4C0F68CE43409867EE4CC12E5AC088910C4368CE4340B03E2145C12E5AC05F542F8568CE4340945B6B39C12E5AC01838E5E668CE4340C080C42DC12E5AC098A4FD3F69CE43408098CA29C12E5AC07771FE6D69CE43408C4FE825C12E5AC0A8DEBD8A69CE434050F0FA1DC12E5AC028E2DEE369CE4340C4FFF119C12E5AC08853661D6ACE4340F834EB15C12E5AC06879E9566ACE4340BCCFD10DC12E5AC02718DBCC6ACE434070B39A09C12E5AC070D806266BCE4340E4C8BD05C12E5AC0E01FC4426BCE434084FB9405C12E5AC0D8E0965F6BCE43401CF25E01C12E5AC0DF2FBCB86BCE434010AAF9FCC02E5AC068479B316CCE43406FF4D6F8C02E5AC0403E667C6CCE434047D39AF8C02E5AC0E7A297A76CCE434043303AF8C02E5AC0102AACEC6CCE43401478F5F7C02E5AC0B86F9A1D6DCE434008A157F7C02E5AC0C735DE8D6DCE4340D0AEA9F6C02E5AC0907AA6096ECE434010E2BEF9C02E5AC0DF29B67F6ECE43403428D5FCC02E5AC0F8FEC7F56ECE4340C8659D03C12E5AC057DCE86B6FCE4340A0146C0AC12E5AC0EF9307E26FCE43400C6FE90DC12E5AC068B5430D70CE4340F4773611C12E5AC05071285870CE4340F8DCA514C12E5AC040E4FF8B70CE4340E056BA1BC12E5AC0F85658CE70CE4340ECB94526C12E5AC0A0E99C4171CE434018082C2DC12E5AC0C85F95A371CE4340B7ABCA30C12E5AC0085EC6B771CE434098296734C12E5AC0C8A7FBCB71CE4340B8510C3FC12E5AC0F0F8192B72CE434038AC494DC12E5AC0E04F50A172CE43405F005C54C12E5AC0D0C180E672CE4340583F8A5BC12E5AC04F36A81473CE4340985A5A66C12E5AC01084265473CE4340CF73866DC12E5AC048FF0C8873CE4340CCF3867FC12E5AC0BF7C6DFB73CE4340DBC97D91C12E5AC040D5F36B74CE4340086732A7C12E5AC0FF5A63DF74CE43407C0E6FADC12E5AC030CAC6FF74CE434090CBE3BCC12E5AC00896F64F75CE4340887BD9C7C12E5AC0480B467B75CE4340388E5BD6C12E5AC0901C8EC075CE434050BA88DDC12E5AC0486BB3EE75CE4340046413ECC12E5AC0100C1D3176CE4340C4A64AFEC12E5AC058B46D7676CE434098D23809C22E5AC0A7C0C5A176CE43409058651BC22E5AC0700D9DF276CE4340BC49AC22C22E5AC070FC800F77CE4340B4643D31C22E5AC03769424977CE4340743CD93FC22E5AC048CF447D77CE4340EC0B664EC22E5AC0D01DEBB977CE43401042075DC22E5AC098ED0CEB77CE4340885A367AC22E5AC0B075F45578CE4340A03E2990C22E5AC08800529B78CE4340F89322A6C22E5AC01F8CD7DD78CE4340E7F06CC9C92E5AC08FD1E81894CE4340F8DC2494CB2E5AC0E761DBEE9ACE4340989E60C7CB2E5AC0E08DCC909BCE434063C142E8CB2E5AC0F8D196FE9BCE4340E027D8F6CB2E5AC048CE79359CCE4340B49C6810CC2E5AC0D85EEF919CCE4340880F5526CC2E5AC0B0A52FDA9CCE4340E41D5031CC2E5AC03814C0FF9CCE4340DC551652CC2E5AC08856BB819DCE43406B76D57DCC2E5AC0680747299ECE4340C8E2D7A5CC2E5AC0006CA6D39ECE43405F29D8CDCC2E5AC0D0AA037E9FCE434060460DE0CC2E5AC09F161EC99FCE43401FF4DFF5CC2E5AC028356528A0CE43409097FF07CD2E5AC0A888FB7EA0CE4340EC0A6A16CD2E5AC067B68FD5A0CE4340382A8228CD2E5AC068F2C934A1CE4340509DAD3ACD2E5AC0B8CEC282A1CE434050491B49CD2E5AC0B03F4CD9A1CE4340ACBC8557CD2E5AC0706DE02FA2CE4340DC1A0C66CD2E5AC0F89C4372A2CE4340FC1E1478CD2E5AC0F03100DDA2CE43401C1DF089CD2E5AC0A0AD9164A3CE4340A03E2B91CD2E5AC0D838ED8CA3CE4340445E42AACD2E5AC090FCE43CA4CE4340A838C0BFCD2E5AC080DF9ED8A4CE4340643255C3CD2E5AC0C099B2EFA4CE434033FCB4D8CD2E5AC0E0E3949FA5CE4340B8F607F5CD2E5AC0B0BBCBA8A6CE43406031A988D02E5AC0087089AFB8CE4340131253C9D02E5AC068E16E5ABACE43403881B70DD12E5AC0900F5F05BCCE43402C209259D12E5AC0502993ADBDCE4340D32725A9D12E5AC090D2E852BFCE4340700977FCD12E5AC0285F73F5C0CE43403BEFCF3DD22E5AC078F5AF21C2CE43403BE23B57D22E5AC080A80898C2CE43401C82BEB5D22E5AC0B064F434C4CE43408F9DFA17D32E5AC0E020E0D1C5CE43401C80F47DD32E5AC0A031076CC7CE43403CF0FFB0D32E5AC088D17630C8CE4340089BB2E7D32E5AC0F0AC7500C9CE43401F02B230D42E5AC0A71B6007CACE4340D33BDD58D42E5AC04856F594CACE43407CC9C7CDD42E5AC0B071A326CCCE4340E0567346D52E5AC0A08EA1B2CDCE43401428655CD52E5AC0EF89DDFACDCE4340D85FD8C2D52E5AC05868AA3ECFCE43405FE9BA25D62E5AC0C0137668D0CE43403BD1B646D62E5AC0686905C5D0CE4340DFE0586BD62E5AC048450030D1CE4340E3B39BCAD62E5AC0B0CC9845D2CE4340F467F155D72E5AC0108C4EC6D3CE4340C01B08E5D72E5AC0B7444541D5CE4340C483DB77D82E5AC0F05177B9D6CE4340B833FDB9D82E5AC0C7FF8C5BD7CE4340B89F2C12D92E5AC0B860F92BD8CE43408CCE7EACD92E5AC008FF9C9BD9CE43403CC533C6D92E5AC0A86548DBD9CE434098B18D4ADA2E5AC058786608DBCE43400B4244C0DA2E5AC0B82B4107DCCE4340735B1BF0DA2E5AC000F4A76CDCCE4340C8936399DB2E5AC0F877F8D0DDCE4340A8AFF428DC2E5AC0886F9BF5DECE43402063B442DC2E5AC0A8EDA22CDFCE43407805D863DC2E5AC020FB4C6FDFCE434008887DF3DC2E5AC0AF468285E0CE4340444E02A8DD2E5AC0088396DBE1CE4340D05D10F9DD2E5AC0805A766FE2CE4340A8988E5CDE2E5AC0588B0229E3CE4340D0A6B92EDF2E5AC0288D5A99E4CE43404075B90FE02E5AC050278C12E6CE4340C425DCCBE02E5AC040F70849E7CE4340C4FBC18BE12E5AC040C1EE76E8CE4340F8FD10B8E12E5AC0C82ABBB9E8CE434060E4694FE22E5AC0082305A2E9CE4340705BCC16E32E5AC057EF62C7EACE4340F8F7F1E1E32E5AC0F84B0AE7EBCE4340E0CCDBB0E42E5AC027AA0101EDCE4340C47B8483E52E5AC0D0DB3715EECE4340EC16ECDFE52E5AC0F830748CEECE434024763156E62E5AC088E89326EFCE434088D37DF1E62E5AC0E038B7E3EFCE434078245C30E72E5AC090F7672FF0CE4340F4568E0AE82E5AC020437232F1CE434023A60742E82E5AC040988272F1CE43407420C9E4E82E5AC0F01EC62FF2CE4340803F7CC6E92E5AC010226C27F3CE4340D8663EA8EA2E5AC037F16916F4CE4340A41CBB8DEB2E5AC0B03AA502F5CE434000063404EC2E5AC0886E5077F5CE4340CC0AFC76EC2E5AC0E8DE31E6F5CE434060E107C1EC2E5AC0EF17202FF6CE43401C7D4460ED2E5AC09FA9E8C6F6CE4340A4245A0EEE2E5AC06FA6A164F7CE4340F8440551EE2E5AC0B876179FF7CE434067AC1C3EEF2E5AC00053936EF8CE4340CC250A97EF2E5AC0303B8FBAF8CE4340B300F42EF02E5AC0D755393BF9CE4340D8418B23F12E5AC080784AFFF9CE4340A48CE983F12E5AC0C0BC684BFACE43403CCEE71BF22E5AC03051A7BDFACE4340C077E09EF22E5AC0F0626E1FFBCE4340781F5CA1F22E5AC0B8204B21FBCE434000B94914F32E5AC020234576FBCE43408B60B60CF42E5AC0980C3F26FCCE4340981992CDF42E5AC0A0006AA7FCCE43402BA99F0CF52E5AC0A08E91D0FCCE4340C0DA0F13F52E5AC0C0F8AFD4FCCE4340DC45DB08F62E5AC05F3D2472FDCE4340205ED008F72E5AC0B01B180EFECE4340A858930CF82E5AC050F34CA4FECE434078C45C10F92E5AC068E2DD31FFCE4340845C1295FD2E5AC0EF012AF100CF43407C7BB424FF2E5AC0C7CDB08B01CF4340FB760631032F5AC0F8DFD21D03CF434094C29291042F5AC0DF3A53E303CF43404805CBC7042F5AC0B873960304CF434008717C52052F5AC028F4165604CF4340E3E4FB98052F5AC0F891348204CF43402815B30F062F5AC030F8B6CB04CF43409B0CE894062F5AC0D04BA22405CF43406078D7CC062F5AC03099F64905CF434060273F86072F5AC068390ECB05CF434080C3A53F082F5AC098BAE25106CF4340AFA78373082F5AC0000ADB7A06CF4340BFDB43F5082F5AC000F771DE06CF434014832BF9082F5AC0681786E106CF43409C95DCAA092F5AC0A8C9E16D07CF4340BCE5FDF4092F5AC0F8C768A807CF4340F40371600A2F5AC0C8310A0308CF43401CB958F80A2F5AC03097908608CF4340748545120B2F5AC0E87AEF9D08CF4340CC8212C40B2F5AC04016983E09CF43408C6D1D720C2F5AC0903F12E209CF4340EC8F591E0D2F5AC0A03393890ACF4340AC1F25200D2F5AC0D8E0518B0ACF4340747D43670D2F5AC0C81427EF0ACF4340908FE4BF0E2F5AC090DFE3D20CCF434028E2A6CC0E2F5AC038C0CCE40CCF43400C9808CD0E2F5AC070CDD94C0FCF43404CB0F7C90E2F5AC08069098011CF4340E751F2C90E2F5AC0B823F28311CF434000B463D9222F5AC0B80A51E12FCF434068465CB9242F5AC0172142B732CF434090042528252F5AC050D6E95933CF4340FC33F496252F5AC00884AAF933CF4340F045D005262F5AC087FEEA8D34CF43403F2CE610262F5AC020BB769C34CF4340E4330F7C262F5AC05808CE2D35CF4340244EFAEA262F5AC0E0E56EB935CF43408464C5F9262F5AC078F5E5CA35CF43407F0DAA68272F5AC040AD845636CF4340DB61BEE2272F5AC060EE2EE536CF4340B46ACE5C282F5AC0F827F27037CF4340741DE8D6282F5AC098A5D2F937CF434088468C3E292F5AC0E094B76538CF4340F0CFC254292F5AC0C824037D38CF43402018A86E292F5AC0B85A259A38CF4340C4499AD2292F5AC070582F0039CF43401F08E2322A2F5AC02822915D39CF434057C332542A2F5AC01F42A77D39CF4340F0D3D3D52A2F5AC020E142F839CF4340331398232B2F5AC0F0D63B413ACF434087E474572B2F5AC0589EF96F3ACF434034BCD3DC2B2F5AC0585CD8E43ACF43401C81C5352C2F5AC0C0F9F72D3BCF43400C183A622C2F5AC09713F8533BCF43400495859D2C2F5AC00F5CB2853BCF4340E40AA9E72C2F5AC020803BC03BCF434048208F5A2D2F5AC040CD11183CCF434030F5C9702D2F5AC03813A9293CCF43401FAFF6F92D2F5AC0C09E2F903CCF434018002C832E2F5AC09823F7F03CCF4340A7CC1A102F2F5AC0771AED4E3DCF4340B0B2F11E2F2F5AC0CF1BBE573DCF4340BC7B169D2F2F5AC090731BA73DCF434054B68017302F5AC050788DF33DCF4340ACA60A2A302F5AC0B86352FF3DCF4340E8D90DB7302F5AC0986ABD513ECF4340D42E74F2302F5AC010E22F723ECF43403CD4CE47312F5AC0C8987A9E3ECF4340D47F00C6312F5AC047264FE23ECF4340AFBB8ED8312F5AC0F8C637EB3ECF4340074D5869322F5AC0207D2F323FCF43404CB6E5AF322F5AC0F82AC2523FCF4340279B2CFA322F5AC0D80666733FCF434047E9008B332F5AC048FABBB13FCF43409FEB911F342F5AC087EE39ED3FCF434080A74B57342F5AC0C0041B0240CF43405C4C28B4342F5AC04872D92540CF43401C44C748352F5AC00815BC5840CF4340183AAA8B352F5AC05887BF6D40CF434060876ADD352F5AC0C8A8D08540CF43406C871872362F5AC090AE13B040CF4340B761C406372F5AC0983B69D740CF43403FAD1F51372F5AC0004EACE940CF4340D828309F372F5AC02786F1FB40CF43408BF8E933382F5AC0B0C1AB1A41CF43402C0B0382382F5AC060CD2F2741CF4340DCDA65CC382F5AC0C767AD3341CF4340188D2C61392F5AC00F95C14941CF4340ABEB215E3A2F5AC010F9F46641CF4340D0C47C25422F5AC0B84FFF7A41CF4340E74BD03B422F5AC028794A7B41CF4340F8C8AF4A422F5AC04FBF607E41CF43407CD4495D422F5AC038C2818141CF43408C51296C422F5AC0D7BC938441CF434098CE087B422F5AC0B891A38741CF434044EDE289422F5AC05848988D41CF4340D01EBE98422F5AC0F8FE8C9341CF434000899CA7422F5AC098B5819941CF4340285C72B6422F5AC0780257A241CF43402F4249C5422F5AC0D80328AB41CF4340173B21D4422F5AC0F02AFBB341CF43407FE8F4E2422F5AC090C2ACBF41CF434088CECBF1422F5AC0F8F18EC841CF434010699E00432F5AC0488131D441CF4340C0ADB90B432F5AC0A03EE5DF41CF4340B41BA125432F5AC0500301FD41CF43403CB67334432F5AC070E6B60842CF4340C876873F432F5AC08013211A42CF4340384A9C4A432F5AC010D0AC2842CF43405FACAA55432F5AC060D7143A42CF43401C66D56B432F5AC0C8A2EF5C42CF4340E800E776432F5AC05884556E42CF434098AEF981432F5AC0A88BBD7F42CF43403491C68C432F5AC06832929342CF43401CD8048D432F5AC0C00B139442CF4340171D5F94432F5AC0885548A842CF43408C16B59B432F5AC08FA78CBC42CF43401814E8A1432F5AC070A4ECC742CF4340988BC4A6432F5AC0E06AD7D042CF4340684C17AE432F5AC0E72513E542CF43401C206BB5432F5AC0B0C233FC42CF43409419C1BC432F5AC0F8EE751043CF4340103958C0432F5AC07883872743CF4340A31FADC7432F5AC04020A83E43CF4340601942CB432F5AC078DABB5543CF434014ED95D2432F5AC04077DC6C43CF4340344530D6432F5AC08031F08343CF4340CF51C6D9432F5AC0C0EB039B43CF43406C5E5CDD432F5AC07816F6B443CF4340ECAEBADB432F5AC0D01B9A9145CF4340E488E3C3432F5AC0E05919CC60CF434064BF83D9432F5AC0A0355A097BCF434097F82FE5432F5AC0B0AB253189CF4340847661E0432F5AC04FE7B8F789CF434063E0D5DF432F5AC020E3735C8ACF43401C98C6D6432F5AC08F1AC2878BCF434020E7FEC9432F5AC070D8FAB28CCF434078E34AC1432F5AC0403D30998DCF4340631035BD432F5AC0587159DB8DCF43401B8B06B9432F5AC028C1A6318ECF4340207BFBA8432F5AC0E78F7A068FCF43401FDCB79C432F5AC0B89CD4CF8FCF434038ADBE94432F5AC078AE9B3190CF434020C2CD7C432F5AC07F2ECD5991CF4340045A2869432F5AC0884A741A92CF4340E0DF695D432F5AC08080ED8192CF43406023083E432F5AC0B81503AA93CF4340706FF41A432F5AC0A81438CF94CF4340A8CE20F4422F5AC0D0BF59F495CF4340AC1099C9422F5AC070886E1997CF4340443B75B2422F5AC0807BF2AB97CF4340D43F109F422F5AC0D7BAA23B98CF434044859278422F5AC0985D342799CF4340D40E1D6D422F5AC0A873C15D99CF4340A0E36C65422F5AC0C0C7D48899CF4340707F243B422F5AC0B806DE7F9ACF4340F0871C11422F5AC0C04AD5489BCF4340EC8FC101422F5AC0F7F2FB9E9BCF434028C660C8412F5AC038BA3FBB9CCF4340B83E52C83E2F5AC07066D704ABCF434068CD20B53E2F5AC0E0B9436CABCF4340806F43873E2F5AC018F3344CACCF43405CB0FF603E2F5AC0E856970CADCF4340A86731463E2F5AC0B0A46C96ADCF4340BC647A273E2F5AC0A8D7C642AECF4340541E8F0C3E2F5AC0A0F5BBE0AECF4340C4CA8FF53D2F5AC0F810395CAFCF434040AFEAD23D2F5AC0104EF22DB0CF4340309B6CA83D2F5AC07008614AB1CF4340B4FEB5A03D2F5AC090D4397BB1CF43403B7E36723D2F5AC090F970CBB2CF4340D0AA8B4B3D2F5AC0C836ABD6B3CF43408323B9433D2F5AC05844AA1BB4CF434040BCFB373D2F5AC050360686B4CF43409061A91C3D2F5AC0EF9ED96EB5CF43406448120D3D2F5AC010862FF0B5CF434080B29AF53C2F5AC008892ABFB6CF4340349CF9D53C2F5AC068F34F15B8CF43400365F1CD3C2F5AC068CF9F82B8CF4340203A15BA3C2F5AC01841AA68B9CF434057E2E3A13C2F5AC05096EBBEBACF43404884B2953C2F5AC0A0D9E479BBCF4340B477B1893C2F5AC008373115BCCF43403804F2783C2F5AC0882B8A6BBDCF43401432EE6B3C2F5AC0E073F6C1BECF4340105BE5623C2F5AC087519AE4BFCF4340406A9D623C2F5AC038536B18C0CF4340DF30075D3C2F5AC057ABCD71C1CF4340544D285B3C2F5AC000E63CCBC2CF4340E8501C593C2F5AC0F09A7AF1C6CF434078D12F5B3C2F5AC0F8AC2194DECF43405BE4305B3C2F5AC078BEE7A6DECF434084E9172C422F5AC0C08035A6DECF43409C68AC6A5A2F5AC078BC43A3DECF434013F20C705A2F5AC078BC43A3DECF43404CA6D69D5C2F5AC04804FFA2DECF4340CC35CED1632F5AC0683C19A2DECF434043624BBA662F5AC028BFBAA1DECF434084B4CAA2692F5AC060F657A1DECF43407F2C4C8B6C2F5AC02079F9A0DECF4340FC58C9736F2F5AC020D698A0DECF43401CBE495C722F5AC0580D36A0DECF434097EAC644752F5AC09044D39FDECF4340F429452D782F5AC0D07B709FDECF4340307CC4157B2F5AC008B30D9FDECF43406BCE43FE7D2F5AC088C4A89EDECF4340A720C3E6802F5AC000D6439EDECF4340046041CF832F5AC078E7DE9DDECF4340808CBEB7862F5AC0F0F8799DDECF4340C37546A0892F5AC0A8E4129DDECF43409C69C0888C2F5AC068D0AB9CDECF4340BCCE40718F2F5AC020BC449CDECF434037FBBD59922F5AC01082DB9BDECF434034733F42952F5AC0D06D749BDECF4340AF9FBC2A982F5AC0000E099BDECF4340D0043D139B2F5AC0F8D39F9ADECF4340C0B067289E2F5AC0E8022E9ADECF4340FF02E710A12F5AC0A0EEC699DECF43405C4265F9A32F5AC0D88E5B99DECF4340B881E3E1A62F5AC0102FF098DECF4340A0B24240A72F5AC0884CE398DECF4340183CA345A72F5AC0BF26E198DECF43403829A245A72F5AC0F08DDE99DECF4340240F40A7F62F5AC08850830EDFCF43405C25174A03305AC078AECA20DFCF43405C25174A03305AC008792720DFCF4340F096875903305AC0580EE84AD9CF4340DF1BA975FD2F5AC010CEAEB5D6CF43402B9D4F7DFD2F5AC0F05A5890D6CF4340446A3981FD2F5AC0C80CDC6DD6CF43406C67D888FD2F5AC0E82F664BD6CF4340C00EC08CFD2F5AC08007EC28D6CF4340B8EEAA90FD2F5AC018486906D6CF4340685D8F94FD2F5AC0EFF9ECE3D5CF434034226A98FD2F5AC000DFF0CCD5CF4340A4B4AC98FD2F5AC05899029CD5CF4340D86E959CFD2F5AC0B0FF8179D5CF4340FCF8C89CFD2F5AC000131654D5CF434028A93D99FD2F5AC088258231D5CF4340C7E76C99FD2F5AC09883F20ED5CF434078E3E595FD2F5AC060B479E9D4CF434018221596FD2F5AC0B083F0C6D4CF4340C81D8E92FD2F5AC0B04A58A4D4CF4340247AEF8EFD2F5AC038722990D4CF43403CFAAE87FD2F5AC038E6A56AD4CF4340C8712084FD2F5AC0B8D9CE4DD4CF434017CA3979FD2F5AC07830D719D4CF4340B0FEF471FD2F5AC0B0F616FAD3CF434090A65A6EFD2F5AC0783C03E3D3CF434057870267FD2F5AC0EFF1A5D1D3CF434064D9B05FFD2F5AC0680AA9B7D3CF4340C8CC1A5CFD2F5AC0285095A0D3CF4340FBC66D4DFD2F5AC098F71078D3CF43403C5A6D42FD2F5AC04FDB4358D3CF43404CD0B833FD2F5AC0B8B66738D3CF434087CCAF28FD2F5AC0A8307B1BD3CF434050D1F419FD2F5AC0981386FED2CF4340B051F30EFD2F5AC0D06797E1D2CF434038E53100FD2F5AC0F0B271C7D2CF434000EA76F1FC2F5AC0200783AAD2CF43404CA55BE6FC2F5AC0C8E0D79ED2CF434034E714D5FC2F5AC0F89EB283D2CF434008164ED2FC2F5AC0289A427FD2CF4340988239D0FC2F5AC0203BFF7BD2CF4340044FBABDFC2F5AC0C8D1DD61D2CF4340987BA5B2FC2F5AC0B8A47350D2CF4340BBD47B9CFC2F5AC010776033D2CF43408868F989FC2F5AC0303A001FD2CF434054FC7677FC2F5AC090D79D0AD2CF43407CDBE233FC2F5AC008290DDCD1CF434004A67E04FC2F5AC0284D69BBD1CF4340483CEB430C305AC0B876DFF4D8CF434093DF1AEC0E305AC098F0C32FDFCF43403CA48DC00B305AC0E8267190EFCF43400C8515F509305AC0688FC8F829D04340742E9B1F10305AC0F00AE4493AD04340F001394215305AC0284B57643FD04340C8698D7619305AC0000CC2513CD043406C1C0ED11C305AC0E021AA623DD0434010F25F2521305AC02F17141A49D043407C9A4AD425305AC008459B6C4ED04340D8FC24162F305AC0D05E075252D043401867B9BD3B305AC060D151D47AD043401CD1C72640305AC0882A302F83D0434084E73AC73F305AC037C0F2ABAFD043408C72EBC63F305AC0D0FE22D0AFD04340F0BD9AC43F305AC0F0085DE5B0D04340E087B9C03F305AC0E008E8B1B2D043408C6F0AB93F305AC088FDCD47B6D043407834A4A33F305AC000EE123FC0D04340F8C81F9F3F305AC0C843AC59C2D043403047258E3F305AC027C6D341CAD04340E47B35893F305AC028F3208ECCD043405405FCDDA52F5AC0789D3AE7CBD04340DFAABE62292F5AC0E8BF6E48CBD0434024DFABD7262F5AC010D115FDC5D043401C78A443252F5AC0E065EEDDC3D0434020C19D09222F5AC0A8FC4CD9C3D04340502249431F2F5AC0A7B684B1C2D04340D8CA4F001E2F5AC008C44F49C1D04340A82C97AE1C2F5AC020984F35BED04340781717901B2F5AC080784AFFB9D043406F29E0641C2F5AC0FFDCD5BAB3D04340848886B81E2F5AC068C6D547AED0434058F9A8961F2F5AC0C0C767ECAAD043400F18EC651F2F5AC047460992A8D043400B3CAF921E2F5AC08836D306A5D04340CBED520E192F5AC09798119290D043405FAD61B9172F5AC0786A3FD08CD0434044BF060F162F5AC0B79687C186D0434068A31D11152F5AC0FFC8FB4584D043408893413C122F5AC0E899565E80D043409C8DF453102F5AC0DF31267E7ED043403049DFE30D2F5AC0C068EE3F77D043406090D95E0D2F5AC058A1564576D04340F8C41C92072F5AC0484CEB046ED04340287D0321062F5AC0101ED16A6CD0434030445ED4032F5AC00097375C6BD043403C244D00012F5AC0875D9D9B6AD0434080182EF9FF2E5AC02891146369D043403068F412FF2E5AC02892D33766D043401092FE24FF2E5AC038E5204161D04340289737CE002F5AC00721E76A5CD04340E0A231C5022F5AC060CA72DF58D04340E0084EBA032F5AC028FA6BD555D043407C213CBE032F5AC057C6ABEB51D043406CBE6BDF022F5AC018BBD9334CD0434000C676CE012F5AC060CD379849D04340C071918FFE2E5AC060F6FB3048D04340E819CBECFA2E5AC0C0A00E6346D04340F085CD56F72E5AC070D18A5B44D0434098D014CFF32E5AC0D06D491B42D0434007DC1957F02E5AC0786332A33FD04340A0CD4AF0EC2E5AC02F9358F43CD04340588F3904EB2E5AC080074B513BD04340284EC9FAEC2E5AC0185D72A43FD043404CAB4011F12E5AC040D40B1846D0434007F759B1F42E5AC0D8477F534BD0434070A40EB3F62E5AC0D04271504DD04340B858AB85F82E5AC03FFCB4D84DD04340406F350EFC2E5AC0F88BD8484ED043408CD9C690FD2E5AC000F4609750D043400F463634FE2E5AC0983C2CAE53D043401CFDD0AEFD2E5AC0F0AAD39057D0434044221776FA2E5AC00879C9AE5DD04340C840EEEDF92E5AC06023ABF061D04340B87BC153FA2E5AC098E8909467D04340B8ADE4F8FB2E5AC0E8E540146ED043409F7C2C70FE2E5AC088539AC572D04340D0E10EF2FF2E5AC0A0B65D0D75D04340786C5D2D012F5AC078823BF275D04340EF4063DA022F5AC0F8622E9777D04340E85CBA77042F5AC0D858D57478D0434014B62610082F5AC0F79E56137CD043401463ABF4082F5AC08FADAADB7CD0434034B3A72E0A2F5AC0987ADAD77DD0434080D41F270D2F5AC0885CD59E81D04340B8F803670E2F5AC050C5DD5884D043400BCEC5E40F2F5AC0A7EEC89A89D043403C5E05B4112F5AC078C7B2988CD0434008195F19132F5AC0A0D07E2090D04340DCA68AD2132F5AC07F64FB7C93D04340A7C39E64152F5AC0804776DA97D04340EF19E3D2172F5AC0F03EEB7A9FD0434027593E60182F5AC0CF55DEEEA2D04340479E9BFE182F5AC0B86A6B9FA9D04340047A29EE182F5AC0A895E9D4ABD0434054AAAD06182F5AC0A075F42FB1D043401404A956172F5AC0B08CEA3AB5D04340630B3D9F172F5AC098914D3BBAD0434044101664172F5AC098C585EFBCD04340580F5BB7172F5AC030203EC7BFD04340F4B74746192F5AC0701061F4C3D0434048772A221B2F5AC0C889B3E5C6D0434018E546E21C2F5AC00864E31FC8D043405B1C54C91D2F5AC0B8AF6D38C9D04340F8BC18E61E2F5AC0C7E5BF47CAD043402FD1F115202F5AC06095FB1FCBD04340440CEE46202F5AC04063A33CCBD04340B0EF03F00C2F5AC0D81D8A23CBD043406F2413C70C2F5AC078AD0023CBD043406CC6440B6F2E5AC0A0327040CBD04340F40C55EF182E5AC00035134BCBD04340FC14A7BDF52D5AC0C8FF194FCBD04340A09400CAD92D5AC088FA8054CBD0434030000000481B335BE22E5AC068DFF620DECF4340D0471F74E22E5AC0201AD8F5CECF434074F18A4FDD2E5AC070DE6919C6CF4340E0CE84DADA2E5AC0D0273565BBCF434004977E57DE2E5AC000B622C2A3CF4340C4B02254DD2E5AC0D8E384978DCF43401B515676DB2E5AC000B8EC8C86CF43401071E4C8DD2E5AC0209DB5A67CCF4340983BD89AE12E5AC070D43C7A55CF434070004896E12E5AC090CECE164ECF4340F8FBEE7EDE2E5AC010DFBB004DCF4340E0091EBBD82E5AC048702F4546CF434073A06716D52E5AC0106D13D03FCF43400CD8B8D8D62E5AC018D8EE814ACF4340A0621C89DB2E5AC06727E43951CF4340F82B0385DC2E5AC0C8582FBE5CCF43409C5E1EA2D92E5AC030FEC4086ECF4340831A5434D92E5AC07818C5A57CCF43406B77BC1AD72E5AC03893C26B80CF4340DC76A203D62E5AC070E713FD8ACF434084D76C1AD82E5AC0C097DD8E8ECF4340D4A27BBFD92E5AC02F6FED359ECF434004DEB355D92E5AC098477D4DA7CF4340A822A4BAD52E5AC0AFD0E82AB7CF434020330F47D42E5AC0A077495AB9CF434050DA3DD4D12E5AC0FF50D05DBACF43402CE39DDECD2E5AC097CA2F7EB8CF434088A6EC87C72E5AC058BBE6F5AACF4340B3FC4F4ABD2E5AC0D88D72FBA8CF4340B43AB19BBA2E5AC0901D50E0A4CF43407C64CFD8B62E5AC01820C8B696CF4340678E1D9BB72E5AC0A08A77948BCF4340FCA4EFD5B12E5AC000A818DD7DCF434044105F3AAE2E5AC0B80BAED578CF4340EC0F2D42A92E5AC088E074DC7ACF4340BC172EC4AD2E5AC0E8C1DFC782CF4340D72AC7E5B12E5AC0A72037178DCF4340FBDC8FC2B22E5AC06005328799CF4340D03978B9B72E5AC09F6FD358A8CF4340A8AFDB8BBB2E5AC0183CF71EAECF43404C3EA79AC52E5AC0484C56A0B1CF434088A6C777C82E5AC0D82B6318B7CF43404058037DCA2E5AC0180B0135BFCF4340885C35A4D12E5AC00897EA16C2CF4340C4EFBE41D32E5AC05FEC21DAC1CF434060F3FE60DC2E5AC0E83018F6D0CF4340A0C5BB51DB2E5AC05069801CDECF4340481B335BE22E5AC068DFF620DECF434046000000C44021DFDE2E5AC02816D5E8B6D04340FB1D9C78DC2E5AC090F1EB9EB4D043403FF4B97AD82E5AC0F800BFAEB5D0434037D5120CD62E5AC0604F10E9B5D0434047C41412D42E5AC0507C9C73B4D043409BFDFBCBCF2E5AC02F428BBBADD043402809A854CF2E5AC0E8B10E63ACD0434018B80A2FCF2E5AC0B8130B3AA9D043409C5B22F6CD2E5AC0C0FD8D9EA4D04340886D8E7DCD2E5AC0182B11E5A1D04340170F7104CD2E5AC0D8B4ED499AD043407857DB7FCA2E5AC03794B6328DD04340BC3F19A1C72E5AC01039F6B988D0434068AFAA03C52E5AC078A019F480D0434098F9AA9CC32E5AC018AA634E7DD0434018502E74C22E5AC058926F3579D04340E4A8DF8FBF2E5AC090D0F04F71D0434094FAA6D4B82E5AC0A8B6A47564D04340E862D74CB72E5AC06F88A2C15FD0434028C6BB81B42E5AC0A7B964755BD043407FDF670BB12E5AC080F53A144BD043401C52DB9FAE2E5AC09879FBAB3BD0434000C27478AD2E5AC0A86B3B7A37D0434078D47D05AB2E5AC0E05D53CC33D04340C34398089F2E5AC068D3291327D04340601E30779C2E5AC008804C3329D04340342D211A9B2E5AC040DB236E2AD04340471BC1059A2E5AC0A88FD06E2BD04340B8CE1D629B2E5AC018CA8E9C2BD0434010F13B259E2E5AC060CE0A512DD04340EC8B78D6A12E5AC010015FF533D043404C6335A2A32E5AC0A8EFCED735D04340C88DAAA5A72E5AC03889586937D0434074C99AB9A92E5AC038EBCE783AD0434004B51115AB2E5AC050AE1F8B42D04340503E3E00AD2E5AC0C09D1ABB46D04340E083734DAE2E5AC080CB402B51D04340444D5F28B02E5AC09836E85A5AD0434094CF0A23B52E5AC0C8DE9AC763D043401F95AA66B62E5AC018C9AF7069D04340ACF31D99BA2E5AC040D777186FD04340D4E967E7BD2E5AC060C8AE1777D0434050591D31C12E5AC07070128581D0434040AFEB89C12E5AC038D7FFBF85D04340707CE672C42E5AC0A861E4118ED0434004B3E2B1C52E5AC090821A0A94D043403C10C2D1C52E5AC0C74D454B9AD043401BD2CC19C52E5AC088ACEB739CD04340D70176ECC22E5AC0083FBDFB9ED0434014754021C12E5AC04FA772D0A1D043403FB3D643C02E5AC0C069A459A4D04340E04404E1BF2E5AC010BE1D6CA7D04340ECA0CA4AC22E5AC00871F484A7D04340ECD7859BC32E5AC0CF9D06FDA6D04340307668DCC42E5AC010D5CF33A7D0434018D882C2C62E5AC02F570189A9D04340A00658B3C72E5AC0E82B5A83AED04340E36F6FA2C92E5AC0080043D3B5D04340F4EEF36BCD2E5AC0A0A757F0B9D04340BC2FB890CE2E5AC03884AEDABBD04340DBE80215D02E5AC07859275CBFD043406F508231D42E5AC01814658DBFD04340A45A51D1D82E5AC050223EF4C0D04340A89BFE5EDD2E5AC038C45EE6C3D043409000B0C5E42E5AC0C0979E2ACBD04340701FE046F32E5AC078760D28CBD04340B0AE822CEC2E5AC0788C7B2BC6D04340B3B86CF8E62E5AC0D009DD9CBFD04340F816163CE22E5AC050319B5BBCD04340C44021DFDE2E5AC02816D5E8B6D043402100000007B2A65E292E5AC0B0DF8B1D77D04340F47A8974292E5AC08041E9B067D0434020F7F5F51C2E5AC0382E54B267D043402006F0321D2E5AC0D00296C468D04340AC6573811D2E5AC00FF7EDCB69D043401000B3E01D2E5AC078E1BBC56AD04340CBE289A91F2E5AC068CAA1F46ED04340642D6918202E5AC0C890681170D04340B408BB77202E5AC088095D3B71D04340E8E0D6C6202E5AC040A16A7072D0434004A43805212E5AC0F8A689AE73D04340FC806E32212E5AC0302B83F374D0434018742F4E212E5AC0A7AF263D76D043403895D656212E5AC0F025235A77D04340B8B00F94162E5AC0B8B8BBAB77D04340A42735AC102E5AC0008C7CD877D0434050E74E39112E5AC0876F2A575AD0434074203B3A112E5AC0CFFB2A265AD0434068E863C7112E5AC0189A3FA13CD04340F07F55C8112E5AC0E8AADB6E3CD04340BC0B69271F2E5AC05068370C3CD0434050A198F40B2E5AC0B8B01E993CD04340A862A054032E5AC0F0BFC2DA3CD043405C11592A032E5AC0A857FB8F5AD043401BBF9524032E5AC0E0E1469C5ED04340F0FF05DDF92D5AC0482D4DE45ED04340509DA3C8F92D5AC0F00EDDE45ED04340609384C3F92D5AC0E852657D62D04340901A02C9F92D5AC010D1407D62D0434030B5761F032E5AC008EA6B3562D04340A0C85FFC022E5AC090BA633F78D04340D797E293162E5AC018D8FDAC77D0434007B2A65E292E5AC0B0DF8B1D77D04340'::geometry(geometry, 4326), -73.9366690032303 - -104.729102126902, 40.7045120351809 - 39.620441302097); +END; +$$ LANGUAGE plpgsql; + +--Used to expand a column based response to a table based one. Give it the desired +--columns and it will return a partial query for rolling them out to a table. +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_BuildSnapshotQuery(names text[]) +RETURNS TEXT +AS $$ +DECLARE + q text; + i numeric; +BEGIN + + q := 'SELECT '; + + FOR i IN 1..array_upper(names,1) + LOOP + q = q || format(' vals[%s] As %I', i, names[i]); + IF i < array_upper(names, 1) THEN + q= q || ','; + END IF; + END LOOP; + RETURN q; + +END; +$$ LANGUAGE plpgsql; + +-- Function that replaces all non digits or letters with _ trims and lowercases the +-- passed measure name + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_StandardizeMeasureName(measure_name text) +RETURNS text +AS $$ +DECLARE + result text; +BEGIN + -- Turn non letter or digits to _ + result = regexp_replace(measure_name, '[^\dA-Za-z]+','_', 'g'); + -- Remove duplicate _'s + result = regexp_replace(result,'_{2,}','_', 'g'); + -- Trim _'s from beginning and end + result = trim(both '_' from result); + result = lower(result); + RETURN result; +END; +$$ LANGUAGE plpgsql; + +-- Function that returns the currently deployed obs_dump_version from the +-- remote table of the same name. + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DumpVersion( +) + RETURNS TEXT +AS $$ +DECLARE + result text; +BEGIN + EXECUTE ' + SELECT MAX(dump_id) FROM observatory.obs_dump_version + ' INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; + + +-- Function we can call to raise an exception in the midst of a SQL statement +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_RaiseNotice( + message TEXT +) RETURNS TEXT +AS $$ +BEGIN + RAISE NOTICE '%', message; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + + +-- Create a function that always returns the first non-NULL item +CREATE OR REPLACE FUNCTION cdb_observatory.first_agg ( anyelement, anyelement ) +RETURNS anyelement LANGUAGE SQL IMMUTABLE STRICT AS $$ + SELECT $1; +$$; + +DROP AGGREGATE IF EXISTS cdb_observatory.FIRST (anyelement); + +-- And then wrap an aggregate around it +CREATE AGGREGATE cdb_observatory.FIRST ( + sfunc = cdb_observatory.first_agg, + basetype = anyelement, + stype = anyelement +); + +CREATE OR REPLACE FUNCTION cdb_observatory.isnumeric ( + typename varchar +) +RETURNS BOOLEAN LANGUAGE SQL IMMUTABLE STRICT AS $$ + SELECT LOWER(typename) IN ( + 'smallint', + 'integer', + 'bigint', + 'decimal', + 'numeric', + 'real', + 'double precision' + ) +$$; + +-- Attempt to perform intersection, if there's an exception then buffer +-- https://gis.stackexchange.com/questions/50399/how-best-to-fix-a-non-noded-intersection-problem-in-postgis +CREATE OR REPLACE FUNCTION cdb_observatory.safe_intersection( + geom_a Geometry(Geometry, 4326), + geom_b Geometry(Geometry, 4326) +) +RETURNS Geometry(Geometry, 4326) AS +$$ +BEGIN + RETURN ST_MakeValid(ST_Intersection(geom_a, geom_b)); + EXCEPTION + WHEN OTHERS THEN + BEGIN + RETURN ST_MakeValid(ST_Intersection(ST_Buffer(geom_a, 0.0000001), ST_Buffer(geom_b, 0.0000001))); + EXCEPTION + WHEN OTHERS THEN + RETURN NULL; + END; +END +$$ +LANGUAGE 'plpgsql' STABLE STRICT; +--Functions for augmenting specific tables +-------------------------------------------------------------------------------- + +-- Creates a table of demographic snapshot + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetDemographicSnapshot(geom geometry(Geometry, 4326), + timespan text DEFAULT NULL, + boundary_id text DEFAULT NULL +) RETURNS SETOF JSON +AS $$ +DECLARE + meta JSON; +BEGIN + boundary_id = COALESCE(boundary_id, 'us.census.tiger.census_tract'); + + EXECUTE $query$ SELECT cdb_observatory.OBS_GetMeta($1, + ('[ ' || +'{"numer_id": "us.census.acs.B01003001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B01001002", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B01001026", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B01002001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002003", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002004", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002006", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002012", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002005", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002008", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002009", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B03002002", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B11001001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003017", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003019", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003020", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003021", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003022", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B15003023", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19013001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19083001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19301001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25001001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25002003", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25004002", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25004004", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25058001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25071001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25075001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25075025", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B25081002", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134002", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134003", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134004", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134005", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134006", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134007", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134008", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134009", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08134010", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B08135001", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001002", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001003", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001004", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001005", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001006", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001007", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001008", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001009", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001010", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001011", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001012", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001013", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001014", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001015", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001016", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '},' || +'{"numer_id": "us.census.acs.B19001017", "numer_timespan": ' || $2 || ', "geom_id": ' || $3 || '}' || + ']')::JSON) + $query$ + INTO meta + USING geom, + COALESCE('"' || timespan || '"', 'null'), + COALESCE('"' || boundary_id || '"', 'null'); + + RETURN QUERY EXECUTE $query$ + WITH vals AS (SELECT JSON_Array_Elements(data)->'value' val, + JSON_Array_Elements($2) meta + FROM cdb_observatory.OBS_GetData( ARRAY[($1, 1)::geomval], $2)) + SELECT JSON_Build_Object( + 'value', val, + 'id', meta->'numer_id', + 'name', meta->'numer_name', + 'type', meta->'numer_type', + 'description', meta->'numer_description' + ) FROM vals + $query$ + USING geom, meta + RETURN; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeta( + geom geometry(Geometry, 4326), + params JSON, + num_timespan_options INTEGER DEFAULT NULL, -- how many timespan options to show + num_score_options INTEGER DEFAULT NULL, -- how many score options to show + target_geoms INTEGER DEFAULT NULL +) +RETURNS JSON +AS $$ +DECLARE + numer_filters TEXT[]; + geom_filters TEXT[]; + meta_filter_clause TEXT; + scores_clause TEXT; + result JSON; +BEGIN + IF num_timespan_options IS NULL THEN + num_timespan_options := 1; + END IF; + IF num_score_options IS NULL THEN + num_score_options := 1; + END IF; + + numer_filters := (SELECT Array_Agg(val) FILTER (WHERE val IS NOT NULL) FROM (SELECT (JSON_Array_Elements(params))->>'numer_id' val) foo); + geom_filters := (SELECT Array_Agg(val) FILTER (WHERE val IS NOT NULL) FROM (SELECT (JSON_Array_Elements(params))->>'geom_id' val) bar); + meta_filter_clause := '(m.numer_id = ANY ($6) OR m.geom_id = ANY ($7))'; + + scores_clause := ' agg_geoms AS ( + SELECT target_geoms, target_area, ARRAY_AGG(geom_id) geom_ids + FROM meta + GROUP BY target_geoms, target_area + ), scores AS ( + SELECT target_geoms, target_area, + CASE target_area + -- point-specific, just order by numgeoms instead of score + WHEN 0 THEN scores.numgeoms + -- has some area, use proper scoring + ELSE scores.score + END AS score, + scores.numgeoms, scores.table_id, scores.column_id + FROM agg_geoms, + LATERAL cdb_observatory._OBS_GetGeometryScores($1, + geom_ids, COALESCE(target_geoms, $2), target_area) scores + ) '; + + IF JSON_Array_Length(params) = 1 THEN + IF numer_filters IS NULL AND geom_filters IS NOT NULL THEN + meta_filter_clause := 'm.geom_id = ($7)[1]'; + ELSIF geom_filters IS NULL AND numer_filters IS NOT NULL THEN + meta_filter_clause := 'm.numer_id = ($6)[1]'; + ELSIF numer_filters IS NOT NULL AND geom_filters IS NOT NULL THEN + meta_filter_clause := 'm.numer_id = ($6)[1] AND m.geom_id = ($7)[1]'; + ELSE + RAISE EXCEPTION 'Must pass either numer_id or geom_id to every key in GetMeta'; + END IF; + + IF geom_filters IS NOT NULL AND numer_filters IS NOT NULL THEN + scores_clause := 'scores AS ( + SELECT NULL::INTEGER target_geoms, NULL::Numeric target_area, + 1 score, null, geom_tid table_id, geom_id column_id, + NULL::Integer numgeoms + FROM meta) '; + END IF; + END IF; + + EXECUTE format($string$ + WITH _filters AS (SELECT + row_number() over () id, * + FROM json_to_recordset($3) + AS x(numer_id TEXT, denom_id TEXT, geom_id TEXT, numer_timespan TEXT, + geom_timespan TEXT, normalization TEXT, max_timespan_rank TEXT, + max_score_rank TEXT, target_geoms INTEGER, target_area Numeric + ) + ), meta AS (SELECT + id, + f.numer_id, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_aggregate END numer_aggregate, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_colname END numer_colname, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_geomref_colname END numer_geomref_colname, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_tablename END numer_tablename, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_type END numer_type, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_name END numer_name, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_description END numer_description, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE numer_t_description END numer_t_description, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE m.numer_timespan END numer_timespan, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE m.denom_id END denom_id, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_aggregate END denom_aggregate, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_colname END denom_colname, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_geomref_colname END denom_geomref_colname, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_tablename END denom_tablename, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_name END denom_name, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_description END denom_description, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_t_description END denom_t_description, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_type END denom_type, + CASE WHEN f.numer_id IS NULL THEN NULL ELSE denom_reltype END denom_reltype, + m.geom_id, + m.geom_timespan, + geom_colname, + geom_tid, + geom_geomref_colname, + geom_tablename, + geom_name, + geom_description, + geom_t_description, + geom_type, + Coalesce(normalization, + -- automatically assign normalization to numeric numerators + CASE WHEN cdb_observatory.isnumeric(numer_type) THEN + CASE WHEN denom_reltype ILIKE 'denominator' THEN 'denominated' + WHEN numer_aggregate ILIKE 'sum' THEN 'area' + WHEN numer_aggregate IN ('median', 'average') AND denom_reltype ILIKE 'universe' + THEN 'prenormalized' + ELSE 'prenormalized' + END ELSE NULL + END + ) normalization, + max_timespan_rank, + max_score_rank, + target_geoms, + target_area + FROM observatory.obs_meta m JOIN _filters f + ON CASE WHEN f.numer_id IS NULL THEN m.geom_id ELSE m.numer_id END = + CASE WHEN f.numer_id IS NULL THEN f.geom_id ELSE f.numer_id END + WHERE + %s + AND (m.numer_id = f.numer_id OR COALESCE(f.numer_id, '') = '') + AND (m.denom_id = f.denom_id OR COALESCE(f.denom_id, '') = '') + AND (m.geom_id = f.geom_id OR COALESCE(f.geom_id, '') = '') + AND (m.geom_timespan = f.geom_timespan OR COALESCE(f.geom_timespan, '') = '') + AND (m.numer_timespan = f.numer_timespan OR COALESCE(f.numer_timespan, '') = '') + ), %s + , groups AS (SELECT + id, + scores.score, + numer_timespan, + dense_rank() OVER (PARTITION BY id ORDER BY numer_timespan DESC) timespan_rank, + dense_rank() OVER (PARTITION BY id ORDER BY score DESC) score_rank, + json_build_object( + 'id', id, + 'numer_id', numer_id, + 'timespan_rank', dense_rank() OVER (PARTITION BY id ORDER BY numer_timespan DESC), + 'score_rank', dense_rank() OVER (PARTITION BY id ORDER BY score DESC), + 'timespan_rownum', row_number() over + (PARTITION BY id, score ORDER BY numer_timespan DESC, Coalesce(denom_id, '')), + 'score_rownum', row_number() over + (PARTITION BY id, numer_timespan ORDER BY score DESC, Coalesce(denom_id, '')), + 'score', scores.score, + 'suggested_name', cdb_observatory.FIRST( + LOWER(TRIM(BOTH '_' FROM regexp_replace(CASE WHEN numer_id IS NOT NULL + THEN CASE + WHEN normalization ILIKE 'area%%' THEN numer_colname || ' per sq km' + WHEN normalization ILIKE 'denom%%' THEN numer_colname || ' rate' + ELSE numer_colname + END || ' ' || numer_timespan + ELSE geom_name || ' ' || geom_timespan + END, '[^a-zA-Z0-9]+', '_', 'g'))) + ), + 'numer_aggregate', cdb_observatory.FIRST(meta.numer_aggregate), + 'numer_colname', cdb_observatory.FIRST(meta.numer_colname), + 'numer_geomref_colname', cdb_observatory.FIRST(meta.numer_geomref_colname), + 'numer_tablename', cdb_observatory.FIRST(meta.numer_tablename), + 'numer_type', cdb_observatory.FIRST(meta.numer_type), + 'numer_description', cdb_observatory.FIRST(meta.numer_description), + 'numer_t_description', cdb_observatory.FIRST(meta.numer_t_description), + 'denom_aggregate', cdb_observatory.FIRST(meta.denom_aggregate), + 'denom_colname', cdb_observatory.FIRST(denom_colname), + 'denom_geomref_colname', cdb_observatory.FIRST(denom_geomref_colname), + 'denom_tablename', cdb_observatory.FIRST(denom_tablename), + 'denom_type', cdb_observatory.FIRST(meta.denom_type), + 'denom_reltype', cdb_observatory.FIRST(meta.denom_reltype), + 'denom_description', cdb_observatory.FIRST(meta.denom_description), + 'denom_t_description', cdb_observatory.FIRST(meta.denom_t_description), + 'geom_colname', cdb_observatory.FIRST(geom_colname), + 'geom_geomref_colname', cdb_observatory.FIRST(geom_geomref_colname), + 'geom_tablename', cdb_observatory.FIRST(geom_tablename), + 'geom_type', cdb_observatory.FIRST(meta.geom_type), + 'geom_timespan', cdb_observatory.FIRST(meta.geom_timespan), + 'geom_description', cdb_observatory.FIRST(meta.geom_description), + 'geom_t_description', cdb_observatory.FIRST(meta.geom_t_description), + 'numer_timespan', cdb_observatory.FIRST(numer_timespan), + 'numer_name', cdb_observatory.FIRST(numer_name), + 'denom_name', cdb_observatory.FIRST(denom_name), + 'geom_name', cdb_observatory.FIRST(geom_name), + 'normalization', cdb_observatory.FIRST(normalization), + 'max_timespan_rank', cdb_observatory.FIRST(max_timespan_rank), + 'max_score_rank', cdb_observatory.FIRST(max_score_rank), + 'target_geoms', cdb_observatory.FIRST(scores.target_geoms), + 'target_area', cdb_observatory.FIRST(scores.target_area), + 'num_geoms', cdb_observatory.FIRST(scores.numgeoms), + 'denom_id', denom_id, + 'geom_id', meta.geom_id + ) metadata + FROM meta, scores + WHERE meta.geom_id = scores.column_id + AND meta.geom_tid = scores.table_id + AND COALESCE(meta.target_geoms, 0) = COALESCE(scores.target_geoms, 0) + AND COALESCE(meta.target_area, 0) = COALESCE(scores.target_area, 0) + GROUP BY id, score, numer_id, denom_id, geom_id, numer_timespan + ) SELECT JSON_AGG(metadata ORDER BY id) + FROM groups + WHERE timespan_rank <= Coalesce((metadata->>'max_timespan_rank')::INTEGER, 'infinity'::FLOAT) + AND score_rank <= Coalesce((metadata->>'max_score_rank')::INTEGER, 1) + AND (metadata->>'timespan_rownum')::INTEGER <= $4 + AND (metadata->>'score_rownum')::INTEGER <= $5 + $string$, meta_filter_clause, scores_clause) + INTO result + USING + CASE WHEN ST_GeometryType(geom) = 'ST_Point' THEN + ST_Buffer(geom::geography, 200)::geometry(geometry, 4326) + ELSE geom + END, + target_geoms, + params, + num_timespan_options, + num_score_options, numer_filters, geom_filters + ; + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasure( + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL, + simplification NUMERIC DEFAULT 0.00001 +) +RETURNS NUMERIC +AS $$ +DECLARE + geom_type TEXT; + params JSON; + map_type TEXT; + result Numeric; + numer_aggregate TEXT; +BEGIN + IF geom IS NULL THEN + RETURN NULL; + END IF; + + IF simplification IS NOT NULL THEN + geom := ST_Simplify(geom, simplification); + END IF; + + IF ST_GeometryType(geom) = 'ST_Point' THEN + geom_type := 'point'; + ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + geom_type := 'polygon'; + geom := ST_CollectionExtract(ST_MakeValid(geom), 3); + ELSE + RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', + ST_GeometryType(geom); + END IF; + + params := (SELECT cdb_observatory.OBS_GetMeta( + geom, JSON_Build_Array(JSON_Build_Object('numer_id', measure_id, + 'geom_id', boundary_id, + 'numer_timespan', time_span + )), 1, 1, 500)); + numer_aggregate := params->0->>'numer_aggregate'; + + IF normalize ILIKE 'area%' AND numer_aggregate ILIKE 'sum' THEN + map_type := 'areaNormalized'; + ELSIF normalize ILIKE 'denom%' THEN + map_type := 'denominated'; + ELSIF normalize ILIKE 'pre%' THEN + map_type := 'predenominated'; + ELSE + -- defaults: area normalization for point if it's possible and none for + -- polygon or non-summable point + IF geom_type = 'point' AND numer_aggregate ILIKE 'sum' THEN + map_type := 'areaNormalized'; + ELSE + map_type := 'predenominated'; + END IF; + END IF; + + params := JSON_Build_Array(JSONB_Set((params::JSONB)->0, '{normalization}', to_jsonb(map_type))::JSON); + + IF params->0->>'geom_id' IS NULL THEN + RAISE NOTICE 'No boundary found for geom'; + RETURN NULL; + ELSE + RAISE NOTICE 'Using boundary %', params->0->>'geom_id'; + END IF; + + EXECUTE $query$ + SELECT (data->0->>'value')::Numeric FROM + cdb_observatory.OBS_GetData(ARRAY[($1, 1)::geomval], $2) + $query$ + INTO result + USING geom, params; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureById( + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL +) +RETURNS NUMERIC +AS $$ +DECLARE + result NUMERIC; +BEGIN + IF geom_ref IS NULL THEN + RETURN NULL; + ELSIF boundary_id IS NULL THEN + RETURN NULL; + END IF; + + EXECUTE $query$ + SELECT data->0->>'value' + FROM cdb_observatory.OBS_GetData(Array[$1], + cdb_observatory.OBS_GetMeta(ST_MakeEnvelope(-180, -90, 180, 90, 4326), + JSON_Build_Array(JSON_Build_Object( + 'numer_id', $2, + 'geom_id', $3, + 'numer_timespan', $4, + 'normalization', 'predenominated' + )))) + $query$ + INTO result + USING geom_ref, measure_id, boundary_id, time_span; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + + +-- GetData that obtains data from array of geomrefs +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetData( + geomrefs text[], + params JSON +) +RETURNS TABLE ( + id TEXT, + data JSON +) +AS $$ +DECLARE + colspecs TEXT; + tables TEXT; + obs_wheres TEXT; + user_wheres TEXT; + + q text; +BEGIN + IF params IS NULL OR JSON_ARRAY_LENGTH(params) = 0 THEN + RETURN QUERY EXECUTE $query$ SELECT NULL::TEXT, NULL::JSON LIMIT 0 $query$; + RETURN; + END IF; + + EXECUTE + $query$ + WITH _meta AS (SELECT + generate_series(1, array_length($1, 1)) colid, + (unnest($1))->>'id' id, + (unnest($1))->>'numer_id' numer_id, + (unnest($1))->>'numer_aggregate' numer_aggregate, + (unnest($1))->>'numer_colname' numer_colname, + (unnest($1))->>'numer_geomref_colname' numer_geomref_colname, + (unnest($1))->>'numer_tablename' numer_tablename, + (unnest($1))->>'numer_type' numer_type, + (unnest($1))->>'denom_id' denom_id, + (unnest($1))->>'denom_aggregate' denom_aggregate, + (unnest($1))->>'denom_colname' denom_colname, + (unnest($1))->>'denom_geomref_colname' denom_geomref_colname, + (unnest($1))->>'denom_tablename' denom_tablename, + (unnest($1))->>'denom_type' denom_type, + (unnest($1))->>'denom_reltype' denom_reltype, + (unnest($1))->>'geom_id' geom_id, + (unnest($1))->>'geom_colname' geom_colname, + (unnest($1))->>'geom_geomref_colname' geom_geomref_colname, + (unnest($1))->>'geom_tablename' geom_tablename, + (unnest($1))->>'geom_type' geom_type, + (unnest($1))->>'geom_timespan' geom_timespan, + (unnest($1))->>'numer_timespan' numer_timespan, + (unnest($1))->>'normalization' normalization, + (unnest($1))->>'api_method' api_method, + (unnest($1))->'api_args' api_args + ) + SELECT String_Agg( + -- numeric + 'JSON_Build_Object(' || CASE + WHEN api_method IS NOT NULL THEN + '''value'', ' || + 'ARRAY_AGG( ' || + api_method || '.' || numer_colname || ')::' || numer_type || '[]' + -- numeric internal values + WHEN cdb_observatory.isnumeric(numer_type) THEN + '''value'', ' || CASE + -- denominated + WHEN LOWER(normalization) LIKE 'denom%' OR (normalization IS NULL AND denom_id IS NOT NULL) + THEN 'cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || + ' / NullIf(' || denom_tablename || '.' || denom_colname || ', 0))' + -- areaNormalized + WHEN LOWER(normalization) LIKE 'area%' OR (normalization IS NULL AND numer_aggregate ILIKE 'sum') + THEN 'cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || + ' / (ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography)/1000000))' + -- prenormalized + ELSE 'cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ')' + END || ':: ' || numer_type + + -- categorical/text + WHEN LOWER(numer_type) LIKE 'text' THEN + '''value'', ' || 'cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' + + -- geometry + WHEN numer_id IS NULL THEN + '''geomref'', ' || 'cdb_observatory.FIRST(' || geom_tablename || + '.' || geom_geomref_colname || '), ' || + '''value'', ' || 'cdb_observatory.FIRST(' || geom_tablename || + '.' || geom_colname || ')' + ELSE '' + END || ')', ', ') + AS colspecs, + + (SELECT String_Agg(DISTINCT CASE + -- External API + WHEN tablename LIKE 'cdb_observatory.%' THEN + 'LATERAL (SELECT * FROM ' || tablename || ') ' || + REPLACE(split_part(tablename, '(', 1), 'cdb_observatory.', '') + -- Internal obs_ table + ELSE 'observatory.' || tablename + END, ', ') FROM ( + SELECT DISTINCT UNNEST(tablenames_ary) tablename FROM ( + SELECT ARRAY_AGG(numer_tablename) || + ARRAY_AGG(denom_tablename) || + ARRAY_AGG(geom_tablename) || + ARRAY_AGG('cdb_observatory.' || api_method || '(_geomrefs.id' || COALESCE(', ' || + (SELECT STRING_AGG(REPLACE(val::text, '"', ''''), ', ') + FROM (SELECT json_array_elements(api_args) as val) as vals), + '') || ')') + tablenames_ary + ) tablenames_inner + ) tablenames_outer) tablenames, + + String_Agg(DISTINCT array_to_string(ARRAY[ + CASE WHEN numer_tablename != geom_tablename + THEN numer_tablename || '.' || numer_geomref_colname || ' = ' || + geom_tablename || '.' || geom_geomref_colname + ELSE NULL END, + CASE WHEN numer_tablename != denom_tablename + THEN numer_tablename || '.' || numer_geomref_colname || ' = ' || + denom_tablename || '.' || denom_geomref_colname + ELSE NULL END + ], ' AND '), + ' AND ') AS obs_wheres, + + String_Agg(geom_tablename || '.' || geom_geomref_colname || ' = ' || + '_geomrefs.id', ' AND ') + AS user_wheres + FROM _meta + ; + $query$ + INTO colspecs, tables, obs_wheres, user_wheres + USING (SELECT ARRAY(SELECT json_array_elements_text(params))::json[]); + + RETURN QUERY EXECUTE format($query$ + WITH _geomrefs AS (SELECT UNNEST($1) as id) + SELECT _geomrefs.id, Array_to_JSON(ARRAY[%s]::JSON[]) + FROM _geomrefs, %s + %s + GROUP BY _geomrefs.id + ORDER BY _geomrefs.id + $query$, colspecs, tables, + 'WHERE ' || NULLIF(ARRAY_TO_STRING(ARRAY[ + Nullif(obs_wheres, ''), Nullif(user_wheres, '') + ], ' AND '), '') + ) + USING geomrefs; + RETURN; +END; +$$ LANGUAGE plpgsql STABLE; + + +-- GetData that obtains data from array of (geom, id) geomvals. +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetData( + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True +) +RETURNS TABLE ( + id INT, + data JSON +) +AS $$ +DECLARE + procgeom_clauses TEXT; + val_clauses TEXT; + json_clause TEXT; + geomtype TEXT; +BEGIN + IF params IS NULL OR JSON_ARRAY_LENGTH(params) = 0 OR ARRAY_LENGTH(geomvals, 1) IS NULL THEN + RETURN QUERY EXECUTE $query$ SELECT NULL::INT, NULL::JSON LIMIT 0 $query$; + RETURN; + END IF; + + geomtype := ST_GeometryType(geomvals[1].geom); + + /* Read metadata to generate clauses for query */ + EXECUTE $query$ + WITH _meta AS (SELECT + row_number() over () colid, * + FROM json_to_recordset($1) + AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, + numer_geomref_colname TEXT, numer_tablename TEXT, numer_type TEXT, + denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, + denom_geomref_colname TEXT, denom_tablename TEXT, denom_type TEXT, + denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, + geom_geomref_colname TEXT, geom_tablename TEXT, geom_type TEXT, + numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, + api_method TEXT, api_args JSON) + ), + + -- Generate procgeom clauses. + -- These join the users' geoms to the relevant geometries for the + -- asked-for measures in the Observatory. + _procgeom_clauses AS ( + SELECT + '_procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS (' || + CASE WHEN api_method IS NULL THEN + 'SELECT _geoms.id, ' || + CASE $3 WHEN True THEN '_geoms.geom' + ELSE geom_tablename || '.' || geom_colname + END || ' AS geom, ' || + geom_tablename || '.' || geom_geomref_colname || ' AS geomref, ' || + CASE + WHEN $2 = 'ST_Point' THEN + ' Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography), 0)/1000000 ' || + ' AS area' + -- for numeric areas, include more complex calcs + ELSE + 'CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') + THEN ST_Area(_geoms.geom) / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0) + WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) + THEN 1 + ELSE ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) / + Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0) + END pct_obs' + END || ' + FROM _geoms, observatory.' || geom_tablename || ' + WHERE ST_Intersects(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')' + -- pass through input geometries for api_method + ELSE 'SELECT _geoms.id, _geoms.geom FROM _geoms' + END || + ') ' + AS procgeom_clause + FROM _meta + GROUP BY api_method, geom_tablename, geom_geomref_colname, geom_colname + ), + + -- Generate val clauses. + -- These perform interpolations or other necessary calculations to + -- provide values according to users geometries. + _val_clauses AS ( + SELECT + '_vals_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS ( + SELECT _procgeoms.id, ' || + String_Agg('json_build_object(' || CASE + -- api-delivered values + WHEN api_method IS NOT NULL THEN + '''value'', ' || + 'ARRAY_AGG( ' || + api_method || '.' || numer_colname || ')::' || numer_type || '[]' + -- numeric internal values + WHEN cdb_observatory.isnumeric(numer_type) THEN + '''value'', ' || CASE + -- denominated + WHEN LOWER(normalization) LIKE 'denom%' + THEN CASE + WHEN denom_tablename IS NULL THEN ' NULL ' + -- denominated point-in-poly + WHEN $2 = 'ST_Point' THEN + ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || + ' / NullIf(' || denom_tablename || '.' || denom_colname || ', 0))' + -- denominated polygon interpolation + -- SUM (numer * (% OBS geom in user geom)) / SUM (denom * (% OBS geom in user geom)) + ELSE + ' SUM(' || numer_tablename || '.' || numer_colname || ' ' || + ' * _procgeoms.pct_obs ' || + ' ) / NULLIF(SUM(' || denom_tablename || '.' || denom_colname || ' ' || + ' * _procgeoms.pct_obs), 0) ' + END + -- areaNormalized + WHEN LOWER(normalization) LIKE 'area%' + THEN CASE + -- areaNormalized point-in-poly + WHEN $2 = 'ST_Point' THEN + ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || + ' / _procgeoms.area)' + -- areaNormalized polygon interpolation + -- SUM (numer * (% OBS geom in user geom)) / area of big geom + ELSE + --' NULL END ' + ' SUM(' || numer_tablename || '.' || numer_colname || ' ' || + ' * _procgeoms.pct_obs' || + ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.geom)::Geography), 0) / 1000000) ' + END + -- median/average measures with universe + WHEN LOWER(numer_aggregate) IN ('median', 'average') AND + denom_reltype ILIKE 'universe' AND LOWER(normalization) LIKE 'pre%' + THEN CASE + -- predenominated point-in-poly + WHEN $2 = 'ST_Point' THEN + ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' + ELSE + -- predenominated polygon interpolation weighted by universe + -- SUM (numer * denom * (% user geom in OBS geom)) / SUM (denom * (% user geom in OBS geom)) + -- (10 * 1000 * 1) / (1000 * 1) = 10 + -- (10 * 1000 * 1 + 50 * 10 * 1) / (1000 + 10) = 10500 / 10000 = 10.5 + ' SUM(' || numer_tablename || '.' || numer_colname || + ' * ' || denom_tablename || '.' || denom_colname || + ' * _procgeoms.pct_obs ' || + ' ) / Nullif(SUM(' || denom_tablename || '.' || denom_colname || + ' * _procgeoms.pct_obs ' || '), 0) ' + END + -- prenormalized for summable measures. point or summable only! + WHEN numer_aggregate ILIKE 'sum' AND LOWER(normalization) LIKE 'pre%' + THEN CASE + -- predenominated point-in-poly + WHEN $2 = 'ST_Point' THEN + ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' + ELSE + -- predenominated polygon interpolation + -- SUM (numer * (% user geom in OBS geom)) + ' SUM(' || numer_tablename || '.' || numer_colname || ' ' || + ' * _procgeoms.pct_obs) ' + END + -- Everything else. Point only! + ELSE CASE + WHEN $2 = 'ST_Point' THEN + ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' + ELSE + ' cdb_observatory._OBS_RaiseNotice(''Cannot perform calculation over polygon for ' || + numer_id || '/' || coalesce(denom_id, '') || '/' || geom_id || '/' || numer_timespan || ''')::Numeric ' + END + END || '::' || numer_type + + -- categorical/text + WHEN LOWER(numer_type) LIKE 'text' THEN + '''value'', ' || 'MODE() WITHIN GROUP (ORDER BY ' || numer_tablename || '.' || numer_colname || ') ' + -- geometry + WHEN numer_id IS NULL THEN + '''geomref'', _procgeoms.geomref, ' || + '''value'', ' || 'cdb_observatory.FIRST(_procgeoms.geom)::TEXT' + -- code below will return the intersection of the user's geom and the + -- OBS geom + --'''value'', ' || 'ST_Union(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || + -- '.' || geom_colname || '))::TEXT' + ELSE '' + END + || ') val_' || colid, ', ') + || ' + FROM _procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' || + Coalesce(String_Agg(DISTINCT + Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname, + ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' || + (SELECT STRING_AGG(REPLACE(val::text, '"', ''''), ', ') + FROM (SELECT JSON_Array_Elements(api_args) as val) as vals), + '') || ')) AS ' || api_method) + , ' '), '') || + CASE $3 WHEN True THEN E'\n GROUP BY _procgeoms.id ORDER BY _procgeoms.id ' + ELSE E'\n GROUP BY _procgeoms.id, _procgeoms.geomref + ORDER BY _procgeoms.id, _procgeoms.geomref' END + || ')' + AS val_clause, + '_vals_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) AS cte_name + FROM _meta + GROUP BY geom_tablename, geom_geomref_colname, geom_colname, api_method + ), + + -- Generate clauses necessary to join together val_clauses + _val_joins AS ( + SELECT String_Agg(a.cte_name || '.id = ' || b.cte_name || '.id ', ' AND ') val_joins + FROM _val_clauses a, _val_clauses b + WHERE a.cte_name != b.cte_name + AND a.cte_name < b.cte_name + ), + + -- Generate JSON clause. This puts together vals from val_clauses + _json_clause AS (SELECT + 'SELECT ' || cdb_observatory.FIRST(cte_name) || '.id::INT, + Array_to_JSON(ARRAY[' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || ']) + FROM ' || String_Agg(cte_name, ', ') || + Coalesce(' WHERE ' || val_joins, '') + AS json_clause + FROM _val_clauses, _val_joins + GROUP BY val_joins + ) + + SELECT (SELECT String_Agg(procgeom_clause, E',\n ') FROM _procgeom_clauses), + (SELECT String_Agg(val_clause, E',\n ') FROM _val_clauses), + json_clause + FROM _json_clause + $query$ INTO + procgeom_clauses, + val_clauses, + json_clause + USING params, geomtype, merge; + + /* Execute query */ + RETURN QUERY EXECUTE format($query$ + WITH _raw_geoms AS (%s), + _geoms AS (SELECT id, + CASE WHEN (ST_NPoints(geom) > 1000) + THEN ST_CollectionExtract(ST_MakeValid(ST_SimplifyVW(geom, 0.00001)), 3) + ELSE geom END geom + FROM _raw_geoms), + -- procgeom_clauses + %s, + + -- val_clauses + %s + + -- json_clause + %s + $query$, CASE WHEN ARRAY_LENGTH(geomvals, 1) = 1 + THEN ' SELECT $1[1].val as id, $1[1].geom as geom ' + ELSE ' SELECT val as id, geom FROM UNNEST($1) ' + END, + String_Agg(procgeom_clauses, E',\n '), + String_Agg(val_clauses, E',\n '), + json_clause) + USING geomvals; + RETURN; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetCategory( + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL, + simplification NUMERIC DEFAULT 0.00001 +) +RETURNS TEXT +AS $$ +DECLARE + geom_type TEXT; + params JSON; + map_type TEXT; + result TEXT; +BEGIN + IF geom IS NULL THEN + RETURN NULL; + END IF; + + IF simplification IS NOT NULL THEN + geom := ST_Simplify(geom, simplification); + END IF; + + IF ST_GeometryType(geom) = 'ST_Point' THEN + geom_type := 'point'; + ELSIF ST_GeometryType(geom) IN ('ST_Polygon', 'ST_MultiPolygon') THEN + geom_type := 'polygon'; + geom := ST_CollectionExtract(ST_MakeValid(geom), 3); + ELSE + RAISE EXCEPTION 'Invalid geometry type (%), can only handle ''ST_Point'', ''ST_Polygon'', and ''ST_MultiPolygon''', + ST_GeometryType(geom); + END IF; + + params := (SELECT cdb_observatory.OBS_GetMeta( + geom, JSON_Build_Array(JSON_Build_Object('numer_id', category_id, + 'geom_id', boundary_id, + 'numer_timespan', time_span + )), 1, 1, 500)); + + IF params->0->>'geom_id' IS NULL THEN + RAISE NOTICE 'No boundary found for geom'; + RETURN NULL; + ELSE + RAISE NOTICE 'Using boundary %', params->0->>'geom_id'; + END IF; + + EXECUTE $query$ + SELECT data->0->>'value' FROM + cdb_observatory.OBS_GetData(ARRAY[($1, 1)::geomval], $2) + $query$ + INTO result + USING geom, params; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetUSCensusMeasure( + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL + ) +RETURNS NUMERIC AS $$ +DECLARE + standardized_name text; + measure_id text; + result Numeric; +BEGIN + standardized_name = cdb_observatory._OBS_StandardizeMeasureName(name); + + EXECUTE $string$ + SELECT c.id + FROM observatory.obs_column c + JOIN observatory.obs_column_tag ct + ON c.id = ct.column_id + WHERE cdb_observatory._OBS_StandardizeMeasureName(c.name) = $1 + AND ct.tag_id ILIKE 'us.census%' + $string$ + INTO measure_id + USING standardized_name; + + EXECUTE 'SELECT cdb_observatory.OBS_GetMeasure($1, $2, $3, $4, $5)' + INTO result + USING geom, measure_id, normalize, boundary_id, time_span; + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetUSCensusCategory( + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL + ) +RETURNS TEXT AS $$ +DECLARE + standardized_name TEXT; + category_id TEXT; + result TEXT; +BEGIN + standardized_name = cdb_observatory._OBS_StandardizeMeasureName(name); + + EXECUTE $string$ + SELECT c.id + FROM observatory.obs_column c + --JOIN observatory.obs_column_tag ct + -- ON c.id = ct.column_id + WHERE cdb_observatory._OBS_StandardizeMeasureName(c.name) = $1 + AND c.type ILIKE 'TEXT' + AND c.id ILIKE 'us.census%' -- TODO this should be done by tag + --AND ct.tag_id = 'us.census.acs.demographics' + $string$ + INTO category_id + USING standardized_name; + + EXECUTE 'SELECT cdb_observatory.OBS_GetCategory($1, $2, $3, $4)' + INTO result + USING geom, category_id, boundary_id, time_span; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetPopulation( + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT NULL, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL +) +RETURNS NUMERIC +AS $$ +DECLARE + population_measure_id TEXT; + result Numeric; +BEGIN + -- TODO use a super-column for global pop + population_measure_id := 'us.census.acs.B01003001'; + + EXECUTE $query$ SELECT cdb_observatory.OBS_GetMeasure( + $1, $2, $3, $4, $5 + ) LIMIT 1 + $query$ + INTO result + USING geom, population_measure_id, normalize, boundary_id, time_span; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetSegmentSnapshot( + geom geometry(Geometry, 4326), + boundary_id text DEFAULT NULL +) +RETURNS JSON +AS $$ +DECLARE + meta JSON; + data JSON; + result JSON; +BEGIN + boundary_id = COALESCE(boundary_id, 'us.census.tiger.census_tract'); + + EXECUTE $query$ + SELECT cdb_observatory.OBS_GetMeta($1, ('[ ' || + '{"numer_id": "us.census.acs.B01003001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B01001002_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B01001026_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B01002001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B03002003_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B03002004_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B03002006_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B03002012_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B05001006_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08006001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08006002_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08301010_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08006009_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08006011_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08006015_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B08006017_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B09001001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B11001001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B14001001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B14001002_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B14001005_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B14001006_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B14001007_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B14001008_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B15003001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B15003017_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B15003022_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B15003023_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B16001001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B16001002_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B16001003_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B17001001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B17001002_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B19013001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B19083001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B19301001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25001001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25002003_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25004002_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25004004_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25058001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25071001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25075001_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.acs.B25075025_quantile", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.spielman_singleton_segments.X10", "geom_id": ' || $2 || '},' || + '{"numer_id": "us.census.spielman_singleton_segments.X55", "geom_id": ' || $2 || '}' || + ']')::JSON) + $query$ + INTO meta + USING geom, COALESCE('"' || boundary_id || '"', 'null'); + + EXECUTE $query$ + SELECT data FROM cdb_observatory.OBS_GetData( + ARRAY[($1, 1)::geomval], $2) + $query$ + INTO data + USING geom, meta; + + EXECUTE $query$ + WITH els AS (SELECT + REPLACE(REPLACE(JSON_Array_Elements($1)->>'numer_id', + 'us.census.spielman_singleton_segments.X55', 'x55_segment'), + 'us.census.spielman_singleton_segments.X10', 'x10_segment') k, + JSON_Array_Elements($2)->>'value' v) + SELECT JSON_Object_Agg(k, v) FROM els + $query$ + INTO result + USING meta, data; + + RETURN result; +END; +$$ LANGUAGE plpgsql STABLE; + +-- TODO: implement search for timespan + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_SearchTables( + search_term text, + time_span text DEFAULT NULL +) +RETURNS table(tablename text, timespan text) +As $$ +DECLARE + out_var text[]; +BEGIN + + IF time_span IS NULL + THEN + RETURN QUERY + EXECUTE + 'SELECT tablename::text, timespan::text + FROM observatory.obs_table t + JOIN observatory.obs_column_table ct + ON ct.table_id = t.id + JOIN observatory.obs_column c + ON ct.column_id = c.id + WHERE c.type ILIKE ''geometry'' + AND c.id = $1' + USING search_term; + RETURN; + ELSE + RETURN QUERY + EXECUTE + 'SELECT tablename::text, timespan::text + FROM observatory.obs_table t + JOIN observatory.obs_column_table ct + ON ct.table_id = t.id + JOIN observatory.obs_column c + ON ct.column_id = c.id + WHERE c.type ILIKE ''geometry'' + AND c.id = $1 + AND t.timespan = $2' + USING search_term, time_span; + RETURN; + END IF; + +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +-- Functions used to search the observatory for measures +-------------------------------------------------------------------------------- +-- TODO allow the user to specify the boundary to search for measures +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_Search( + search_term text, + relevant_boundary text DEFAULT null +) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) as $$ +DECLARE + boundary_term text; +BEGIN + IF relevant_boundary then + boundary_term = ''; + else + boundary_term = ''; + END IF; + + RETURN QUERY + EXECUTE format($string$ + SELECT id::text, description::text, + name::text, + aggregate::text, + NULL::TEXT source -- TODO use tags + FROM observatory.OBS_column + where name ilike '%%' || %L || '%%' + or description ilike '%%' || %L || '%%' + %s + $string$, search_term, search_term,boundary_term); + RETURN; +END +$$ LANGUAGE plpgsql; + + +-- Functions to return the geometry levels that a point is part of +-------------------------------------------------------------------------------- +-- TODO add test response + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableBoundaries( + geom geometry(Geometry, 4326), + timespan text DEFAULT null) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) as $$ +DECLARE + timespan_query TEXT DEFAULT ''; +BEGIN + + IF timespan != NULL + THEN + timespan_query = format('AND timespan = %L', timespan); + END IF; + + RETURN QUERY + EXECUTE + $string$ + SELECT + column_id::text As column_id, + obs_column.description::text As description, + timespan::text As timespan, + tablename::text As tablename + FROM + observatory.OBS_table, + observatory.OBS_column_table, + observatory.OBS_column + WHERE + observatory.OBS_column_table.column_id = observatory.obs_column.id AND + observatory.OBS_column_table.table_id = observatory.obs_table.id + AND + observatory.OBS_column.type = 'Geometry' + AND + ST_Intersects($1, st_setsrid(observatory.obs_table.the_geom, 4326)) + $string$ || timespan_query + USING geom; + RETURN; +END +$$ LANGUAGE plpgsql; + +-- Functions the interface works from to identify available numerators, +-- denominators, geometries, and timespans + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableNumerators( + bounds GEOMETRY DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL +) RETURNS TABLE ( + numer_id TEXT, + numer_name TEXT, + numer_description TEXT, + numer_weight NUMERIC, + numer_license TEXT, + numer_source TEXT, + numer_type TEXT, + numer_aggregate TEXT, + numer_extra JSONB, + numer_tags JSONB, + valid_denom BOOLEAN, + valid_geom BOOLEAN, + valid_timespan BOOLEAN +) AS $$ +DECLARE + geom_clause TEXT; +BEGIN + filter_tags := COALESCE(filter_tags, (ARRAY[])::TEXT[]); + denom_id := COALESCE(denom_id, ''); + geom_id := COALESCE(geom_id, ''); + timespan := COALESCE(timespan, ''); + IF bounds IS NULL THEN + geom_clause := ''; + ELSE + geom_clause := 'ST_Intersects(the_geom, $5) AND'; + END IF; + RETURN QUERY + EXECUTE + format($string$ + SELECT numer_id::TEXT, + numer_name::TEXT, + numer_description::TEXT, + numer_weight::NUMERIC, + NULL::TEXT license, + NULL::TEXT source, + numer_type numer_type, + numer_aggregate numer_aggregate, + numer_extra::JSONB numer_extra, + numer_tags numer_tags, + $1 = ANY(denoms) valid_denom, + $2 = ANY(geoms) valid_geom, + $3 = ANY(timespans) valid_timespan + FROM observatory.obs_meta_numer + WHERE %s (numer_tags ?& $4 OR CARDINALITY($4) = 0) + $string$, geom_clause) + USING denom_id, geom_id, timespan, filter_tags, bounds; + RETURN; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetNumerators( + bounds GEOMETRY DEFAULT NULL, + section_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + other_tags TEXT[] DEFAULT ARRAY[]::TEXT[], + ids TEXT[] DEFAULT ARRAY[]::TEXT[], + name TEXT DEFAULT NULL, + denom_id TEXT DEFAULT '', + geom_id TEXT DEFAULT '', + timespan TEXT DEFAULT '' +) RETURNS TABLE ( + numer_id TEXT, + numer_name TEXT, + numer_description TEXT, + numer_weight NUMERIC, + numer_license TEXT, + numer_source TEXT, + numer_type TEXT, + numer_aggregate TEXT, + numer_extra JSONB, + numer_tags JSONB, + valid_denom BOOLEAN, + valid_geom BOOLEAN, + valid_timespan BOOLEAN +) AS $$ +DECLARE + where_clause_elements TEXT[]; + geom_clause TEXT; + where_clause TEXT; +BEGIN + where_clause_elements := (ARRAY[])::TEXT[]; + where_clause := ''; + + IF bounds IS NOT NULL THEN + where_clause_elements := array_append(where_clause_elements, format($data$ST_Intersects(the_geom, '%s'::geometry)$data$, bounds)); + END IF; + IF cardinality(section_tags) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, section_tags)); + END IF; + IF cardinality(subsection_tags) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, subsection_tags)); + END IF; + IF cardinality(other_tags) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, other_tags)); + END IF; + IF cardinality(ids) > 0 THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_id IN (array_to_string('%s'::text[], ','))$data$, ids)); + END IF; + IF name IS NOT NULL AND name != '' THEN + where_clause_elements := array_append(where_clause_elements, format($data$numer_name ilike '%%%s%%'$data$, name)); + END IF; + IF cardinality(where_clause_elements) > 0 THEN + where_clause := format($clause$WHERE %s$clause$, array_to_string(where_clause_elements, ' AND ')); + END IF; + RAISE DEBUG '%', array_to_string(where_clause_elements, ' AND '); + + RETURN QUERY + EXECUTE + format($string$ + SELECT numer_id::TEXT, + numer_name::TEXT, + numer_description::TEXT, + numer_weight::NUMERIC, + NULL::TEXT license, + NULL::TEXT source, + numer_type numer_type, + numer_aggregate numer_aggregate, + numer_extra::JSONB numer_extra, + numer_tags numer_tags, + $1 = ANY(denoms) valid_denom, + $2 = ANY(geoms) valid_geom, + $3 = ANY(timespans) valid_timespan + FROM observatory.obs_meta_numer + %s + $string$, where_clause) + USING denom_id, geom_id, timespan; + RETURN; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableDenominators( + bounds GEOMETRY DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL +) RETURNS TABLE ( + denom_id TEXT, + denom_name TEXT, + denom_description TEXT, + denom_weight NUMERIC, + denom_license TEXT, + denom_source TEXT, + denom_type TEXT, + denom_aggregate TEXT, + denom_extra JSONB, + denom_tags JSONB, + valid_numer BOOLEAN, + valid_geom BOOLEAN, + valid_timespan BOOLEAN +) AS $$ +DECLARE + geom_clause TEXT; +BEGIN + filter_tags := COALESCE(filter_tags, (ARRAY[])::TEXT[]); + numer_id := COALESCE(numer_id, ''); + geom_id := COALESCE(geom_id, ''); + timespan := COALESCE(timespan, ''); + IF bounds IS NULL THEN + geom_clause := ''; + ELSE + geom_clause := 'ST_Intersects(the_geom, $5) AND'; + END IF; + RETURN QUERY + EXECUTE + format($string$ + SELECT denom_id::TEXT, + denom_name::TEXT, + denom_description::TEXT, + denom_weight::NUMERIC, + NULL::TEXT license, + NULL::TEXT source, + denom_type::TEXT, + denom_aggregate::TEXT, + denom_extra::JSONB, + denom_tags::JSONB, + $1 = ANY(numers) valid_numer, + $2 = ANY(geoms) valid_geom, + $3 = ANY(timespans) valid_timespan + FROM observatory.obs_meta_denom + WHERE %s (denom_tags ?& $4 OR CARDINALITY($4) = 0) + $string$, geom_clause) + USING numer_id, geom_id, timespan, filter_tags, bounds; + RETURN; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableGeometries( + bounds GEOMETRY DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + timespan TEXT DEFAULT NULL +) RETURNS TABLE ( + geom_id TEXT, + geom_name TEXT, + geom_description TEXT, + geom_weight NUMERIC, + geom_aggregate TEXT, + geom_license TEXT, + geom_source TEXT, + geom_type TEXT, + geom_extra JSONB, + geom_tags JSONB, + valid_numer BOOLEAN, + valid_denom BOOLEAN, + valid_timespan BOOLEAN, + score NUMERIC, + numtiles BIGINT, + notnull_percent NUMERIC, + numgeoms NUMERIC, + percentfill NUMERIC, + estnumgeoms NUMERIC, + meanmediansize NUMERIC +) AS $$ +DECLARE + geom_clause TEXT; +BEGIN + filter_tags := COALESCE(filter_tags, (ARRAY[])::TEXT[]); + numer_id := COALESCE(numer_id, ''); + denom_id := COALESCE(denom_id, ''); + timespan := COALESCE(timespan, ''); + IF bounds IS NULL THEN + geom_clause := ''; + ELSE + geom_clause := 'ST_Intersects(the_geom, $5) AND'; + END IF; + RETURN QUERY + EXECUTE + format($string$ + WITH available_geoms AS ( + SELECT geom_id::TEXT, + geom_name::TEXT, + geom_description::TEXT, + geom_weight::NUMERIC, + NULL::TEXT geom_aggregate, + NULL::TEXT license, + NULL::TEXT source, + geom_type::TEXT, + geom_extra::JSONB, + geom_tags::JSONB, + $1 = ANY(numers) valid_numer, + $2 = ANY(denoms) valid_denom, + CASE WHEN $3 IS NOT NULL AND $3 != '' THEN + -- Here we are looking for geometries with: a) geometry timespan or b) numerators linked to that geometries that fit in the + -- timespan passed. For example it look for geometries with timespan '2015 - 2015' or numerators linked to that geometry that has + -- '2015 - 2015' as one of the valid timespans. + -- If we pass a numerator_id, we filter by that numerator + CASE WHEN $1 IS NOT NULL AND $1 != '' THEN + EXISTS (SELECT 1 FROM observatory.obs_meta_geom_numer_timespan onu WHERE o.geom_id = onu.geom_id AND onu.numer_id = $1 AND ($3 = ANY(onu.timespans) OR $3 IN (select(unnest(o.timespans))))) + ELSE + EXISTS (SELECT 1 FROM observatory.obs_meta_geom_numer_timespan onu WHERE o.geom_id = onu.geom_id AND ($3 = ANY(onu.timespans) OR $3 IN (select(unnest(o.timespans))))) + END + ELSE + false + END as valid_timespan + FROM observatory.obs_meta_geom o + WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0) + ), scores AS ( + SELECT * FROM cdb_observatory._OBS_GetGeometryScores($5, + (SELECT ARRAY_AGG(geom_id) FROM available_geoms) + ) + ) SELECT DISTINCT ON (geom_id) available_geoms.*, score, numtiles, notnull_percent, numgeoms, + percentfill, estnumgeoms, meanmediansize + FROM available_geoms, scores + WHERE available_geoms.geom_id = scores.column_id + $string$, geom_clause) + USING numer_id, denom_id, timespan, filter_tags, bounds; + RETURN; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableTimespans( + bounds GEOMETRY DEFAULT NULL, + filter_tags TEXT[] DEFAULT NULL, + numer_id TEXT DEFAULT NULL, + denom_id TEXT DEFAULT NULL, + geom_id TEXT DEFAULT NULL +) RETURNS TABLE ( + timespan_id TEXT, + timespan_name TEXT, + timespan_description TEXT, + timespan_weight NUMERIC, + timespan_aggregate TEXT, + timespan_license TEXT, + timespan_source TEXT, + timespan_type TEXT, + timespan_extra JSONB, + timespan_tags JSONB, + valid_numer BOOLEAN, + valid_denom BOOLEAN, + valid_geom BOOLEAN +) AS $$ +DECLARE + geom_clause TEXT; +BEGIN + filter_tags := COALESCE(filter_tags, (ARRAY[])::TEXT[]); + numer_id := COALESCE(numer_id, ''); + denom_id := COALESCE(denom_id, ''); + geom_id := COALESCE(geom_id, ''); + IF bounds IS NULL THEN + geom_clause := ''; + ELSE + geom_clause := 'ST_Intersects(the_geom, $5) AND'; + END IF; + RETURN QUERY + EXECUTE + format($string$ + SELECT timespan_id::TEXT, + timespan_name::TEXT, + timespan_description::TEXT, + timespan_weight::NUMERIC, + NULL::TEXT timespan_aggregate, + NULL::TEXT timespan_license, + NULL::TEXT timespan_source, + NULL::TEXT timespan_type, + NULL::JSONB timespan_extra, + NULL::JSONB timespan_tags, + $1 = ANY(numers) valid_numer, + $2 = ANY(denoms) valid_denom, + $3 = ANY(geoms) valid_geom_id + FROM observatory.obs_meta_timespan + WHERE %s (timespan_tags ?& $4 OR CARDINALITY($4) = 0) + $string$, geom_clause) + USING numer_id, denom_id, geom_id, filter_tags, bounds; + RETURN; +END +$$ LANGUAGE plpgsql; + + +-- Function below should replace SQL in +-- https://github.com/CartoDB/cartodb/blob/ab465cb2918c917940e955963b0cd8a050c06600/lib/assets/javascripts/cartodb3/editor/layers/layer-content-views/analyses/data-observatory-metadata.js +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_LegacyBuilderMetadata( + aggregate_type TEXT DEFAULT NULL +) +RETURNS TABLE ( + name TEXT, + subsection JSONB +) AS $$ +DECLARE + aggregate_condition TEXT DEFAULT ''; +BEGIN + IF LOWER(aggregate_type) ILIKE 'sum' THEN + aggregate_condition := ' AND numer_aggregate IN (''sum'', ''median'', ''average'') '; + ELSIF aggregate_type IS NOT NULL THEN + aggregate_condition := format(' AND numer_aggregate ILIKE %L ', aggregate_type); + END IF; + RETURN QUERY + EXECUTE format($string$ + WITH expanded AS ( + SELECT JSONB_Build_Object('id', numer_id, 'name', numer_name) "column", + SUBSTR((sections).key, 9) section_id, (sections).value section_name, + SUBSTR((subsections).key, 12) subsection_id, (subsections).value subsection_name + FROM ( + SELECT numer_id, numer_name, + jsonb_each_text(numer_tags) as sections, + jsonb_each_text as subsections + FROM (SELECT numer_id, numer_name, numer_tags, + jsonb_each_text(numer_tags) + FROM cdb_observatory.obs_getavailablenumerators() + WHERE numer_weight > 0 %s + ) foo + ) bar + WHERE (sections).key LIKE 'section/%%' + AND (subsections).key LIKE 'subsection/%%' + ), grouped_by_subsections AS ( + SELECT JSONB_Agg(JSONB_Build_Object('f1', "column")) AS columns, + section_id, section_name, subsection_id, subsection_name + FROM expanded + GROUP BY section_id, section_name, subsection_id, subsection_name + ) + SELECT section_name as name, JSONB_Agg( + JSONB_Build_Object( + 'f1', JSONB_Build_Object( + 'name', subsection_name, + 'id', subsection_id, + 'columns', columns + ) + ) + ) as subsection + FROM grouped_by_subsections + GROUP BY section_name + $string$, aggregate_condition); + RETURN; +END +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetGeometryScores( + bounds Geometry(Geometry, 4326) DEFAULT NULL, + filter_geom_ids TEXT[] DEFAULT NULL, + desired_num_geoms INTEGER DEFAULT NULL, + desired_area NUMERIC DEFAULT NULL +) RETURNS TABLE ( + score NUMERIC, + numtiles BIGINT, + table_id TEXT, + column_id TEXT, + notnull_percent NUMERIC, + numgeoms NUMERIC, + percentfill NUMERIC, + estnumgeoms NUMERIC, + meanmediansize NUMERIC +) AS $$ +DECLARE + num_geoms_multiplier Numeric; +BEGIN + IF desired_num_geoms IS NULL THEN + desired_num_geoms := 3000; + END IF; + filter_geom_ids := COALESCE(filter_geom_ids, (ARRAY[])::TEXT[]); + -- Very complex geometries simply fail. For a boundary check, we can + -- comfortably get away with the simplicity of an envelope + IF ST_Npoints(bounds) > 10000 THEN + bounds := ST_Envelope(bounds); + END IF; + IF desired_area IS NULL THEN + desired_area := ST_Area(bounds); + END IF; + + -- In case of points, desired_area will be 0. We still want an accurate + -- estimate of numgeoms in that case. + IF desired_area = 0 THEN + num_geoms_multiplier := 1; + ELSE + num_geoms_multiplier := Coalesce(desired_area / Nullif(ST_Area(bounds), 0), 1); + END IF; + + RETURN QUERY + EXECUTE $string$ + WITH clipped_geom AS ( + SELECT column_id, table_id + , CASE WHEN $1 IS NOT NULL THEN ST_Clip(tile, $1, True) -- -20 + ELSE tile END clipped_tile + , tile + FROM observatory.obs_column_table_tile_simple + WHERE ($1 IS NULL OR ST_Intersects($1, tile)) + AND (column_id = ANY($2) OR cardinality($2) = 0) + ), clipped_geom_countagg AS ( + SELECT column_id, table_id + , BOOL_AND(ST_BandIsNoData(clipped_tile, 1)) nodata + FROM clipped_geom + GROUP BY column_id, table_id + ), clipped_geom_reagg AS ( + SELECT COUNT(*)::BIGINT cnt, a.column_id, a.table_id, + cdb_observatory.FIRST(nodata) first_nodata, + cdb_observatory.FIRST(tile) first_tile, + (ST_SummaryStatsAgg(clipped_tile, 1, False)).sum::Numeric sum_geoms, -- ND + (ST_SummaryStatsAgg(clipped_tile, 2, False)).mean::Numeric / 255 mean_fill --ND + FROM clipped_geom_countagg a, clipped_geom b + WHERE a.table_id = b.table_id + AND a.column_id = b.column_id + GROUP BY a.column_id, a.table_id + ), final AS ( + SELECT + cnt, table_id, column_id + , NULL::Numeric AS notnull_percent + , (CASE WHEN first_nodata IS FALSE + THEN sum_geoms + ELSE COALESCE(ST_Value(first_tile, 1, ST_PointOnSurface($1)), 0) + * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(first_tile, 0, 0))) + END)::Numeric * $4 + AS numgeoms + , (CASE WHEN first_nodata IS FALSE + THEN mean_fill + ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2 + END)::Numeric + AS percentfill + , null::numeric estnumgeoms + , null::numeric meanmediansize + FROM clipped_geom_reagg + ) SELECT + ((100.0 / (1+abs(log(0.0001 + $3) - log(0.0001 + numgeoms::Numeric)))) * percentfill)::Numeric + AS score, * + FROM final + $string$ USING bounds, filter_geom_ids, desired_num_geoms, num_geoms_multiplier; + RETURN; +END +$$ LANGUAGE plpgsql IMMUTABLE; +-- Data Observatory -- Welcome to the Future +-- These Data Observatory functions provide access to boundary polyons (and +-- their ids) such as those available through the US Census Tiger, Who's on +-- First, the Spanish Census, and so on + + +-- OBS_GetBoundary +-- +-- Returns the boundary polygon(s) that overlap with the input point geometry. +-- From an input point geometry, find the boundary which intersects with the +-- centroid of the input geometry +-- Inputs: +-- geom geometry: input point geometry +-- boundary_id text: source id of boundaries +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- boundary geometry: geometry boundary that intersects with geom, is at the +-- resolution requested with boundary_id, and time_span +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundary( + geom geometry(Point, 4326), + boundary_id text, + time_span text DEFAULT NULL) +RETURNS geometry(Geometry, 4326) +AS $$ +DECLARE + boundary geometry(Geometry, 4326); + target_table text; +BEGIN + + -- TODO: Check if SRID = 4326, if not transform? + + -- if not a point, raise error + IF ST_GeometryType(geom) != 'ST_Point' + THEN + RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_Point''', ST_GeometryType(geom); + END IF; + + -- return the first boundary in intersections + EXECUTE $query$ + SELECT * FROM cdb_observatory._OBS_GetBoundariesByGeometry($1, $2, $3) LIMIT 1 + $query$ INTO boundary + USING geom, boundary_id, time_span; + + RETURN boundary; + +END; +$$ LANGUAGE plpgsql; + +-- OBS_GetBoundaryId +-- +-- retrieves the boundary identifier (e.g., '36047' = Kings County/Brooklyn, NY) +-- corresponding to the location geom and boundary types (e.g., +-- us.census.tiger.county) + +-- Inputs: +-- geom geometry: location where the boundary is requested to overlap with +-- boundary_id text: source id of boundaries (e.g., us.census.tiger.county) +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- geometry_id text: identifier of the geometry which overlaps with the input +-- point geom in the table corresponding to boundary_id and +-- time_span +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundaryId( + geom geometry(Point, 4326), + boundary_id text, + time_span text DEFAULT NULL +) +RETURNS text +AS $$ +DECLARE + result TEXT; +BEGIN + + EXECUTE $query$ + SELECT geom_refs FROM cdb_observatory._OBS_GetBoundariesByGeometry( + $1, $2, $3) LIMIT 1 + $query$ + INTO result + USING geom, boundary_id, time_span; + + RETURN result; +END; +$$ LANGUAGE plpgsql; + + +-- OBS_GetBoundaryById +-- +-- Given a geometry reference (e.g., geoid for US Census), and it's geometry +-- level (see OBS_ListGeomColumns() for all available boundary ids), give back +-- the boundary that corresponds to that geometry_id, boundary_id, and +-- time_span + +-- Inputs: +-- geometry_id text: geometry id of the requested boundary +-- boundary_id text: source id of boundaries (e.g., us.census.tiger.county) +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- boundary geometry: geometry boundary that matches geometry_id, is at the +-- resolution requested with boundary_id, and time_span +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundaryById( + geometry_id text, -- ex: '36047' + boundary_id text, -- ex: 'us.census.tiger.county' + time_span text DEFAULT NULL -- ex: '2009' +) +RETURNS geometry(geometry, 4326) +AS $$ +DECLARE + result GEOMETRY; +BEGIN + + EXECUTE $query$ + SELECT (data->0->>'value')::Geometry + FROM cdb_observatory.OBS_GetData( + ARRAY[$1], + cdb_observatory.OBS_GetMeta( + ST_MakeEnvelope(-180, -90, 180, 90, 4326), + ('[{"geom_id": "' || $2 || '"}]')::JSON)) + $query$ + INTO result + USING geometry_id, boundary_id; + + RETURN result; +END; +$$ LANGUAGE plpgsql; + +-- _OBS_GetBoundariesByGeometry +-- internal function for retrieving geometries based on an input geometry +-- see OBS_GetBoundariesByGeometry or OBS_GetBoundariesByPointAndRadius for +-- more information + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetBoundariesByGeometry( + geom geometry(Geometry, 4326), + boundary_id text, + time_span text DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE ( + the_geom geometry, + geom_refs text +) AS $$ +DECLARE + meta JSON; +BEGIN + overlap_type := COALESCE(overlap_type, 'intersects'); + -- check inputs + IF lower(overlap_type) NOT IN ('contains', 'intersects', 'within') + THEN + -- recognized overlap type (map to ST_Contains, ST_Intersects, and ST_Within) + RAISE EXCEPTION 'Overlap type ''%'' is not an accepted type (choose intersects, within, or contains)', overlap_type; + END IF; + + EXECUTE $query$ + SELECT cdb_observatory.OBS_GetMeta($1, JSON_Build_Array(JSON_Build_Object( + 'geom_id', $2, 'geom_timespan', $3))) + $query$ + INTO meta + USING geom, boundary_id, time_span; + + IF meta->0->>'geom_id' IS NULL THEN + RETURN QUERY EXECUTE 'SELECT NULL::Geometry, NULL::Text LIMIT 0'; + RETURN; + END IF; + + -- return first boundary in intersections + RETURN QUERY EXECUTE $query$ + SELECT (data->0->>'value')::Geometry the_geom, data->0->>'geomref' geom_refs + FROM cdb_observatory.OBS_GetData( + ARRAY[($1, 1)::geomval], $2, False + ) + $query$ USING geom, meta; + RETURN; + +END; +$$ LANGUAGE plpgsql; + +-- OBS_GetBoundariesByGeometry +-- +-- Given a bounding box (or a polygon), and it's geometry level (see +-- OBS_ListGeomColumns() for all available boundary ids), give back the +-- boundaries that are contained within the bounding box polygon and the +-- associated geometry ids + +-- Inputs: +-- geom geometry: bounding box (or polygon) of the region of interest +-- boundary_id text: source id of boundaries (e.g., us.census.tiger.county) +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- table with the following columns +-- boundary geometry: geometry boundary that is contained within the input +-- bounding box at the requested geometry level +-- with boundary_id, and time_span +-- geom_refs text: geometry identifiers (e.g., geoid for the US Census) +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundariesByGeometry( + geom geometry(Geometry, 4326), + boundary_id text, + time_span text DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) +AS $$ +BEGIN + + RETURN QUERY SELECT * + FROM cdb_observatory._OBS_GetBoundariesByGeometry( + geom, + boundary_id, + time_span, + overlap_type + ); + RETURN; + +END; +$$ LANGUAGE plpgsql; + +-- OBS_GetBoundariesByPointAndRadius +-- +-- Given a point and radius, and it's geometry level (see +-- OBS_ListGeomColumns() for all available boundary ids), give back the +-- boundaries that are contained within the point buffered by radius meters and +-- the associated geometry ids + +-- Inputs: +-- geom geometry: point geometry centered on area of interest +-- radius numeric: radius (in meters) of a circle centered on geom for +-- selecting polygons +-- boundary_id text: source id of boundaries (e.g., us.census.tiger.county) +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- table with the following columns +-- boundary geometry: geometry boundary that is contained within the input +-- bounding box at the requested geometry level +-- with boundary_id, and time_span +-- geom_refs text: geometry identifiers (e.g., geoid for the US Census) +-- +-- TODO: move to ST_DWithin instead of buffer + intersects? +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetBoundariesByPointAndRadius( + geom geometry(Point, 4326), -- point + radius numeric, -- radius in meters + boundary_id text, + time_span text DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) +AS $$ +DECLARE + circle_boundary geometry(Geometry, 4326); +BEGIN + + IF ST_GeometryType(geom) != 'ST_Point' + THEN + RAISE EXCEPTION 'Input geometry ''%'' is not a point', ST_AsText(geom); + ELSE + circle_boundary := ST_Buffer(geom::geography, radius)::geometry; + END IF; + + RETURN QUERY SELECT * + FROM cdb_observatory._OBS_GetBoundariesByGeometry( + circle_boundary, + boundary_id, + time_span, + overlap_type); + RETURN; +END; +$$ LANGUAGE plpgsql; + +-- _OBS_GetPointsByGeometry + + +CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetPointsByGeometry( + geom geometry(Geometry, 4326), + boundary_id text, + time_span text DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) +AS $$ +DECLARE + boundary geometry(Geometry, 4326); + geom_colname text; + geoid_colname text; + target_table text; +BEGIN + overlap_type := COALESCE(overlap_type, 'intersects'); + + IF lower(overlap_type) NOT IN ('contains', 'within', 'intersects') + THEN + RAISE EXCEPTION 'Overlap type ''%'' is not an accepted type (choose intersects, within, or contains)', overlap_type; + ELSIF ST_GeometryType(geom) NOT IN ('ST_Polygon', 'ST_MultiPolygon') + THEN + RAISE EXCEPTION 'Invalid geometry type (%), expecting ''ST_MultiPolygon'' or ''ST_Polygon''', ST_GeometryType(geom); + END IF; + + -- return first boundary in intersections + RETURN QUERY EXECUTE $query$ + SELECT ST_PointOnSurface(the_geom), geom_refs + FROM cdb_observatory._OBS_GetBoundariesByGeometry($1, $2) + $query$ USING geom, boundary_id; + RETURN; + +END; +$$ LANGUAGE plpgsql; + +-- OBS_GetPointsByGeometry +-- +-- Given a polygon, and it's geometry level (see +-- OBS_ListGeomColumns() for all available boundary ids), give back a point +-- which lies in a boundary from the requested geometry level that is contained +-- within the bounding box polygon and the associated geometry ids +-- +-- Inputs: +-- geom geometry: bounding box (or polygon) of the region of interest +-- boundary_id text: source id of boundaries (e.g., us.census.tiger.county) +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- table with the following columns +-- boundary geometry: point that lies on a boundary that is contained within +-- the input bounding box at the requested geometry +-- level with boundary_id, and time_span +-- geom_refs text: geometry identifiers (e.g., geoid for the US Census) +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetPointsByGeometry( + geom geometry(Geometry, 4326), + boundary_id text, + time_span text DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) +AS $$ +BEGIN + + RETURN QUERY SELECT * + FROM cdb_observatory._OBS_GetPointsByGeometry( + geom, + boundary_id, + time_span, + overlap_type); + RETURN; + +END; +$$ LANGUAGE plpgsql; + +-- OBS_GetBoundariesByPointAndRadius +-- +-- Given a point and radius, and it's geometry level (see +-- OBS_ListGeomColumns() for all available boundary ids), give back the +-- boundaries that are contained within the point buffered by radius meters and +-- the associated geometry ids + +-- Inputs: +-- geom geometry: point geometry centered on area of interest +-- radius numeric: radius (in meters) of a circle centered on geom for +-- selecting polygons +-- boundary_id text: source id of boundaries (e.g., us.census.tiger.county) +-- see function OBS_ListGeomColumns for all avaiable +-- boundary ids +-- time_span text: time span that the geometries were collected (optional) +-- +-- Output: +-- table with the following columns +-- boundary geometry: geometry boundary that is contained within the input +-- bounding box at the requested geometry level +-- with boundary_id, and time_span +-- geom_refs text: geometry identifiers (e.g., geoid for the US Census) +-- + +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetPointsByPointAndRadius( + geom geometry(Point, 4326), -- point + radius numeric, -- radius in meters + boundary_id text, + time_span text DEFAULT NULL, + overlap_type text DEFAULT NULL) +RETURNS TABLE(the_geom geometry, geom_refs text) +AS $$ +DECLARE + circle_boundary geometry(Geometry, 4326); +BEGIN + + IF ST_GeometryType(geom) != 'ST_Point' + THEN + RAISE EXCEPTION 'Input geometry ''%'' is not a point', ST_AsText(geom); + ELSE + circle_boundary := ST_Buffer(geom::geography, radius)::geometry; + END IF; + + RETURN QUERY SELECT * + FROM cdb_observatory._OBS_GetPointsByGeometry( + ST_Buffer(geom::geography, radius)::geometry, + boundary_id, + time_span, + overlap_type); + RETURN; +END; +$$ LANGUAGE plpgsql; +-- Placeholder for permission tweaks at creation time. +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_observatory FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +-- GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory FROM PUBLIC, publicuser; diff --git a/release/observatory.control b/release/observatory.control index dc9c991..ef9b995 100644 --- a/release/observatory.control +++ b/release/observatory.control @@ -1,5 +1,5 @@ comment = 'CartoDB Observatory backend extension' -default_version = '1.5.1' +default_version = '1.6.0' requires = 'postgis' superuser = true schema = cdb_observatory From 028c93170c966350ed03153c7097579ebb3fa868 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 20 Jul 2017 10:56:00 +0200 Subject: [PATCH 7/7] Updated NEWS with version 1.6.0 --- NEWS.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 81ffa53..9ff5c87 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,19 @@ -1.6.0 (2017-mm-dd) +1.6.0 (2017-07-20) + +__Improvements__ + +* The current OBS_GetAvailableNumerators is not designed with our + UI in mind so it's causing a lot of troubles and we're doing so + many hacks to fit our UI needs and the interface of the function so this + function it's a better fit for our purposes. ([#300](https://github.com/CartoDB/observatory-extension/pull/300)) +* Now use the new meta table `obs_meta_geom_numer_timespan` to filter + the geometries by geometries timespan and/or numerator timespan (which + is what we get when we use the obs_getavailabletimespans) ([#302](https://github.com/CartoDB/observatory-extension/pull/302)) + +__Bugfixes__ + +* Right now we're doing INNER JOINS when we JOIN the `_procgeoms` and + the data so we end up with NULL value instead of id, NULL value. ([#298](https://github.com/CartoDB/observatory-extension/pull/298)) 1.5.1 (2017-05-16)