cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'obs-meta-internal' into release-v-1.0.2-preview

Browse Source
This commit is contained in:
John Krauss
2016-07-12 15:57:44 -04:00
parent 568996930b ebc27dbbb7
commit 54701d595a
11 changed files with 12257 additions and 8557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
scripts/generate_fixtures.py
View File

@@ -34,7 +34,7 @@ def select_star(tablename):
cdb = Dumpr('observatory.cartodb.com','')
metadata = ['obs_table', 'obs_column_table', 'obs_column', 'obs_column_tag',
'obs_tag', 'obs_column_to_column', 'obs_dump_version']
'obs_tag', 'obs_column_to_column', 'obs_dump_version', ]
fixtures = [
('us.census.tiger.census_tract', 'us.census.tiger.census_tract', '2014'),
@@ -90,3 +90,124 @@ with open('src/pg/test/fixtures/load_fixtures.sql', 'w') as outfile:
cdb.dump(' '.join([select_star(tablename), "WHERE {}::text {} {}".format(colname, compare, where)]),
tablename, outfile, schema='observatory')
dropfiles.write('DROP TABLE IF EXISTS observatory.{};\n'.format(tablename))
outfile.write('''
ALTER TABLE observatory.obs_table
ADD PRIMARY KEY (id);
ALTER TABLE observatory.obs_column_table
ADD PRIMARY KEY (column_id, table_id);
CREATE UNIQUE INDEX ON observatory.obs_column_table (table_id, column_id);
CREATE UNIQUE INDEX ON observatory.obs_column_table (table_id, colname);
ALTER TABLE observatory.obs_column
ADD PRIMARY KEY (id);
ALTER TABLE observatory.obs_column_to_column
ADD PRIMARY KEY (source_id, target_id, reltype);
CREATE UNIQUE INDEX ON observatory.obs_column_to_column (target_id, source_id, reltype);
CREATE INDEX ON observatory.obs_column_to_column (reltype);
ALTER TABLE observatory.obs_column_tag
ADD PRIMARY KEY (column_id, tag_id);
CREATE UNIQUE INDEX ON observatory.obs_column_tag (tag_id, column_id);
ALTER TABLE observatory.obs_tag
ADD PRIMARY KEY (id);
CREATE INDEX ON observatory.obs_tag (type);
VACUUM ANALYZE observatory.obs_table;
VACUUM ANALYZE observatory.obs_column_table;
VACUUM ANALYZE observatory.obs_column;
VACUUM ANALYZE observatory.obs_column_to_column;
VACUUM ANALYZE observatory.obs_column_tag;
VACUUM ANALYZE observatory.obs_tag;
CREATE TABLE observatory.obs_meta AS
SELECT numer_c.id numer_id,
denom_c.id denom_id,
geom_c.id geom_id,
MAX(numer_c.name) numer_name,
MAX(denom_c.name) denom_name,
MAX(geom_c.name) geom_name,
MAX(numer_c.description) numer_description,
MAX(denom_c.description) denom_description,
MAX(geom_c.description) geom_description,
MAX(numer_c.aggregate) numer_aggregate,
MAX(denom_c.aggregate) denom_aggregate,
MAX(geom_c.aggregate) geom_aggregate,
MAX(numer_c.type) numer_type,
MAX(denom_c.type) denom_type,
MAX(geom_c.type) geom_type,
MAX(numer_data_ct.colname) numer_colname,
MAX(denom_data_ct.colname) denom_colname,
MAX(geom_geom_ct.colname) geom_colname,
MAX(numer_geomref_ct.colname) numer_geomref_colname,
MAX(denom_geomref_ct.colname) denom_geomref_colname,
MAX(geom_geomref_ct.colname) geom_geomref_colname,
MAX(numer_t.tablename) numer_tablename,
MAX(denom_t.tablename) denom_tablename,
MAX(geom_t.tablename) geom_tablename,
MAX(numer_t.timespan) numer_timespan,
MAX(denom_t.timespan) denom_timespan,
MAX(numer_c.weight) numer_weight,
MAX(denom_c.weight) denom_weight,
MAX(geom_c.weight) geom_weight,
MAX(geom_t.timespan) geom_timespan,
MAX(geom_t.the_geom_webmercator)::geometry AS the_geom_webmercator,
ARRAY_AGG(DISTINCT s_tag.id) section_tags,
ARRAY_AGG(DISTINCT ss_tag.id) subsection_tags,
ARRAY_AGG(DISTINCT unit_tag.id) unit_tags
FROM observatory.obs_column_table numer_data_ct,
observatory.obs_table numer_t,
observatory.obs_column_table numer_geomref_ct,
observatory.obs_column geomref_c,
observatory.obs_column_to_column geomref_c2c,
observatory.obs_column geom_c,
observatory.obs_column_table geom_geom_ct,
observatory.obs_column_table geom_geomref_ct,
observatory.obs_table geom_t,
observatory.obs_column_tag ss_ctag,
observatory.obs_tag ss_tag,
observatory.obs_column_tag s_ctag,
observatory.obs_tag s_tag,
observatory.obs_column_tag unit_ctag,
observatory.obs_tag unit_tag,
observatory.obs_column numer_c
LEFT JOIN (
observatory.obs_column_to_column denom_c2c
JOIN observatory.obs_column denom_c ON denom_c2c.target_id = denom_c.id
JOIN observatory.obs_column_table denom_data_ct ON denom_data_ct.column_id = denom_c.id
JOIN observatory.obs_table denom_t ON denom_data_ct.table_id = denom_t.id
JOIN observatory.obs_column_table denom_geomref_ct ON denom_geomref_ct.table_id = denom_t.id
) ON denom_c2c.source_id = numer_c.id
WHERE numer_c.id = numer_data_ct.column_id
AND numer_data_ct.table_id = numer_t.id
AND numer_t.id = numer_geomref_ct.table_id
AND numer_geomref_ct.column_id = geomref_c.id
AND geomref_c2c.reltype = 'geom_ref'
AND geomref_c.id = geomref_c2c.source_id
AND geom_c.id = geomref_c2c.target_id
AND geom_geomref_ct.column_id = geomref_c.id
AND geom_geomref_ct.table_id = geom_t.id
AND geom_geom_ct.column_id = geom_c.id
AND geom_geom_ct.table_id = geom_t.id
AND geom_c.type ILIKE 'geometry'
AND numer_c.type NOT ILIKE 'geometry'
AND numer_t.id != geom_t.id
AND numer_c.id != geomref_c.id
AND unit_tag.type = 'unit'
AND ss_tag.type = 'subsection'
AND s_tag.type = 'section'
AND unit_ctag.column_id = numer_c.id
AND unit_ctag.tag_id = unit_tag.id
AND ss_ctag.column_id = numer_c.id
AND ss_ctag.tag_id = ss_tag.id
AND s_ctag.column_id = numer_c.id
AND s_ctag.tag_id = s_tag.id
AND (denom_c2c.reltype = 'denominator' OR denom_c2c.reltype IS NULL)
AND (denom_geomref_ct.column_id = geomref_c.id OR denom_geomref_ct.column_id IS NULL)
AND (denom_t.timespan = numer_t.timespan OR denom_t.timespan IS NULL)
GROUP BY numer_c.id, denom_c.id, geom_c.id,
numer_t.id, denom_t.id, geom_t.id;
''')
dropfiles.write('''
DROP TABLE IF EXISTS observatory.obs_meta;
''')

30
src/pg/sql/41_observatory_augmentation.sql
View File

@@ -396,30 +396,28 @@ DECLARE
test_query TEXT;
BEGIN
SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table
FROM cdb_observatory._OBS_GetColumnData(boundary_id, Array[measure_id], time_span) As x;
EXECUTE
format('SELECT ct.colname
FROM observatory.obs_column_to_column c2c,
observatory.obs_column_table ct,
observatory.obs_table t
WHERE c2c.reltype = ''geom_ref''
AND ct.column_id = c2c.source_id
AND ct.table_id = t.id
AND t.tablename = %L'
, target_table)
INTO data_geoid_colname;
$query$
SELECT numer_colname, numer_geomref_colname, numer_tablename
FROM observatory.obs_meta
WHERE (geom_id = $1 OR ($1 = ''))
AND numer_id = $2
AND (numer_timespan = $3 OR ($3 = ''))
ORDER BY geom_weight DESC, numer_timespan DESC
LIMIT 1
$query$
INTO colname, data_geoid_colname, target_table
USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, '');
RAISE DEBUG 'target_table %, colname %', target_table, colname;
EXECUTE format(
'SELECT %I
FROM observatory.%I
WHERE %I.%I = %L',
FROM observatory.%I data
WHERE data.%I = %L',
colname,
target_table,
target_table, data_geoid_colname, geom_ref)
data_geoid_colname, geom_ref)
INTO measure_val;
RETURN measure_val;

3
src/pg/test/expected/41_observatory_augmentation_test.out
View File

@@ -1,5 +1,4 @@
\i test/fixtures/load_fixtures.sql
SET client_min_messages TO WARNING;
\pset format unaligned
\set ECHO none
obs_getdemographicsnapshot_test_no_returns
t

3
src/pg/test/expected/42_observatory_exploration_test.out
View File

@@ -1,5 +1,4 @@
\i test/fixtures/load_fixtures.sql
SET client_min_messages TO WARNING;
\pset format unaligned
\set ECHO none
_obs_searchtables_tables_match|_obs_searchtables_timespan_matches
t|t

3
src/pg/test/expected/44_observatory_geometries_test.out
View File

@@ -1,7 +1,4 @@
\pset format unaligned
\set ECHO all
\i test/fixtures/load_fixtures.sql
SET client_min_messages TO WARNING;
\set ECHO none
obs_getboundary_cartodb_census_tract
t

2
src/pg/test/fixtures/drop_fixtures.sql vendored
View File

@@ -20,3 +20,5 @@ DROP TABLE IF EXISTS observatory.obs_6c1309a64d8f3e6986061f4d1ca7b57743e75e74;
DROP TABLE IF EXISTS observatory.obs_d39f7fe5959891c8296490d83c22ded31c54af13;
DROP TABLE IF EXISTS observatory.obs_144e8b4f906885b2e057ac4842644a553ae49c6e;
DROP TABLE IF EXISTS observatory.obs_c6fb99c47d61289fbb8e561ff7773799d3fcc308;
DROP TABLE IF EXISTS observatory.obs_meta;

20633
src/pg/test/fixtures/load_fixtures.sql vendored
View File

File diff suppressed because one or more lines are too long

4
src/pg/test/sql/40_observatory_utility_test.sql
View File

@@ -1,6 +1,8 @@
\pset format unaligned
\set ECHO all
\i test/fixtures/load_fixtures.sql
SET client_min_messages TO WARNING;
\set ECHO none
-- OBS_GeomTable
-- get table with known geometry_id
@@ -77,5 +79,3 @@ SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}') =
SELECT cdb_observatory.OBS_DumpVersion()
IS NOT NULL AS OBS_DumpVersion_notnull;
\i test/fixtures/drop_fixtures.sql

4
src/pg/test/sql/41_observatory_augmentation_test.sql
View File

@@ -1,6 +1,6 @@
\i test/fixtures/load_fixtures.sql
\pset format unaligned
\set ECHO none
SET client_min_messages TO WARNING;
--
WITH result as(
@@ -236,5 +236,3 @@ SELECT cdb_observatory.OBS_GetMeasureById(
'us.census.tiger.block_group',
'2010 - 2014'
) IS NULL As OBS_GetMeasureById_nulls;
\i test/fixtures/drop_fixtures.sql

5
src/pg/test/sql/42_observatory_exploration_test.sql
View File

@@ -1,5 +1,6 @@
\i test/fixtures/load_fixtures.sql
\pset format unaligned
\set ECHO none
SET client_min_messages TO WARNING;
-- set up variables for use in testing
@@ -32,5 +33,3 @@ SELECT COUNT(*) > 0 AS _OBS_GetAvailableBoundariesExist
FROM cdb_observatory.OBS_GetAvailableBoundaries(
cdb_observatory._TestPoint()
) AS t(boundary_id, description, time_span, tablename);
\i test/fixtures/drop_fixtures.sql

4
src/pg/test/sql/44_observatory_geometries_test.sql
View File

@@ -1,6 +1,6 @@
\pset format unaligned
\set ECHO all
\i test/fixtures/load_fixtures.sql
\set ECHO none
SET client_min_messages TO WARNING;
-- set up variables for use in testing