cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

faster generation of obs-meta via indexes

Browse Source
This commit is contained in:
John Krauss
2016-07-12 10:42:00 -04:00
parent fe6343c73f
commit 897cf38d42
3 changed files with 218 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/generate_fixtures.py
+108 -88
View File
@@ -93,95 +93,115 @@ with open('src/pg/test/fixtures/load_fixtures.sql', 'w') as outfile:
outfile.write('''
CREATE TABLE observatory.obs_meta AS
SELECT numer_c.id numer_id,
denom_c.id denom_id,
geom_c.id geom_id,
MAX(numer_c.name) numer_name,
MAX(denom_c.name) denom_name,
MAX(geom_c.name) geom_name,
MAX(numer_c.description) numer_description,
MAX(denom_c.description) denom_description,
MAX(geom_c.description) geom_description,
MAX(numer_c.aggregate) numer_aggregate,
MAX(denom_c.aggregate) denom_aggregate,
MAX(geom_c.aggregate) geom_aggregate,
MAX(numer_c.type) numer_type,
MAX(denom_c.type) denom_type,
MAX(geom_c.type) geom_type,
MAX(numer_data_ct.colname) numer_colname,
MAX(denom_data_ct.colname) denom_colname,
MAX(geom_geom_ct.colname) geom_colname,
MAX(numer_geomref_ct.colname) numer_geomref_colname,
MAX(denom_geomref_ct.colname) denom_geomref_colname,
MAX(geom_geomref_ct.colname) geom_geomref_colname,
MAX(numer_t.tablename) numer_tablename,
MAX(denom_t.tablename) denom_tablename,
MAX(geom_t.tablename) geom_tablename,
MAX(numer_t.timespan) numer_timespan,
MAX(denom_t.timespan) denom_timespan,
MAX(numer_c.weight) numer_weight,
MAX(denom_c.weight) denom_weight,
MAX(geom_c.weight) geom_weight,
MAX(geom_t.timespan) geom_timespan,
MAX(geom_t.the_geom_webmercator)::geometry AS the_geom_webmercator,
ARRAY_AGG(DISTINCT s_tag.id) section_tags,
ARRAY_AGG(DISTINCT ss_tag.id) subsection_tags,
ARRAY_AGG(DISTINCT unit_tag.id) unit_tags
FROM observatory.obs_column_table numer_data_ct,
observatory.obs_table numer_t,
observatory.obs_column_table numer_geomref_ct,
observatory.obs_column geomref_c,
observatory.obs_column_to_column geomref_c2c,
observatory.obs_column geom_c,
observatory.obs_column_table geom_geom_ct,
observatory.obs_column_table geom_geomref_ct,
observatory.obs_table geom_t,
observatory.obs_column_tag ss_ctag,
observatory.obs_tag ss_tag,
observatory.obs_column_tag s_ctag,
observatory.obs_tag s_tag,
observatory.obs_column_tag unit_ctag,
observatory.obs_tag unit_tag,
observatory.obs_column numer_c
LEFT JOIN (
observatory.obs_column_to_column denom_c2c
JOIN observatory.obs_column denom_c ON denom_c2c.target_id = denom_c.id
JOIN observatory.obs_column_table denom_data_ct ON denom_data_ct.column_id = denom_c.id
JOIN observatory.obs_table denom_t ON denom_data_ct.table_id = denom_t.id
JOIN observatory.obs_column_table denom_geomref_ct ON denom_geomref_ct.table_id = denom_t.id
) ON denom_c2c.source_id = numer_c.id
WHERE numer_c.id = numer_data_ct.column_id
AND numer_data_ct.table_id = numer_t.id
AND numer_t.id = numer_geomref_ct.table_id
AND numer_geomref_ct.column_id = geomref_c.id
AND geomref_c2c.reltype = 'geom_ref'
AND geomref_c.id = geomref_c2c.source_id
AND geom_c.id = geomref_c2c.target_id
AND geom_geomref_ct.column_id = geomref_c.id
AND geom_geomref_ct.table_id = geom_t.id
AND geom_geom_ct.column_id = geom_c.id
AND geom_geom_ct.table_id = geom_t.id
AND geom_c.type ILIKE 'geometry'
AND numer_c.type NOT ILIKE 'geometry'
AND numer_t.id != geom_t.id
AND numer_c.id != geomref_c.id
AND unit_tag.type = 'unit'
AND ss_tag.type = 'subsection'
AND s_tag.type = 'section'
AND unit_ctag.column_id = numer_c.id
AND unit_ctag.tag_id = unit_tag.id
AND ss_ctag.column_id = numer_c.id
AND ss_ctag.tag_id = ss_tag.id
AND s_ctag.column_id = numer_c.id
AND s_ctag.tag_id = s_tag.id
AND (denom_c2c.reltype = 'denominator' OR denom_c2c.reltype IS NULL)
AND (denom_geomref_ct.column_id = geomref_c.id OR denom_geomref_ct.column_id IS NULL)
AND (denom_t.timespan = numer_t.timespan OR denom_t.timespan IS NULL)
GROUP BY numer_c.id, denom_c.id, geom_c.id,
numer_t.id, denom_t.id, geom_t.id;
ALTER TABLE observatory.obs_table
ADD PRIMARY KEY (id);
ALTER TABLE observatory.obs_column_table
ADD PRIMARY KEY (column_id, table_id),
ADD FOREIGN KEY (column_id) REFERENCES observatory.obs_column(id) ON DELETE CASCADE,
ADD FOREIGN KEY (table_id) REFERENCES observatory.obs_table(id) ON DELETE CASCADE;
CREATE UNIQUE INDEX ON observatory.obs_column_table (table_id, colname);
ALTER TABLE observatory.obs_column
ADD PRIMARY KEY (id);
ALTER TABLE observatory.obs_column_to_column
ADD PRIMARY KEY (source_id, target_id, reltype),
ADD FOREIGN KEY (source_id) REFERENCES observatory.obs_column(id) ON DELETE CASCADE,
ADD FOREIGN KEY (target_id) REFERENCES observatory.obs_column(id) ON DELETE CASCADE;
ALTER TABLE observatory.obs_column_tag
ADD PRIMARY KEY (column_id, tag_id),
ADD FOREIGN KEY (column_id) REFERENCES observatory.obs_column(id) ON DELETE CASCADE,
ADD FOREIGN KEY (tag_id) REFERENCES observatory.obs_tag(id) ON DELETE CASCADE;
ALTER TABLE observatory.obs_tag
ADD PRIMARY KEY (id);
CREATE TABLE observatory.obs_meta AS
SELECT numer_c.id numer_id,
denom_c.id denom_id,
geom_c.id geom_id,
MAX(numer_c.name) numer_name,
MAX(denom_c.name) denom_name,
MAX(geom_c.name) geom_name,
MAX(numer_c.description) numer_description,
MAX(denom_c.description) denom_description,
MAX(geom_c.description) geom_description,
MAX(numer_c.aggregate) numer_aggregate,
MAX(denom_c.aggregate) denom_aggregate,
MAX(geom_c.aggregate) geom_aggregate,
MAX(numer_c.type) numer_type,
MAX(denom_c.type) denom_type,
MAX(geom_c.type) geom_type,
MAX(numer_data_ct.colname) numer_colname,
MAX(denom_data_ct.colname) denom_colname,
MAX(geom_geom_ct.colname) geom_colname,
MAX(numer_geomref_ct.colname) numer_geomref_colname,
MAX(denom_geomref_ct.colname) denom_geomref_colname,
MAX(geom_geomref_ct.colname) geom_geomref_colname,
MAX(numer_t.tablename) numer_tablename,
MAX(denom_t.tablename) denom_tablename,
MAX(geom_t.tablename) geom_tablename,
MAX(numer_t.timespan) numer_timespan,
MAX(denom_t.timespan) denom_timespan,
MAX(numer_c.weight) numer_weight,
MAX(denom_c.weight) denom_weight,
MAX(geom_c.weight) geom_weight,
MAX(geom_t.timespan) geom_timespan,
MAX(geom_t.the_geom_webmercator)::geometry AS the_geom_webmercator,
ARRAY_AGG(DISTINCT s_tag.id) section_tags,
ARRAY_AGG(DISTINCT ss_tag.id) subsection_tags,
ARRAY_AGG(DISTINCT unit_tag.id) unit_tags
FROM observatory.obs_column_table numer_data_ct,
observatory.obs_table numer_t,
observatory.obs_column_table numer_geomref_ct,
observatory.obs_column geomref_c,
observatory.obs_column_to_column geomref_c2c,
observatory.obs_column geom_c,
observatory.obs_column_table geom_geom_ct,
observatory.obs_column_table geom_geomref_ct,
observatory.obs_table geom_t,
observatory.obs_column_tag ss_ctag,
observatory.obs_tag ss_tag,
observatory.obs_column_tag s_ctag,
observatory.obs_tag s_tag,
observatory.obs_column_tag unit_ctag,
observatory.obs_tag unit_tag,
observatory.obs_column numer_c
LEFT JOIN (
observatory.obs_column_to_column denom_c2c
JOIN observatory.obs_column denom_c ON denom_c2c.target_id = denom_c.id
JOIN observatory.obs_column_table denom_data_ct ON denom_data_ct.column_id = denom_c.id
JOIN observatory.obs_table denom_t ON denom_data_ct.table_id = denom_t.id
JOIN observatory.obs_column_table denom_geomref_ct ON denom_geomref_ct.table_id = denom_t.id
) ON denom_c2c.source_id = numer_c.id
WHERE numer_c.id = numer_data_ct.column_id
AND numer_data_ct.table_id = numer_t.id
AND numer_t.id = numer_geomref_ct.table_id
AND numer_geomref_ct.column_id = geomref_c.id
AND geomref_c2c.reltype = 'geom_ref'
AND geomref_c.id = geomref_c2c.source_id
AND geom_c.id = geomref_c2c.target_id
AND geom_geomref_ct.column_id = geomref_c.id
AND geom_geomref_ct.table_id = geom_t.id
AND geom_geom_ct.column_id = geom_c.id
AND geom_geom_ct.table_id = geom_t.id
AND geom_c.type ILIKE 'geometry'
AND numer_c.type NOT ILIKE 'geometry'
AND numer_t.id != geom_t.id
AND numer_c.id != geomref_c.id
AND unit_tag.type = 'unit'
AND ss_tag.type = 'subsection'
AND s_tag.type = 'section'
AND unit_ctag.column_id = numer_c.id
AND unit_ctag.tag_id = unit_tag.id
AND ss_ctag.column_id = numer_c.id
AND ss_ctag.tag_id = ss_tag.id
AND s_ctag.column_id = numer_c.id
AND s_ctag.tag_id = s_tag.id
AND (denom_c2c.reltype = 'denominator' OR denom_c2c.reltype IS NULL)
AND (denom_geomref_ct.column_id = geomref_c.id OR denom_geomref_ct.column_id IS NULL)
AND (denom_t.timespan = numer_t.timespan OR denom_t.timespan IS NULL)
GROUP BY numer_c.id, denom_c.id, geom_c.id,
numer_t.id, denom_t.id, geom_t.id;
''')
dropfiles.write('''
DROP TABLE IF EXISTS observatory.obs_meta
DROP TABLE IF EXISTS observatory.obs_meta;
''')