cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests pass, although obs_getmeasure performance suffers

Browse Source
This commit is contained in:
John Krauss
2016-12-14 22:58:22 +00:00
parent 88d1145c12
commit 5a5d5a9386
1 changed files with 243 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
src/pg/sql/41_observatory_augmentation.sql
View File

@@ -335,56 +335,96 @@ END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureMeta(
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureMetaMulti(
geom geometry(Geometry, 4326),
measure_id TEXT,
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL,
params JSON,
max_timespan_rank INTEGER DEFAULT NULL, -- cutoff for timespan ranks when there's ambiguity
max_score_rank INTEGER DEFAULT NULL, -- cutoff for geom ranks when there's ambiguity
target_geoms INTEGER DEFAULT NULL
)
RETURNS TABLE (
numer_aggregate VARCHAR,
numer_colname VARCHAR,
numer_geomref_colname VARCHAR,
numer_tablename VARCHAR,
denom_colname VARCHAR,
denom_geomref_colname VARCHAR,
denom_tablename VARCHAR,
geom_colname VARCHAR,
geom_geomref_colname VARCHAR,
geom_tablename VARCHAR,
numer_name VARCHAR,
denom_name VARCHAR,
geom_name VARCHAR,
denom_id VARCHAR,
geom_id VARCHAR
) AS $$
RETURNS JSON
AS $$
DECLARE
result JSON;
BEGIN
RETURN QUERY
IF max_timespan_rank IS NULL THEN
max_timespan_rank := 1;
END IF;
IF max_score_rank IS NULL THEN
max_score_rank := 1;
END IF;
EXECUTE $string$
WITH meta AS (SELECT numer_aggregate, numer_colname, numer_geomref_colname,
numer_tablename, denom_colname, denom_geomref_colname,
denom_tablename, geom_colname, geom_geomref_colname,
geom_tablename, numer_name, denom_name, geom_name,
denom_id, geom_id
FROM observatory.obs_meta
WHERE (geom_id = $1 OR ($1 = ''))
AND numer_id = $2
AND (numer_timespan = $3 OR ($3 = ''))),
scores AS (SELECT *
FROM cdb_observatory._OBS_GetGeometryScores($4,
(SELECT Array_Agg(geom_id) FROM meta), $5))
SELECT meta.*
FROM meta, scores
WHERE meta.geom_id = scores.geom_id
ORDER BY score DESC
LIMIT 1
$string$ USING COALESCE(boundary_id, ''), measure_id, COALESCE(time_span, ''),
WITH _filters AS (SELECT
generate_series(1, array_length($3, 1)) id,
(unnest($3))->>'numer_id' numer_id,
(unnest($3))->>'denom_id' denom_id,
(unnest($3))->>'geom_id' geom_id,
(unnest($3))->>'timespan' timespan
), meta AS (SELECT
id,
numer_aggregate, numer_colname, numer_geomref_colname,
numer_tablename, denom_aggregate, denom_colname, denom_geomref_colname,
denom_tablename, geom_colname, geom_geomref_colname,
geom_tablename, numer_name, denom_name, geom_name,
numer_type, denom_type, geom_type,
m.denom_id, m.geom_id, m.numer_timespan
FROM observatory.obs_meta m JOIN _filters f
ON m.numer_id = f.numer_id
WHERE
(m.denom_id = f.denom_id OR COALESCE(f.denom_id, '') = '')
AND (m.geom_id = f.geom_id OR COALESCE(f.geom_id, '') = '')
AND (m.numer_timespan = f.timespan OR COALESCE(f.timespan, '') = '')
), scores AS (
SELECT *
FROM cdb_observatory._OBS_GetGeometryScores($1,
(SELECT Array_Agg(geom_id) FROM meta), $2) scores
), groups AS (SELECT
id, scores.score, numer_timespan,
dense_rank() OVER (PARTITION BY id ORDER BY numer_timespan DESC) timespan_rank,
dense_rank() OVER (PARTITION BY id ORDER BY score DESC) score_rank,
json_build_object(
'timespan_rank', dense_rank() OVER (PARTITION BY id ORDER BY numer_timespan DESC),
'score_rank', dense_rank() OVER (PARTITION BY id ORDER BY score DESC),
'score', scores.score,
'numer_aggregate', meta.numer_aggregate,
'numer_colname', meta.numer_colname,
'numer_geomref_colname', meta.numer_geomref_colname,
'numer_tablename', meta.numer_tablename,
'numer_type', meta.numer_type,
'denom_aggregate', meta.denom_aggregate,
'denom_colname', denom_colname,
'denom_geomref_colname', denom_geomref_colname,
'denom_tablename', denom_tablename,
'denom_type', meta.denom_type,
'geom_colname', geom_colname,
'geom_geomref_colname', geom_geomref_colname,
'geom_tablename', geom_tablename,
'geom_type', meta.geom_type,
'timespan', numer_timespan,
'numer_name', numer_name,
'denom_name', denom_name,
'geom_name', geom_name,
'denom_id', denom_id,
'geom_id', meta.geom_id
) metadata
FROM meta, scores
WHERE meta.geom_id = scores.geom_id
) SELECT JSON_AGG(metadata ORDER BY id)
FROM groups
WHERE timespan_rank <= $4
AND score_rank <= $5
$string$
INTO result
USING
CASE WHEN ST_GeometryType(geom) = 'ST_Point' THEN
ST_Buffer(geom::geography, 200)::geometry(geometry, 4326)
ELSE geom
END, target_geoms;
RETURN;
END,
target_geoms,
(SELECT ARRAY(SELECT json_array_elements_text(params))::json[]),
max_timespan_rank,
max_score_rank;
RETURN result;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
@@ -542,17 +582,7 @@ RETURNS NUMERIC
AS $$
DECLARE
geom_type TEXT;
numer_aggregate TEXT;
numer_colname TEXT;
numer_geomref_colname TEXT;
numer_tablename TEXT;
denom_colname TEXT;
denom_geomref_colname TEXT;
denom_tablename TEXT;
geom_colname TEXT;
geom_geomref_colname TEXT;
geom_tablename TEXT;
geom_id TEXT;
params JSON;
result NUMERIC;
numer_name TEXT;
denom_name TEXT;
@@ -578,25 +608,32 @@ BEGIN
ST_GeometryType(geom);
END IF;
SELECT * FROM cdb_observatory.OBS_GetMeasureMeta(geom, measure_id,
boundary_id, time_span, 500)
INTO numer_aggregate, numer_colname, numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename, numer_name,
denom_name, geom_name, denom_id, geom_id;
params := (SELECT cdb_observatory.OBS_GetMeasureMetaMulti(
geom, JSON_Build_Array(JSON_Build_Object('numer_id', measure_id,
'geom_id', boundary_id,
'timespan', time_span
)), 500))->>0;
IF geom_id IS NULL THEN
IF params->>'geom_id' IS NULL THEN
RAISE NOTICE 'No boundary found for geom';
RETURN NULL;
ELSE
RAISE NOTICE 'Using boundary %', geom_id;
RAISE NOTICE 'Using boundary %', params->>'geom_id';
END IF;
SELECT cdb_observatory.OBS_GetMeasureData(geom, geom_type, normalize,
numer_aggregate, numer_colname,
numer_geomref_colname, numer_tablename,
denom_colname, denom_geomref_colname, denom_tablename,
geom_colname, geom_geomref_colname, geom_tablename)
SELECT cdb_observatory.OBS_GetMeasureData(geom,
geom_type,
normalize,
params->>'numer_aggregate',
params->>'numer_colname',
params->>'numer_geomref_colname',
params->>'numer_tablename',
params->>'denom_colname',
params->>'denom_geomref_colname',
params->>'denom_tablename',
params->>'geom_colname',
params->>'geom_geomref_colname',
params->>'geom_tablename')
INTO result;
RETURN result;
@@ -651,6 +688,146 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureDataMulti(
--geoms Geometry(Geometry, 4326)[], params json)
geomvals geomval[], params JSON)
RETURNS SETOF RECORD
AS $$
DECLARE
colspecs TEXT;
tables TEXT;
obs_wheres TEXT;
user_wheres TEXT;
measure_id text;
measures_list text;
measures_query text;
geom_table_name text;
data_table_name text;
BEGIN
EXECUTE
$query$
WITH _meta AS (SELECT
generate_series(1, array_length($1, 1)) colid,
(unnest($1))->>'numer_id' numer_id,
(unnest($1))->>'numer_aggregate' numer_aggregate,
(unnest($1))->>'numer_colname' numer_colname,
(unnest($1))->>'numer_geomref_colname' numer_geomref_colname,
(unnest($1))->>'numer_tablename' numer_tablename,
(unnest($1))->>'numer_type' numer_type,
(unnest($1))->>'denom_id' denom_id,
(unnest($1))->>'denom_aggregate' denom_aggregate,
(unnest($1))->>'denom_colname' denom_colname,
(unnest($1))->>'denom_geomref_colname' denom_geomref_colname,
(unnest($1))->>'denom_tablename' denom_tablename,
(unnest($1))->>'denom_type' denom_type,
(unnest($1))->>'geom_id' geom_id,
(unnest($1))->>'geom_colname' geom_colname,
(unnest($1))->>'geom_geomref_colname' geom_geomref_colname,
(unnest($1))->>'geom_tablename' geom_tablename,
(unnest($1))->>'geom_type' geom_type,
(unnest($1))->>'timespan' timespan
)
SELECT String_Agg(CASE
-- denominated
WHEN denom_id IS NOT NULL THEN ' CASE ' ||
-- denominated point-in-poly or user polygon is same as OBS polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname ||
' / NullIf(' || denom_tablename || '.' || denom_colname || ', 0))' ||
-- denominated polygon interpolation
-- SUM ((numer / denom) * (% user geom in OBS geom))
' ELSE ' ||
--' NULL END '
' SUM((' || numer_tablename || '.' || numer_colname || '/NullIf(' || denom_tablename || '.' || denom_colname || ', 0)) ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) THEN ' ||
' ST_Area(' || geom_tablename || '.' || geom_colname || ') ' ||
' / ST_Area(_geoms.geom)' ||
' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / ST_Area(_geoms.geom))' ||
' END) END '
-- areaNormalized
WHEN numer_aggregate ILIKE 'sum' THEN ' CASE ' ||
-- areaNormalized point-in-poly or user polygon is the same as OBS polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname ||
' / (ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography)/1000000)) ' ||
-- areaNormalized polygon interpolation
-- SUM (numer * (% OBS geom in user geom)) / area of big geom
' ELSE ' ||
--' NULL END '
' SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') ' ||
' THEN ST_Area(_geoms.geom) / ST_Area(' || geom_tablename || '.' || geom_colname || ') ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) ' ||
' THEN 1 ' ||
' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / ST_Area(' || geom_tablename || '.' || geom_colname || '))' ||
' END) END '
-- prenormalized
ELSE ' CASE ' ||
-- predenominated point-in-poly or user polygon is the same as OBS- polygon
' WHEN ST_GeometryType(cdb_observatory.FIRST(_geoms.geom)) = ''ST_Point'' ' ||
' OR cdb_observatory.FIRST(_geoms.geom = ' || geom_tablename || '.' || geom_colname || ')' ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' ||
' ELSE ' ||
-- predenominated polygon interpolation
-- TODO should weight by universe instead of area
-- SUM (numer * (% user geom in OBS geom))
' SUM((' || numer_tablename || '.' || numer_colname || ') ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' ||
' WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) THEN ' ||
' ST_Area(' || geom_tablename || '.' || geom_colname || ') ' ||
' / ST_Area(_geoms.geom)' ||
' ELSE (ST_Area(ST_Intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / ST_Area(_geoms.geom))' ||
' END) END '
END || ':: ' || numer_type || ' AS ' || numer_colname, ', ') AS colspecs,
(SELECT String_Agg(tablename, ', ') FROM (SELECT JSONB_Object_Keys(JSONB_Object(
Array_Cat(Array_Agg('observatory.' || numer_tablename),
Array_Cat(Array_Agg('observatory.' || geom_tablename),
Array_Agg('observatory.' || denom_tablename) FILTER (WHERE denom_tablename IS NOT NULL))),
Array_Cat(Array_Agg(numer_tablename),
Array_Cat(Array_Agg(geom_tablename),
Array_Agg(denom_tablename) FILTER (WHERE denom_tablename IS NOT NULL)))
)) tablename) bar) tablenames,
String_Agg(numer_tablename || '.' || numer_geomref_colname || ' = ' ||
geom_tablename || '.' || geom_geomref_colname ||
Coalesce(' AND ' || numer_tablename || '.' || numer_geomref_colname || ' = ' ||
denom_tablename || '.' || denom_geomref_colname, ''),
' AND ') AS obs_wheres,
String_Agg('ST_Intersects(' || geom_tablename || '.' || geom_colname
|| ', _geoms.geom)', ' AND ')
AS user_wheres
FROM _meta
;
$query$
INTO colspecs, tables, obs_wheres, user_wheres
USING (SELECT ARRAY(SELECT json_array_elements_text(params))::json[]);
RETURN QUERY EXECUTE format($query$
WITH _geoms AS (SELECT
--generate_series(1, array_length($1, 1)) id,
(UNNEST($1)).val as id,
(UNNEST($1)).geom AS geom)
SELECT _geoms.id::INT, %s
FROM %s, _geoms
WHERE %s
AND %s
GROUP BY _geoms.id
ORDER BY _geoms.id
$query$, colspecs, tables, obs_wheres, user_wheres)
USING geomvals;
RETURN;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetCategory(
geom geometry(Geometry, 4326),
category_id TEXT,