cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix issues with python autotest failing for nulls, try removing case statements around geometries in getdata

Browse Source
This commit is contained in:
John Krauss
2017-03-08 21:17:45 +00:00
parent 862db2c33a
commit 22eb6349c2
2 changed files with 50 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pg/sql/41_observatory_augmentation.sql
+46 -45
View File
@@ -514,7 +514,10 @@ BEGIN
GROUP BY _geomrefs.id GROUP BY _geomrefs.id
ORDER BY _geomrefs.id ORDER BY _geomrefs.id
$query$, colspecs, tables, $query$, colspecs, tables,
'WHERE ' || NULLIF(ARRAY_TO_STRING(ARRAY[obs_wheres, user_wheres], ' AND '), '')) 'WHERE ' || NULLIF(ARRAY_TO_STRING(ARRAY[
Nullif(obs_wheres, ''), Nullif(user_wheres, '')
], ' AND '), '')
)
USING geomrefs; USING geomrefs;
RETURN; RETURN;
END; END;
@@ -589,15 +592,14 @@ BEGIN
-- denominated -- denominated
WHEN LOWER(normalization) LIKE 'denom%' OR WHEN LOWER(normalization) LIKE 'denom%' OR
(normalization IS NULL AND LOWER(denom_reltype) LIKE 'denominator') (normalization IS NULL AND LOWER(denom_reltype) LIKE 'denominator')
THEN ' CASE ' || THEN CASE
-- denominated point-in-poly or user polygon is same as OBS polygon -- denominated point-in-poly
' WHEN EVERY(ST_GeometryType(_geoms.geom) = ''ST_Point'') ' || WHEN $2 = 'ST_Point' THEN
' OR EVERY(_geoms.geom::TEXT = ' || geom_tablename || '.' || geom_colname || '::TEXT)' || ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ' / NullIf(' || denom_tablename || '.' || denom_colname || ', 0))'
' / NullIf(' || denom_tablename || '.' || denom_colname || ', 0))' ||
-- denominated polygon interpolation -- denominated polygon interpolation
-- SUM (numer * (% OBS geom in user geom)) / SUM (denom * (% OBS geom in user geom)) -- SUM (numer * (% OBS geom in user geom)) / SUM (denom * (% OBS geom in user geom))
' ELSE ' || ELSE
' SUM(' || numer_tablename || '.' || numer_colname || ' ' || ' SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') ' || ' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') ' ||
' THEN ST_Area(_geoms.geom) / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0) ' || ' THEN ST_Area(_geoms.geom) / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0) ' ||
@@ -614,20 +616,19 @@ BEGIN
' ELSE (ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' || ' ELSE (ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0))' || ' / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0))' ||
' END), 0) ' || ' END), 0) ' ||
' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) ' || ' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) '
' END ' END
-- areaNormalized -- areaNormalized
WHEN LOWER(normalization) LIKE 'area%' OR WHEN LOWER(normalization) LIKE 'area%' OR
(normalization IS NULL AND numer_aggregate ILIKE 'sum') (normalization IS NULL AND numer_aggregate ILIKE 'sum')
THEN ' CASE ' || THEN CASE
-- areaNormalized point-in-poly or user polygon is the same as OBS polygon -- areaNormalized point-in-poly
' WHEN EVERY(ST_GeometryType(_geoms.geom) = ''ST_Point'') ' || WHEN $2 = 'ST_Point' THEN
' OR EVERY(_geoms.geom::TEXT = ' || geom_tablename || '.' || geom_colname || '::TEXT)' || ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname ||
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ' / (Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography), 0)/1000000)) '
' / (Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography), 0)/1000000)) ' ||
-- areaNormalized polygon interpolation -- areaNormalized polygon interpolation
-- SUM (numer * (% OBS geom in user geom)) / area of big geom -- SUM (numer * (% OBS geom in user geom)) / area of big geom
' ELSE ' || ELSE
--' NULL END ' --' NULL END '
' SUM((' || numer_tablename || '.' || numer_colname || ') ' || ' SUM((' || numer_tablename || '.' || numer_colname || ') ' ||
' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' || ' * CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ') THEN 1 ' ||
@@ -637,18 +638,17 @@ BEGIN
' ELSE (ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' || ' ELSE (ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / Nullif(ST_Area(_geoms.geom), 0))' || ' / Nullif(ST_Area(_geoms.geom), 0))' ||
' END / (Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography), 0) / 1000000)) ' || ' END / (Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '::Geography), 0) / 1000000)) ' ||
' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) ' || ' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) '
' END ' END
-- median/average measures with universe -- median/average measures with universe
WHEN LOWER(numer_aggregate) IN ('median', 'average') AND WHEN LOWER(numer_aggregate) IN ('median', 'average') AND
denom_reltype ILIKE 'universe' AND denom_reltype ILIKE 'universe' AND
(normalization IS NULL OR LOWER(normalization) LIKE 'pre%') (normalization IS NULL OR LOWER(normalization) LIKE 'pre%')
THEN ' CASE ' || THEN CASE
-- predenominated point-in-poly or user polygon is the same as OBS- polygon -- predenominated point-in-poly
' WHEN EVERY(ST_GeometryType(_geoms.geom) = ''ST_Point'') ' || WHEN $2 = 'ST_Point' THEN
' OR EVERY(_geoms.geom::TEXT = ' || geom_tablename || '.' || geom_colname || '::TEXT)' || ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') '
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' || ELSE
' ELSE ' ||
-- predenominated polygon interpolation weighted by universe -- predenominated polygon interpolation weighted by universe
-- SUM (numer * denom * (% user geom in OBS geom)) / SUM (denom * (% user geom in OBS geom)) -- SUM (numer * denom * (% user geom in OBS geom)) / SUM (denom * (% user geom in OBS geom))
-- (10 * 1000 * 1) / (1000 * 1) = 10 -- (10 * 1000 * 1) / (1000 * 1) = 10
@@ -670,17 +670,16 @@ BEGIN
' ELSE (ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' || ' ELSE (ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0))' || ' / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0))' ||
' END), 0) ' || ' END), 0) ' ||
' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) ' || ' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) '
'END ' END
-- prenormalized for summable measures. point or summable only! -- prenormalized for summable measures. point or summable only!
WHEN numer_aggregate ILIKE 'sum' AND WHEN numer_aggregate ILIKE 'sum' AND
(normalization IS NULL OR LOWER(normalization) LIKE 'pre%') (normalization IS NULL OR LOWER(normalization) LIKE 'pre%')
THEN ' CASE ' || THEN CASE
-- predenominated point-in-poly or user polygon is the same as OBS- polygon -- predenominated point-in-poly
' WHEN EVERY(ST_GeometryType(_geoms.geom) = ''ST_Point'') ' || WHEN $2 = 'ST_Point' THEN
' OR EVERY(_geoms.geom::TEXT = ' || geom_tablename || '.' || geom_colname || '::TEXT)' || ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') '
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' || ELSE
' ELSE ' ||
-- predenominated polygon interpolation -- predenominated polygon interpolation
-- SUM (numer * (% user geom in OBS geom)) -- SUM (numer * (% user geom in OBS geom))
' SUM(' || numer_tablename || '.' || numer_colname || ' ' || ' SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
@@ -692,17 +691,17 @@ BEGIN
geom_tablename || '.' || geom_colname || ')) ' || geom_tablename || '.' || geom_colname || ')) ' ||
' / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0))' || ' / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0))' ||
' END) ' || ' END) ' ||
' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) ' || ' / (COUNT(*) / COUNT(distinct ' || geom_tablename || '.' || geom_geomref_colname || ')) '
'END ' END
-- Everything else. Point only! -- Everything else. Point only!
ELSE ' CASE ' || ELSE CASE
' WHEN EVERY(ST_GeometryType(_geoms.geom) = ''ST_Point'') ' || WHEN $2 = 'ST_Point' THEN
' OR EVERY(_geoms.geom::TEXT = ' || geom_tablename || '.' || geom_colname || '::TEXT)' || ' cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') '
' THEN cdb_observatory.FIRST(' || numer_tablename || '.' || numer_colname || ') ' || ELSE
' ELSE cdb_observatory._OBS_RaiseNotice(''Cannot perform calculation over polygon for ' || ' cdb_observatory._OBS_RaiseNotice(''Cannot perform calculation over polygon for ' ||
numer_id || '/' || coalesce(denom_id, '') || '/' || geom_id || '/' || numer_timespan || ''')::Numeric ' || numer_id || '/' || coalesce(denom_id, '') || '/' || geom_id || '/' || numer_timespan || ''')::Numeric '
' END ' END
END || ':: ' || numer_type END || '::' || numer_type
-- categorical/text -- categorical/text
WHEN LOWER(numer_type) LIKE 'text' THEN WHEN LOWER(numer_type) LIKE 'text' THEN
@@ -769,7 +768,7 @@ BEGIN
; ;
$query$ $query$
INTO colspecs, geomrefs, tables, obs_wheres, user_wheres INTO colspecs, geomrefs, tables, obs_wheres, user_wheres
USING (SELECT ARRAY(SELECT json_array_elements_text(params))::json[]); USING (SELECT ARRAY(SELECT json_array_elements_text(params))::json[]), geomtype;
RETURN QUERY EXECUTE format($query$ RETURN QUERY EXECUTE format($query$
WITH _raw_geoms AS (SELECT WITH _raw_geoms AS (SELECT
@@ -786,7 +785,9 @@ BEGIN
GROUP BY _geoms.id %s GROUP BY _geoms.id %s
ORDER BY _geoms.id ORDER BY _geoms.id
$query$, colspecs, tables, $query$, colspecs, tables,
'WHERE ' || NULLIF(ARRAY_TO_STRING(ARRAY[obs_wheres, user_wheres], ' AND '), ''), 'WHERE ' || NULLIF(ARRAY_TO_STRING(ARRAY[
Nullif(obs_wheres, ''), Nullif(user_wheres, '')
], ' AND '), ''),
CASE WHEN merge IS False THEN ', ' || geomrefs ELSE '' END) CASE WHEN merge IS False THEN ', ' || geomrefs ELSE '' END)
USING geomvals; USING geomvals;
RETURN; RETURN;
src/python/test/autotest.py
+4 -8
View File
@@ -257,9 +257,10 @@ def test_get_measure_points(numer_ids, numer_aggregate, denom_reltype, section_t
def test_get_measure_areas(numer_ids, numer_aggregate, denom_reltype, section_tags): def test_get_measure_areas(numer_ids, numer_aggregate, denom_reltype, section_tags):
if numer_aggregate.lower() not in ('sum', 'median', 'average'): if numer_aggregate.lower() not in ('sum', 'median', 'average'):
return return
if numer_aggregate.lower() in ('median', 'average') \ if numer_aggregate is None or \
and denom_reltype is not None \ (numer_aggregate.lower() in ('median', 'average') \
and denom_reltype.lower() != 'universe': and (denom_reltype is None \
or denom_reltype.lower() != 'universe')):
return return
_test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_area(numer_ids[0])) _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_area(numer_ids[0]))
@@ -290,12 +291,7 @@ def _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, geom
geom=geom, geom=geom,
params=json.dumps(params).replace(u"'", "''")) params=json.dumps(params).replace(u"'", "''"))
resp = query(q).fetchone() resp = query(q).fetchone()
#try:
assert_is_not_none(resp, 'NULL returned for {}'.format(in_params)) assert_is_not_none(resp, 'NULL returned for {}'.format(in_params))
#except:
# #import pdb
# #pdb.set_trace()
# raise
rawvals = resp[1] rawvals = resp[1]
vals = [v['value'] for v in rawvals] vals = [v['value'] for v in rawvals]