diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 714f990..3ed8651 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -231,3 +231,18 @@ CREATE AGGREGATE cdb_observatory.FIRST ( basetype = anyelement, stype = anyelement ); + +CREATE OR REPLACE FUNCTION cdb_observatory.isnumeric ( + typename varchar +) +RETURNS BOOLEAN LANGUAGE SQL IMMUTABLE STRICT AS $$ + SELECT LOWER(typename) IN ( + 'smallint', + 'integer', + 'bigint', + 'decimal', + 'numeric', + 'real', + 'double precision' + ) +$$; diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 1adb1ef..0f1d1ee 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -437,7 +437,7 @@ BEGIN 'cdb_observatory.FIRST( ' || api_method || '.' || numer_colname || ')::' || numer_type -- numeric internal values - WHEN LOWER(numer_type) LIKE 'numeric' THEN + WHEN cdb_observatory.isnumeric(numer_type) THEN '''value'', ' || CASE -- denominated WHEN LOWER(normalization) LIKE 'denom%' OR (normalization IS NULL AND denom_id IS NOT NULL) @@ -485,10 +485,16 @@ BEGIN ) tablenames_inner ) tablenames_outer) tablenames, - String_Agg(numer_tablename || '.' || numer_geomref_colname || ' = ' || - geom_tablename || '.' || geom_geomref_colname || - Coalesce(' AND ' || numer_tablename || '.' || numer_geomref_colname || ' = ' || - denom_tablename || '.' || denom_geomref_colname, ''), + String_Agg(DISTINCT array_to_string(ARRAY[ + CASE WHEN numer_tablename != geom_tablename + THEN numer_tablename || '.' || numer_geomref_colname || ' = ' || + geom_tablename || '.' || geom_geomref_colname + ELSE NULL END, + CASE WHEN numer_tablename != denom_tablename + THEN numer_tablename || '.' || numer_geomref_colname || ' = ' || + denom_tablename || '.' || denom_geomref_colname + ELSE NULL END + ], ' AND '), ' AND ') AS obs_wheres, String_Agg(geom_tablename || '.' || geom_geomref_colname || ' = ' || @@ -532,12 +538,15 @@ DECLARE tables TEXT; obs_wheres TEXT; user_wheres TEXT; + geomtype TEXT; BEGIN IF params IS NULL OR JSON_ARRAY_LENGTH(params) = 0 THEN RETURN QUERY EXECUTE $query$ SELECT NULL::INT, NULL::JSON LIMIT 0 $query$; RETURN; END IF; + geomtype := ST_GeometryType(geomvals[1].geom); + EXECUTE $query$ WITH _meta AS (SELECT @@ -575,7 +584,7 @@ BEGIN 'cdb_observatory.FIRST( ' || api_method || '.' || numer_colname || ')::' || numer_type -- numeric internal values - WHEN LOWER(numer_type) LIKE 'numeric' THEN + WHEN cdb_observatory.isnumeric(numer_type) THEN '''value'', ' || CASE -- denominated WHEN LOWER(normalization) LIKE 'denom%' OR @@ -739,13 +748,20 @@ BEGIN ) tablenames_inner ) tablenames_outer) tablenames, - String_Agg(DISTINCT numer_tablename || '.' || numer_geomref_colname || ' = ' || - geom_tablename || '.' || geom_geomref_colname || - Coalesce(' AND ' || numer_tablename || '.' || numer_geomref_colname || ' = ' || - denom_tablename || '.' || denom_geomref_colname, ''), - ' AND ') AS obs_wheres, + String_Agg(DISTINCT array_to_string(ARRAY[ + CASE WHEN numer_tablename != geom_tablename + THEN numer_tablename || '.' || numer_geomref_colname || ' = ' || + geom_tablename || '.' || geom_geomref_colname + ELSE NULL END, + CASE WHEN numer_tablename != denom_tablename + THEN numer_tablename || '.' || numer_geomref_colname || ' = ' || + denom_tablename || '.' || denom_geomref_colname + ELSE NULL END + ], ' AND '), + ' AND ') FILTER (WHERE numer_tablename != geom_tablename + OR numer_tablename != denom_tablename) AS obs_wheres, - String_Agg('ST_Intersects(' || geom_tablename || '.' || geom_colname + String_Agg(DISTINCT 'ST_Intersects(' || geom_tablename || '.' || geom_colname || ', _geoms.geom)', ' AND ') AS user_wheres FROM _meta diff --git a/src/python/requirements.txt b/src/python/requirements.txt index dbab3a3..02b0807 100644 --- a/src/python/requirements.txt +++ b/src/python/requirements.txt @@ -1,3 +1,4 @@ nose +nose-timer nose_parameterized psycopg2 diff --git a/src/python/test/autotest.py b/src/python/test/autotest.py index 3b37204..86b277f 100644 --- a/src/python/test/autotest.py +++ b/src/python/test/autotest.py @@ -67,38 +67,47 @@ SKIP_COLUMNS = set([ , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE' , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC' , 'fr.insee.P12_ACTOCC15P_ILT45D' + , 'uk.ons.LC3202WA0007' + , 'uk.ons.LC3202WA0010' + , 'uk.ons.LC3202WA0004' + , 'uk.ons.LC3204WA0004' + , 'uk.ons.LC3204WA0007' + , 'uk.ons.LC3204WA0010' ]) MEASURE_COLUMNS = query(''' SELECT ARRAY_AGG(DISTINCT numer_id) numer_ids, numer_aggregate, + denom_reltype, section_tags FROM observatory.obs_meta WHERE numer_weight > 0 AND numer_id NOT IN ('{skip}') -GROUP BY numer_aggregate, section_tags + AND section_tags IS NOT NULL + AND subsection_tags IS NOT NULL +GROUP BY numer_aggregate, section_tags, denom_reltype '''.format(skip="', '".join(SKIP_COLUMNS))).fetchall() -CATEGORY_COLUMNS = query(''' -SELECT distinct numer_id -FROM observatory.obs_meta -WHERE numer_type ILIKE 'text' -AND numer_weight > 0 -''').fetchall() - -BOUNDARY_COLUMNS = query(''' -SELECT id FROM observatory.obs_column -WHERE type ILIKE 'geometry' -AND weight > 0 -''').fetchall() - -US_CENSUS_MEASURE_COLUMNS = query(''' -SELECT distinct numer_name -FROM observatory.obs_meta -WHERE numer_type ILIKE 'numeric' -AND 'us.census.acs' = ANY (subsection_tags) -AND numer_weight > 0 -''').fetchall() +#CATEGORY_COLUMNS = query(''' +#SELECT distinct numer_id +#FROM observatory.obs_meta +#WHERE numer_type ILIKE 'text' +#AND numer_weight > 0 +#''').fetchall() +# +#BOUNDARY_COLUMNS = query(''' +#SELECT id FROM observatory.obs_column +#WHERE type ILIKE 'geometry' +#AND weight > 0 +#''').fetchall() +# +#US_CENSUS_MEASURE_COLUMNS = query(''' +#SELECT distinct numer_name +#FROM observatory.obs_meta +#WHERE numer_type ILIKE 'numeric' +#AND 'us.census.acs' = ANY (subsection_tags) +#AND numer_weight > 0 +#''').fetchall() #def default_geometry_id(column_id): @@ -233,53 +242,67 @@ def default_area(column_id): # assert_is_not_none(rows[0][0]) -@parameterized(MEASURE_COLUMNS) -def test_get_measure_points(numer_ids, numer_aggregate, section_tags): - all_in_params = [] +def grouped_measure_columns(): + for numer_ids, numer_aggregate, denom_reltype, section_tags in MEASURE_COLUMNS: + for colgroup in grouper(numer_ids, 50): + yield [c for c in colgroup if c], numer_aggregate, denom_reltype, section_tags + + +@parameterized(grouped_measure_columns()) +def test_get_measure_points(numer_ids, numer_aggregate, denom_reltype, section_tags): + _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_point(numer_ids[0])) + + +@parameterized(grouped_measure_columns()) +def test_get_measure_areas(numer_ids, numer_aggregate, denom_reltype, section_tags): + if numer_aggregate.lower() not in ('sum', 'median', 'average'): + return + if numer_aggregate.lower() in ('median', 'average') \ + and denom_reltype is not None \ + and denom_reltype.lower() != 'universe': + return + _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_area(numer_ids[0])) + + +def _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, geom): + in_params = [] for numer_id in numer_ids: - all_in_params.append({ + in_params.append({ 'numer_id': numer_id, 'normalization': 'predenominated' }) - for in_params in grouper(all_in_params, 50): - print('{} {}'.format(numer_aggregate, section_tags)) - in_params = [ip for ip in in_params if ip] - params = query(u''' - SELECT {schema}OBS_GetMeta({point}, '{in_params}') - '''.format(schema='cdb_observatory.' if USE_SCHEMA else '', - point=default_point(numer_ids[0]), - in_params=json.dumps(in_params))).fetchone()[0] - try: - # We can get duplicate IDs from multi-denominators - params = OrderedDict([(p['id'], p) for p in params]).values() - assert_equal(len(params), len(in_params)) - except: - import pdb - pdb.set_trace() - resp = query(u''' - SELECT * FROM {schema}OBS_GetData(ARRAY[({point}, 1)::geomval], '{params}') - '''.format(schema='cdb_observatory.' if USE_SCHEMA else '', - point=default_point(numer_ids[0]), - params=json.dumps(params).replace(u"'", "''"))).fetchone()[1] - vals = [v['value'] for v in resp] - assert_equal(len(vals), len(in_params)) - for i, val in enumerate(vals): - try: - assert_is_not_none(val) - except: - import pdb - pdb.set_trace() - print(val) - raise - #resp = query(''' - #SELECT * FROM {schema}OBS_GetMeasure({point}, '{column_id}') - # '''.format(column_id=column_id, - # schema='cdb_observatory.' if USE_SCHEMA else '', - # point=default_point(column_id))) - #rows = resp.fetchall() - #assert_equal(1, len(rows)) - #assert_is_not_none(rows[0][0]) + params = query(u''' + SELECT {schema}OBS_GetMeta({geom}, '{in_params}') + '''.format(schema='cdb_observatory.' if USE_SCHEMA else '', + geom=geom, + in_params=json.dumps(in_params))).fetchone()[0] + + # We can get duplicate IDs from multi-denominators, so for now we + # compress those measures into a single + params = OrderedDict([(p['id'], p) for p in params]).values() + assert_equal(len(params), len(in_params), + 'Inconsistent out and in params for {}'.format(in_params)) + + q = u''' + SELECT * FROM {schema}OBS_GetData(ARRAY[({geom}, 1)::geomval], '{params}') + '''.format(schema='cdb_observatory.' if USE_SCHEMA else '', + geom=geom, + params=json.dumps(params).replace(u"'", "''")) + resp = query(q).fetchone() + #try: + assert_is_not_none(resp, 'NULL returned for {}'.format(in_params)) + #except: + # #import pdb + # #pdb.set_trace() + # raise + rawvals = resp[1] + vals = [v['value'] for v in rawvals] + + assert_equal(len(vals), len(in_params)) + for i, val in enumerate(vals): + assert_is_not_none(val, 'NULL for {}'.format(in_params[i]['numer_id'])) + #@parameterized(CATEGORY_COLUMNS) #def test_get_category_areas(column_id):