374 lines
14 KiB
Python
374 lines
14 KiB
Python
from nose.tools import assert_equal, assert_is_not_none
|
|
from nose.plugins.skip import SkipTest
|
|
from nose_parameterized import parameterized
|
|
|
|
from itertools import izip_longest
|
|
from util import query
|
|
from collections import OrderedDict
|
|
import json
|
|
|
|
|
|
def grouper(iterable, n, fillvalue=None):
|
|
"Collect data into fixed-length chunks or blocks"
|
|
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
|
|
args = [iter(iterable)] * n
|
|
return izip_longest(fillvalue=fillvalue, *args)
|
|
|
|
|
|
USE_SCHEMA = True
|
|
|
|
SKIP_COLUMNS = set([
|
|
u'mx.inegi_columns.INDI18',
|
|
u'mx.inegi_columns.ECO40',
|
|
u'mx.inegi_columns.POB34',
|
|
u'mx.inegi_columns.POB63',
|
|
u'mx.inegi_columns.INDI7',
|
|
u'mx.inegi_columns.EDU28',
|
|
u'mx.inegi_columns.SCONY10',
|
|
u'mx.inegi_columns.EDU31',
|
|
u'mx.inegi_columns.POB7',
|
|
u'mx.inegi_columns.VIV30',
|
|
u'mx.inegi_columns.INDI12',
|
|
u'mx.inegi_columns.EDU13',
|
|
u'mx.inegi_columns.ECO43',
|
|
u'mx.inegi_columns.VIV9',
|
|
u'mx.inegi_columns.HOGAR25',
|
|
u'mx.inegi_columns.POB32',
|
|
u'mx.inegi_columns.ECO7',
|
|
u'mx.inegi_columns.INDI19',
|
|
u'mx.inegi_columns.INDI16',
|
|
u'mx.inegi_columns.POB65',
|
|
u'mx.inegi_columns.INDI3',
|
|
u'mx.inegi_columns.INDI9',
|
|
u'mx.inegi_columns.POB36',
|
|
u'mx.inegi_columns.POB33',
|
|
u'mx.inegi_columns.POB58',
|
|
u'mx.inegi_columns.DISC4',
|
|
u'mx.inegi_columns.VIV41',
|
|
u'mx.inegi_columns.VIV40',
|
|
u'mx.inegi_columns.VIV17',
|
|
u'mx.inegi_columns.VIV25',
|
|
u'mx.inegi_columns.EDU10',
|
|
u'whosonfirst.wof_disputed_name',
|
|
u'us.census.tiger.fullname',
|
|
u'whosonfirst.wof_marinearea_name',
|
|
u'us.census.tiger.mtfcc',
|
|
u'whosonfirst.wof_county_name',
|
|
u'whosonfirst.wof_region_name',
|
|
'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
|
|
, 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
|
|
, 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
|
|
, 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
|
|
, 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
|
|
, 'fr.insee.P12_ACTOCC15P_ILT45D'
|
|
, 'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
|
|
, 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
|
|
, 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
|
|
, 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
|
|
, 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
|
|
, 'fr.insee.P12_ACTOCC15P_ILT45D'
|
|
, 'uk.ons.LC3202WA0007'
|
|
, 'uk.ons.LC3202WA0010'
|
|
, 'uk.ons.LC3202WA0004'
|
|
, 'uk.ons.LC3204WA0004'
|
|
, 'uk.ons.LC3204WA0007'
|
|
, 'uk.ons.LC3204WA0010'
|
|
, 'br.geo.subdistritos_name'
|
|
])
|
|
|
|
MEASURE_COLUMNS = query('''
|
|
SELECT ARRAY_AGG(DISTINCT numer_id) numer_ids,
|
|
numer_aggregate,
|
|
denom_reltype,
|
|
section_tags
|
|
FROM observatory.obs_meta
|
|
WHERE numer_weight > 0
|
|
AND numer_id NOT IN ('{skip}')
|
|
AND section_tags IS NOT NULL
|
|
AND subsection_tags IS NOT NULL
|
|
GROUP BY numer_aggregate, section_tags, denom_reltype
|
|
'''.format(skip="', '".join(SKIP_COLUMNS))).fetchall()
|
|
|
|
#CATEGORY_COLUMNS = query('''
|
|
#SELECT distinct numer_id
|
|
#FROM observatory.obs_meta
|
|
#WHERE numer_type ILIKE 'text'
|
|
#AND numer_weight > 0
|
|
#''').fetchall()
|
|
#
|
|
#BOUNDARY_COLUMNS = query('''
|
|
#SELECT id FROM observatory.obs_column
|
|
#WHERE type ILIKE 'geometry'
|
|
#AND weight > 0
|
|
#''').fetchall()
|
|
#
|
|
#US_CENSUS_MEASURE_COLUMNS = query('''
|
|
#SELECT distinct numer_name
|
|
#FROM observatory.obs_meta
|
|
#WHERE numer_type ILIKE 'numeric'
|
|
#AND 'us.census.acs' = ANY (subsection_tags)
|
|
#AND numer_weight > 0
|
|
#''').fetchall()
|
|
|
|
|
|
#def default_geometry_id(column_id):
|
|
# '''
|
|
# Returns default test point for the column_id.
|
|
# '''
|
|
# if column_id == 'whosonfirst.wof_disputed_geom':
|
|
# return 'ST_SetSRID(ST_MakePoint(76.57, 33.78), 4326)'
|
|
# elif column_id == 'whosonfirst.wof_marinearea_geom':
|
|
# return 'ST_SetSRID(ST_MakePoint(-68.47, 43.33), 4326)'
|
|
# elif column_id in ('us.census.tiger.school_district_elementary',
|
|
# 'us.census.tiger.school_district_secondary',
|
|
# 'us.census.tiger.school_district_elementary_clipped',
|
|
# 'us.census.tiger.school_district_secondary_clipped'):
|
|
# return 'ST_SetSRID(ST_MakePoint(-73.7067, 40.7025), 4326)'
|
|
# elif column_id.startswith('es.ine'):
|
|
# return 'ST_SetSRID(ST_MakePoint(-2.51141249535454, 42.8226119029222), 4326)'
|
|
# elif column_id.startswith('us.zillow'):
|
|
# return 'ST_SetSRID(ST_MakePoint(-81.3544048197256, 28.3305906291771), 4326)'
|
|
# elif column_id.startswith('ca.'):
|
|
# return ''
|
|
# else:
|
|
# return 'ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326)'
|
|
|
|
|
|
def default_lonlat(column_id):
|
|
'''
|
|
Returns default test point for the column_id.
|
|
'''
|
|
if column_id == 'whosonfirst.wof_disputed_geom':
|
|
return (76.57, 33.78)
|
|
elif column_id == 'whosonfirst.wof_marinearea_geom':
|
|
return (-68.47, 43.33)
|
|
elif column_id in ('us.census.tiger.school_district_elementary',
|
|
'us.census.tiger.school_district_secondary',
|
|
'us.census.tiger.school_district_elementary_clipped',
|
|
'us.census.tiger.school_district_secondary_clipped'):
|
|
return (40.7025, -73.7067)
|
|
elif column_id.startswith('uk'):
|
|
if 'WA' in column_id:
|
|
return (51.46844551219723, -3.184833526611328)
|
|
else:
|
|
return (51.51461834694225, -0.08883476257324219)
|
|
elif column_id.startswith('es'):
|
|
return (42.8226119029222, -2.51141249535454)
|
|
elif column_id.startswith('us.zillow'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('mx.'):
|
|
return (19.41347699386547, -99.17019367218018)
|
|
elif column_id.startswith('th.'):
|
|
return (13.725377712079784, 100.49263000488281)
|
|
# cols for French Guyana only
|
|
#elif column_id in ('fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
|
|
# , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
|
|
# , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
|
|
# , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
|
|
# , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
|
|
# , 'fr.insee.P12_ACTOCC15P_ILT45D'
|
|
# , 'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
|
|
# , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
|
|
# , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
|
|
# , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
|
|
# , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
|
|
# , 'fr.insee.P12_ACTOCC15P_ILT45D'):
|
|
# return (4.938408371206558, -52.32908248901367)
|
|
elif column_id.startswith('fr.'):
|
|
return (48.860875144709475, 2.3613739013671875)
|
|
elif column_id.startswith('ca.'):
|
|
return (43.65594991256823, -79.37965393066406)
|
|
elif column_id.startswith('us.census.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('us.dma.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('us.ihme.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('us.bls.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('us.qcew.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('whosonfirst.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('us.epa.'):
|
|
return (28.3305906291771, -81.3544048197256)
|
|
elif column_id.startswith('eu.'):
|
|
raise SkipTest('No tests for Eurostat!')
|
|
elif column_id.startswith('br.'):
|
|
return (-23.53, -46.63)
|
|
elif column_id.startswith('au.'):
|
|
return (-33.8806, 151.2131)
|
|
else:
|
|
raise Exception('No catalog point set for {}'.format(column_id))
|
|
|
|
|
|
def default_point(column_id):
|
|
lat, lng = default_lonlat(column_id)
|
|
return 'ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326)'.format(
|
|
lat=lat, lng=lng)
|
|
|
|
|
|
def default_area(column_id):
|
|
'''
|
|
Returns default test area for the column_id
|
|
'''
|
|
point = default_point(column_id)
|
|
area = 'ST_Transform(ST_Buffer(ST_Transform({point}, 3857), 250), 4326)'.format(
|
|
point=point)
|
|
return area
|
|
|
|
#@parameterized(US_CENSUS_MEASURE_COLUMNS)
|
|
#def test_get_us_census_measure_points(name):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetUSCensusMeasure({point}, '{name}')
|
|
# '''.format(name=name.replace("'", "''"),
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# point=default_point('')))
|
|
# rows = resp.fetchall()
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
|
|
def grouped_measure_columns():
|
|
for numer_ids, numer_aggregate, denom_reltype, section_tags in MEASURE_COLUMNS:
|
|
for colgroup in grouper(numer_ids, 50):
|
|
yield [c for c in colgroup if c], numer_aggregate, denom_reltype, section_tags
|
|
|
|
|
|
@parameterized(grouped_measure_columns())
|
|
def test_get_measure_points(numer_ids, numer_aggregate, denom_reltype, section_tags):
|
|
_test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_point(numer_ids[0]))
|
|
|
|
|
|
@parameterized(grouped_measure_columns())
|
|
def test_get_measure_areas(numer_ids, numer_aggregate, denom_reltype, section_tags):
|
|
if numer_aggregate is None or numer_aggregate.lower() not in ('sum', 'median', 'average'):
|
|
return
|
|
if numer_aggregate.lower() in ('median', 'average') \
|
|
and (denom_reltype is None \
|
|
or denom_reltype.lower() != 'universe'):
|
|
return
|
|
_test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, default_area(numer_ids[0]))
|
|
|
|
|
|
def _test_measures(numer_ids, numer_aggregate, section_tags, denom_reltype, geom):
|
|
in_params = []
|
|
for numer_id in numer_ids:
|
|
in_params.append({
|
|
'numer_id': numer_id,
|
|
'normalization': 'predenominated'
|
|
})
|
|
|
|
params = query(u'''
|
|
SELECT {schema}OBS_GetMeta({geom}, '{in_params}')
|
|
'''.format(schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
geom=geom,
|
|
in_params=json.dumps(in_params))).fetchone()[0]
|
|
|
|
# We can get duplicate IDs from multi-denominators, so for now we
|
|
# compress those measures into a single
|
|
params = OrderedDict([(p['id'], p) for p in params]).values()
|
|
assert_equal(len(params), len(in_params),
|
|
'Inconsistent out and in params for {}'.format(in_params))
|
|
|
|
q = u'''
|
|
SELECT * FROM {schema}OBS_GetData(ARRAY[({geom}, 1)::geomval], '{params}')
|
|
'''.format(schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
geom=geom,
|
|
params=json.dumps(params).replace(u"'", "''"))
|
|
resp = query(q).fetchone()
|
|
assert_is_not_none(resp, 'NULL returned for {}'.format(in_params))
|
|
rawvals = resp[1]
|
|
vals = [v['value'] for v in rawvals]
|
|
|
|
assert_equal(len(vals), len(in_params))
|
|
for i, val in enumerate(vals):
|
|
assert_is_not_none(val, 'NULL for {}'.format(in_params[i]['numer_id']))
|
|
|
|
|
|
#@parameterized(CATEGORY_COLUMNS)
|
|
#def test_get_category_areas(column_id):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetCategory({area}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# area=default_area(column_id)))
|
|
# assert_equal(resp.status_code, 200)
|
|
# rows = resp.json()['rows']
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
#@parameterized(CATEGORY_COLUMNS)
|
|
#def test_get_category_points(column_id):
|
|
# if column_id in SKIP_COLUMNS:
|
|
# raise SkipTest('Column {} should be skipped'.format(column_id))
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetCategory({point}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# point=default_point(column_id)))
|
|
# rows = resp.fetchall()
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
#@parameterized(BOUNDARY_COLUMNS)
|
|
#def test_get_boundaries_by_geometry(column_id):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetBoundariesByGeometry({area}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# area=default_area(column_id)))
|
|
# assert_equal(resp.status_code, 200)
|
|
# rows = resp.json()['rows']
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
#@parameterized(BOUNDARY_COLUMNS)
|
|
#def test_get_points_by_geometry(column_id):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetPointsByGeometry({area}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# area=default_area(column_id)))
|
|
# assert_equal(resp.status_code, 200)
|
|
# rows = resp.json()['rows']
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
#@parameterized(BOUNDARY_COLUMNS)
|
|
#def test_get_boundary_points(column_id):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetBoundary({point}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# point=default_point(column_id)))
|
|
# assert_equal(resp.status_code, 200)
|
|
# rows = resp.json()['rows']
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
#@parameterized(BOUNDARY_COLUMNS)
|
|
#def test_get_boundary_id(column_id):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetBoundaryId({point}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# point=default_point(column_id)))
|
|
# assert_equal(resp.status_code, 200)
|
|
# rows = resp.json()['rows']
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|
|
#@parameterized(BOUNDARY_COLUMNS)
|
|
#def test_get_boundary_by_id(column_id):
|
|
# resp = query('''
|
|
#SELECT * FROM {schema}OBS_GetBoundaryById({geometry_id}, '{column_id}')
|
|
# '''.format(column_id=column_id,
|
|
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
|
# geometry_id=default_geometry_id(column_id)))
|
|
# assert_equal(resp.status_code, 200)
|
|
# rows = resp.json()['rows']
|
|
# assert_equal(1, len(rows))
|
|
# assert_is_not_none(rows[0][0])
|
|
|