cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed broken tests and refactor

Browse Source
This commit is contained in:
Antonio
2017-08-18 11:19:15 +02:00
parent 48ed086fec
commit fad541c3fc
1 changed files with 68 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
src/python/test/autotest.py
View File

@@ -1,5 +1,4 @@
from nose.tools import assert_equal, assert_is_not_none
from nose.plugins.skip import SkipTest
from nose_parameterized import parameterized
from itertools import izip_longest
@@ -55,38 +54,48 @@ SKIP_COLUMNS = set([
u'us.census.tiger.mtfcc',
u'whosonfirst.wof_county_name',
u'whosonfirst.wof_region_name',
'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
, 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
, 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
, 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
, 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
, 'fr.insee.P12_ACTOCC15P_ILT45D'
, 'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
, 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
, 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
, 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
, 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
, 'fr.insee.P12_ACTOCC15P_ILT45D'
, 'uk.ons.LC3202WA0007'
, 'uk.ons.LC3202WA0010'
, 'uk.ons.LC3202WA0004'
, 'uk.ons.LC3204WA0004'
, 'uk.ons.LC3204WA0007'
, 'uk.ons.LC3204WA0010'
, 'br.geo.subdistritos_name'
u'fr.insee.P12_RP_CHOS',
u'fr.insee.P12_RP_HABFOR',
u'fr.insee.P12_RP_EAUCH',
u'fr.insee.P12_RP_BDWC',
u'fr.insee.P12_RP_MIDUR',
u'fr.insee.P12_RP_CLIM',
u'fr.insee.P12_RP_MIBOIS',
u'fr.insee.P12_RP_CASE',
u'fr.insee.P12_RP_TTEGOU',
u'fr.insee.P12_RP_ELEC',
u'fr.insee.P12_ACTOCC15P_ILT45D',
u'fr.insee.P12_RP_CHOS',
u'fr.insee.P12_RP_HABFOR',
u'fr.insee.P12_RP_EAUCH',
u'fr.insee.P12_RP_BDWC',
u'fr.insee.P12_RP_MIDUR',
u'fr.insee.P12_RP_CLIM',
u'fr.insee.P12_RP_MIBOIS',
u'fr.insee.P12_RP_CASE',
u'fr.insee.P12_RP_TTEGOU',
u'fr.insee.P12_RP_ELEC',
u'fr.insee.P12_ACTOCC15P_ILT45D',
u'uk.ons.LC3202WA0007',
u'uk.ons.LC3202WA0010',
u'uk.ons.LC3202WA0004',
u'uk.ons.LC3204WA0004',
u'uk.ons.LC3204WA0007',
u'uk.ons.LC3204WA0010',
u'br.geo.subdistritos_name'
])
MEASURE_COLUMNS = query('''
SELECT FIRST(distinct numer_id) numer_ids,
numer_aggregate,
denom_reltype,
section_tags
denom_reltype
FROM observatory.obs_meta
WHERE numer_weight > 0
AND numer_id NOT IN ('{skip}')
AND numer_id NOT LIKE 'eu.%' --Skipping Eurostat
AND section_tags IS NOT NULL
AND subsection_tags IS NOT NULL
GROUP BY numer_id, numer_aggregate, section_tags, denom_reltype
GROUP BY numer_id, numer_aggregate, denom_reltype
'''.format(skip="', '".join(SKIP_COLUMNS))).fetchall()
@@ -109,22 +118,6 @@ def default_lonlat(column_id):
return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('mx.'):
return (19.41347699386547, -99.17019367218018)
elif column_id.startswith('th.'):
return (13.725377712079784, 100.49263000488281)
# cols for French Guyana only
#elif column_id in ('fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
# , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
# , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
# , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
# , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
# , 'fr.insee.P12_ACTOCC15P_ILT45D'
# , 'fr.insee.P12_RP_CHOS', 'fr.insee.P12_RP_HABFOR'
# , 'fr.insee.P12_RP_EAUCH', 'fr.insee.P12_RP_BDWC'
# , 'fr.insee.P12_RP_MIDUR', 'fr.insee.P12_RP_CLIM'
# , 'fr.insee.P12_RP_MIBOIS', 'fr.insee.P12_RP_CASE'
# , 'fr.insee.P12_RP_TTEGOU', 'fr.insee.P12_RP_ELEC'
# , 'fr.insee.P12_ACTOCC15P_ILT45D'):
# return (4.938408371206558, -52.32908248901367)
elif column_id.startswith('fr.'):
return (48.860875144709475, 2.3613739013671875)
elif column_id.startswith('ca.'):
@@ -132,12 +125,12 @@ def default_lonlat(column_id):
elif column_id in ('us.census.tiger.school_district_elementary',
'us.census.tiger.school_district_secondary',
'us.census.tiger.school_district_elementary_clipped',
'us.census.tiger.school_district_secondary_clipped'):
'us.census.tiger.school_district_secondary_clipped',
'us.census.tiger.school_district_elementary_geoname',
'us.census.tiger.school_district_secondary_geoname'):
return (40.7025, -73.7067)
elif column_id.startswith('us.census.'):
return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('us.dma.'):
return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('us.ihme.'):
return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('us.bls.'):
@@ -148,8 +141,6 @@ def default_lonlat(column_id):
return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('us.epa.'):
return (28.3305906291771, -81.3544048197256)
elif column_id.startswith('eu.'):
raise SkipTest('No tests for Eurostat!')
elif column_id.startswith('br.'):
return (-23.53, -46.63)
elif column_id.startswith('au.'):
@@ -158,49 +149,62 @@ def default_lonlat(column_id):
raise Exception('No catalog point set for {}'.format(column_id))
def default_point(column_id):
lat, lng = default_lonlat(column_id)
def default_point(test_point):
lat, lng = test_point
return 'ST_SetSRID(ST_MakePoint({lng}, {lat}), 4326)'.format(
lat=lat, lng=lng)
def default_area(column_id):
def default_area(test_point):
'''
Returns default test area for the column_id
'''
point = default_point(column_id)
point = default_point(test_point)
area = 'ST_Transform(ST_Buffer(ST_Transform({point}, 3857), 250), 4326)'.format(
point=point)
return area
def grouped_measure_columns():
def filter_points():
return MEASURE_COLUMNS
def filter_areas():
filtered = []
for numer_ids, numer_aggregate, denom_reltype in MEASURE_COLUMNS:
if numer_aggregate is None or numer_aggregate.lower() not in ('sum', 'median', 'average'):
continue
if numer_aggregate.lower() in ('median', 'average') \
and (denom_reltype is None or denom_reltype.lower() != 'universe'):
continue
filtered.append((numer_ids, numer_aggregate, denom_reltype))
return filtered
def grouped_measure_columns(filtered_columns):
groupbypoint = dict()
for numer_ids, numer_aggregate, denom_reltype, section_tags in MEASURE_COLUMNS:
for row in filtered_columns:
numer_ids = row[0]
point = default_lonlat(numer_ids)
if point in groupbypoint:
groupbypoint[point].append(numer_ids)
else:
groupbypoint[point] = [numer_ids]
for key, value in groupbypoint.iteritems():
for colgroup in grouper(value, 50):
yield [c for c in colgroup if c], numer_aggregate, denom_reltype, section_tags
for point, numer_ids in groupbypoint.iteritems():
for colgroup in grouper(numer_ids, 50):
yield point, [c for c in colgroup if c]
@parameterized(grouped_measure_columns())
def test_get_measure_points(numer_ids, numer_aggregate, denom_reltype, section_tags):
_test_measures(numer_ids, default_point(numer_ids[0]))
@parameterized(grouped_measure_columns(filter_points()))
def test_get_measure_points(point, numer_ids):
_test_measures(numer_ids, default_point(point))
@parameterized(grouped_measure_columns())
def test_get_measure_areas(numer_ids, numer_aggregate, denom_reltype, section_tags):
if numer_aggregate is None or numer_aggregate.lower() not in ('sum', 'median', 'average'):
return
if numer_aggregate.lower() in ('median', 'average') \
and (denom_reltype is None or denom_reltype.lower() != 'universe'):
return
_test_measures(numer_ids, default_area(numer_ids[0]))
@parameterized(grouped_measure_columns(filter_areas()))
def test_get_measure_areas(point, numer_ids):
_test_measures(numer_ids, default_area(point))
def _test_measures(numer_ids, geom):