remove notices from getgeometryscores
This commit is contained in:
John Krauss
@@ -450,18 +450,17 @@ BEGIN
|
||||
AND (column_id = ANY($2) OR cardinality($2) = 0)
|
||||
), clipped_geom_countagg AS (
|
||||
SELECT column_id, table_id
|
||||
, ST_CountAgg(clipped_tile, 1, True)::Numeric notnull_pixels -- -10
|
||||
, BOOL_AND(ST_BandIsNoData(clipped_tile, 1)) nodata
|
||||
, ST_CountAgg(clipped_tile, 1, False)::Numeric pixels -- -10
|
||||
FROM clipped_geom
|
||||
GROUP BY column_id, table_id
|
||||
), clipped_geom_reagg AS (
|
||||
SELECT COUNT(*)::BIGINT cnt, a.column_id, a.table_id,
|
||||
cdb_observatory.FIRST(nodata) first_nodata,
|
||||
cdb_observatory.FIRST(pixels) first_pixel,
|
||||
cdb_observatory.FIRST(notnull_pixels) first_notnull_pixel,
|
||||
cdb_observatory.FIRST(tile) first_tile,
|
||||
(ST_SummaryStatsAgg(clipped_tile, 1, True)).sum::Numeric sum_geoms, -- ND
|
||||
(ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND
|
||||
--(ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND
|
||||
(ST_SummaryStatsAgg(clipped_tile, 1, False)).sum::Numeric sum_geoms, -- ND
|
||||
(ST_SummaryStatsAgg(clipped_tile, 2, False)).mean::Numeric / 255 mean_fill --ND
|
||||
FROM clipped_geom_countagg a, clipped_geom b
|
||||
WHERE a.table_id = b.table_id
|
||||
AND a.column_id = b.column_id
|
||||
@@ -469,19 +468,15 @@ BEGIN
|
||||
), final AS (
|
||||
SELECT
|
||||
cnt, table_id, column_id
|
||||
, (CASE WHEN first_notnull_pixel > 0
|
||||
THEN first_notnull_pixel / first_pixel
|
||||
ELSE 1
|
||||
END)::Numeric
|
||||
AS notnull_percent
|
||||
, (CASE WHEN first_notnull_pixel > 0
|
||||
, NULL::Numeric AS notnull_percent
|
||||
, (CASE WHEN first_nodata IS FALSE
|
||||
THEN sum_geoms
|
||||
ELSE COALESCE(ST_Value(first_tile, 1, ST_PointOnSurface($1)), 0)
|
||||
* (ST_Area($1) / ST_Area(ST_PixelAsPolygon(first_tile, 0, 0))
|
||||
* first_pixel) -- -20
|
||||
END)::Numeric
|
||||
AS numgeoms
|
||||
, (CASE WHEN first_notnull_pixel > 0
|
||||
, (CASE WHEN first_nodata IS FALSE
|
||||
THEN mean_fill
|
||||
ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2
|
||||
END)::Numeric
|
||||
|
||||
@@ -79,7 +79,7 @@ for q in (
|
||||
|
||||
ARGS = {
|
||||
('OBS_GetMeasureByID', None): "name, 'us.census.acs.B01001002', '{}'",
|
||||
('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001', NULL, {}",
|
||||
('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001', null, {}",
|
||||
('OBS_GetMeasure', 'area'): "{}, 'us.census.acs.B01001002', 'area', {}",
|
||||
('OBS_GetMeasure', 'denominator'): "{}, 'us.census.acs.B01001002', 'denominator', {}",
|
||||
('OBS_GetCategory', None): "{}, 'us.census.spielman_singleton_segments.X10', {}",
|
||||
@@ -101,147 +101,148 @@ def record(params, results):
|
||||
}
|
||||
json.dump(tests, fhandle)
|
||||
|
||||
#@parameterized([
|
||||
# ('simple', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
# ('simple', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
# ('simple', '_OBS_GetGeometryScores', 'NULL', 3000),
|
||||
#
|
||||
# ('complex', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
# ('complex', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
# ('complex', '_OBS_GetGeometryScores', 'NULL', 3000),
|
||||
#
|
||||
# ('country_simple', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
# ('country_simple', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
# ('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000),
|
||||
#
|
||||
# ('country_complex', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
# ('country_complex', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
# ('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000),
|
||||
#])
|
||||
#def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms):
|
||||
# print api_method, geom_complexity, filters, target_geoms
|
||||
#
|
||||
# rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,)
|
||||
# results = []
|
||||
# for rows in rownums:
|
||||
# stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms})
|
||||
# FROM obs_perftest_{complexity}
|
||||
# WHERE cartodb_id <= {n}'''.format(
|
||||
# complexity=geom_complexity,
|
||||
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
# api_method=api_method,
|
||||
# filters=filters,
|
||||
# target_geoms=target_geoms,
|
||||
# n=rows)
|
||||
# start = time()
|
||||
# query(stmt)
|
||||
# end = time()
|
||||
# qps = (rows / (end - start))
|
||||
# results.append({
|
||||
# 'rows': rows,
|
||||
# 'qps': qps,
|
||||
# 'stmt': stmt
|
||||
# })
|
||||
# print rows, ': ', qps, ' QPS'
|
||||
#
|
||||
# if 'OBS_RECORD_TEST' in os.environ:
|
||||
# record({
|
||||
# 'geom_complexity': geom_complexity,
|
||||
# 'api_method': api_method,
|
||||
# 'filters': filters,
|
||||
# 'target_geoms': target_geoms
|
||||
# }, results)
|
||||
#
|
||||
#@parameterized([
|
||||
# ('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None),
|
||||
# ('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None),
|
||||
#
|
||||
# ('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetCategory', None, 'point', 'NULL'),
|
||||
# ('simple', 'OBS_GetCategory', None, 'geom', 'NULL'),
|
||||
# ('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
|
||||
#
|
||||
# ('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
|
||||
# ('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
#
|
||||
# ('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetCategory', None, 'point', 'NULL'),
|
||||
# ('complex', 'OBS_GetCategory', None, 'geom', 'NULL'),
|
||||
# ('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
|
||||
#
|
||||
# ('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"),
|
||||
# ('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
|
||||
# ('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
|
||||
# ('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
#])
|
||||
#def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary):
|
||||
# print api_method, geom_complexity, normalization, geom, boundary
|
||||
# col = 'measure' if 'measure' in api_method.lower() else 'category'
|
||||
# results = []
|
||||
#
|
||||
# rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, )
|
||||
# for rows in rownums:
|
||||
# stmt = '''UPDATE obs_perftest_{complexity}
|
||||
# SET {col} = {schema}{api_method}({args})
|
||||
# WHERE cartodb_id <= {n}'''.format(
|
||||
# col=col,
|
||||
# complexity=geom_complexity,
|
||||
# schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
# api_method=api_method,
|
||||
# args=ARGS[api_method, normalization].format(geom, boundary),
|
||||
# n=rows)
|
||||
# start = time()
|
||||
# query(stmt)
|
||||
# end = time()
|
||||
# qps = (rows / (end - start))
|
||||
# results.append({
|
||||
# 'rows': rows,
|
||||
# 'qps': qps,
|
||||
# 'stmt': stmt
|
||||
# })
|
||||
# print rows, ': ', qps, ' QPS'
|
||||
#
|
||||
# if 'OBS_RECORD_TEST' in os.environ:
|
||||
# record({
|
||||
# 'geom_complexity': geom_complexity,
|
||||
# 'api_method': api_method,
|
||||
# 'normalization': normalization,
|
||||
# 'geom': geom
|
||||
# }, results)
|
||||
@parameterized([
|
||||
('simple', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('simple', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('simple', '_OBS_GetGeometryScores', 'NULL', 3000),
|
||||
|
||||
('complex', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('complex', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('complex', '_OBS_GetGeometryScores', 'NULL', 3000),
|
||||
|
||||
('country_simple', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('country_simple', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000),
|
||||
|
||||
('country_complex', '_OBS_GetGeometryScores', 'NULL', 1),
|
||||
('country_complex', '_OBS_GetGeometryScores', 'NULL', 500),
|
||||
('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000),
|
||||
])
|
||||
def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms):
|
||||
print api_method, geom_complexity, filters, target_geoms
|
||||
|
||||
rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,)
|
||||
results = []
|
||||
for rows in rownums:
|
||||
stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms})
|
||||
FROM obs_perftest_{complexity}
|
||||
WHERE cartodb_id <= {n}'''.format(
|
||||
complexity=geom_complexity,
|
||||
schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
api_method=api_method,
|
||||
filters=filters,
|
||||
target_geoms=target_geoms,
|
||||
n=rows)
|
||||
start = time()
|
||||
query(stmt)
|
||||
end = time()
|
||||
qps = (rows / (end - start))
|
||||
results.append({
|
||||
'rows': rows,
|
||||
'qps': qps,
|
||||
'stmt': stmt
|
||||
})
|
||||
print rows, ': ', qps, ' QPS'
|
||||
|
||||
if 'OBS_RECORD_TEST' in os.environ:
|
||||
record({
|
||||
'geom_complexity': geom_complexity,
|
||||
'api_method': api_method,
|
||||
'filters': filters,
|
||||
'target_geoms': target_geoms
|
||||
}, results)
|
||||
|
||||
@parameterized([
|
||||
('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None),
|
||||
('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None),
|
||||
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
|
||||
('simple', 'OBS_GetCategory', None, 'point', 'NULL'),
|
||||
('simple', 'OBS_GetCategory', None, 'geom', 'NULL'),
|
||||
('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
|
||||
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
|
||||
('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'),
|
||||
('complex', 'OBS_GetCategory', None, 'point', 'NULL'),
|
||||
('complex', 'OBS_GetCategory', None, 'geom', 'NULL'),
|
||||
('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'),
|
||||
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"),
|
||||
('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"),
|
||||
('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"),
|
||||
('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"),
|
||||
])
|
||||
def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary):
|
||||
print api_method, geom_complexity, normalization, geom, boundary
|
||||
col = 'measure' if 'measure' in api_method.lower() else 'category'
|
||||
results = []
|
||||
|
||||
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, )
|
||||
for rows in rownums:
|
||||
stmt = '''UPDATE obs_perftest_{complexity}
|
||||
SET {col} = {schema}{api_method}({args})
|
||||
WHERE cartodb_id <= {n}'''.format(
|
||||
col=col,
|
||||
complexity=geom_complexity,
|
||||
schema='cdb_observatory.' if USE_SCHEMA else '',
|
||||
api_method=api_method,
|
||||
args=ARGS[api_method, normalization].format(geom, boundary),
|
||||
n=rows)
|
||||
start = time()
|
||||
query(stmt)
|
||||
end = time()
|
||||
qps = (rows / (end - start))
|
||||
results.append({
|
||||
'rows': rows,
|
||||
'qps': qps,
|
||||
'stmt': stmt
|
||||
})
|
||||
print rows, ': ', qps, ' QPS'
|
||||
|
||||
if 'OBS_RECORD_TEST' in os.environ:
|
||||
record({
|
||||
'geom_complexity': geom_complexity,
|
||||
'api_method': api_method,
|
||||
'normalization': normalization,
|
||||
'boundary': boundary,
|
||||
'geom': geom
|
||||
}, results)
|
||||
|
||||
|
||||
@parameterized([
|
||||
@@ -289,7 +290,7 @@ def test_getmeasure_split_performance(geom_complexity, normalization, geom, boun
|
||||
print geom_complexity, normalization, geom, boundary
|
||||
results = []
|
||||
|
||||
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (100, 500, 1000)
|
||||
rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (10, 50, 100)
|
||||
for rows in rownums:
|
||||
stmt = '''
|
||||
with data as (
|
||||
|
||||
Reference in New Issue
Block a user