From fa96de5aa9d9d16d99e715accdb71d37339b7b8e Mon Sep 17 00:00:00 2001 From: John Krauss Date: Wed, 28 Dec 2016 15:57:56 +0000 Subject: [PATCH] remove notices from getgeometryscores --- src/pg/sql/42_observatory_exploration.sql | 19 +- src/python/test/perftest.py | 287 +++++++++++----------- 2 files changed, 151 insertions(+), 155 deletions(-) diff --git a/src/pg/sql/42_observatory_exploration.sql b/src/pg/sql/42_observatory_exploration.sql index 73fcb28..a46f24c 100644 --- a/src/pg/sql/42_observatory_exploration.sql +++ b/src/pg/sql/42_observatory_exploration.sql @@ -450,18 +450,17 @@ BEGIN AND (column_id = ANY($2) OR cardinality($2) = 0) ), clipped_geom_countagg AS ( SELECT column_id, table_id - , ST_CountAgg(clipped_tile, 1, True)::Numeric notnull_pixels -- -10 + , BOOL_AND(ST_BandIsNoData(clipped_tile, 1)) nodata , ST_CountAgg(clipped_tile, 1, False)::Numeric pixels -- -10 FROM clipped_geom GROUP BY column_id, table_id ), clipped_geom_reagg AS ( SELECT COUNT(*)::BIGINT cnt, a.column_id, a.table_id, + cdb_observatory.FIRST(nodata) first_nodata, cdb_observatory.FIRST(pixels) first_pixel, - cdb_observatory.FIRST(notnull_pixels) first_notnull_pixel, cdb_observatory.FIRST(tile) first_tile, - (ST_SummaryStatsAgg(clipped_tile, 1, True)).sum::Numeric sum_geoms, -- ND - (ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND - --(ST_SummaryStatsAgg(clipped_tile, 2, True)).mean::Numeric / 255 mean_fill --ND + (ST_SummaryStatsAgg(clipped_tile, 1, False)).sum::Numeric sum_geoms, -- ND + (ST_SummaryStatsAgg(clipped_tile, 2, False)).mean::Numeric / 255 mean_fill --ND FROM clipped_geom_countagg a, clipped_geom b WHERE a.table_id = b.table_id AND a.column_id = b.column_id @@ -469,19 +468,15 @@ BEGIN ), final AS ( SELECT cnt, table_id, column_id - , (CASE WHEN first_notnull_pixel > 0 - THEN first_notnull_pixel / first_pixel - ELSE 1 - END)::Numeric - AS notnull_percent - , (CASE WHEN first_notnull_pixel > 0 + , NULL::Numeric AS notnull_percent + , (CASE WHEN first_nodata IS FALSE THEN sum_geoms ELSE COALESCE(ST_Value(first_tile, 1, ST_PointOnSurface($1)), 0) * (ST_Area($1) / ST_Area(ST_PixelAsPolygon(first_tile, 0, 0)) * first_pixel) -- -20 END)::Numeric AS numgeoms - , (CASE WHEN first_notnull_pixel > 0 + , (CASE WHEN first_nodata IS FALSE THEN mean_fill ELSE COALESCE(ST_Value(first_tile, 2, ST_PointOnSurface($1))::Numeric / 255, 0) -- -2 END)::Numeric diff --git a/src/python/test/perftest.py b/src/python/test/perftest.py index d9f14c5..3ffffe8 100644 --- a/src/python/test/perftest.py +++ b/src/python/test/perftest.py @@ -79,7 +79,7 @@ for q in ( ARGS = { ('OBS_GetMeasureByID', None): "name, 'us.census.acs.B01001002', '{}'", - ('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001', NULL, {}", + ('OBS_GetMeasure', 'predenominated'): "{}, 'us.census.acs.B01003001', null, {}", ('OBS_GetMeasure', 'area'): "{}, 'us.census.acs.B01001002', 'area', {}", ('OBS_GetMeasure', 'denominator'): "{}, 'us.census.acs.B01001002', 'denominator', {}", ('OBS_GetCategory', None): "{}, 'us.census.spielman_singleton_segments.X10', {}", @@ -101,147 +101,148 @@ def record(params, results): } json.dump(tests, fhandle) -#@parameterized([ -# ('simple', '_OBS_GetGeometryScores', 'NULL', 1), -# ('simple', '_OBS_GetGeometryScores', 'NULL', 500), -# ('simple', '_OBS_GetGeometryScores', 'NULL', 3000), -# -# ('complex', '_OBS_GetGeometryScores', 'NULL', 1), -# ('complex', '_OBS_GetGeometryScores', 'NULL', 500), -# ('complex', '_OBS_GetGeometryScores', 'NULL', 3000), -# -# ('country_simple', '_OBS_GetGeometryScores', 'NULL', 1), -# ('country_simple', '_OBS_GetGeometryScores', 'NULL', 500), -# ('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000), -# -# ('country_complex', '_OBS_GetGeometryScores', 'NULL', 1), -# ('country_complex', '_OBS_GetGeometryScores', 'NULL', 500), -# ('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000), -#]) -#def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms): -# print api_method, geom_complexity, filters, target_geoms -# -# rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,) -# results = [] -# for rows in rownums: -# stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms}) -# FROM obs_perftest_{complexity} -# WHERE cartodb_id <= {n}'''.format( -# complexity=geom_complexity, -# schema='cdb_observatory.' if USE_SCHEMA else '', -# api_method=api_method, -# filters=filters, -# target_geoms=target_geoms, -# n=rows) -# start = time() -# query(stmt) -# end = time() -# qps = (rows / (end - start)) -# results.append({ -# 'rows': rows, -# 'qps': qps, -# 'stmt': stmt -# }) -# print rows, ': ', qps, ' QPS' -# -# if 'OBS_RECORD_TEST' in os.environ: -# record({ -# 'geom_complexity': geom_complexity, -# 'api_method': api_method, -# 'filters': filters, -# 'target_geoms': target_geoms -# }, results) -# -#@parameterized([ -# ('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None), -# ('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None), -# -# ('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'), -# ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'), -# ('simple', 'OBS_GetCategory', None, 'point', 'NULL'), -# ('simple', 'OBS_GetCategory', None, 'geom', 'NULL'), -# ('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'), -# -# ('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"), -# ('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"), -# -# ('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'), -# ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'), -# ('complex', 'OBS_GetCategory', None, 'point', 'NULL'), -# ('complex', 'OBS_GetCategory', None, 'geom', 'NULL'), -# ('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'), -# -# ('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"), -# ('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"), -# ('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"), -# ('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"), -#]) -#def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary): -# print api_method, geom_complexity, normalization, geom, boundary -# col = 'measure' if 'measure' in api_method.lower() else 'category' -# results = [] -# -# rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, ) -# for rows in rownums: -# stmt = '''UPDATE obs_perftest_{complexity} -# SET {col} = {schema}{api_method}({args}) -# WHERE cartodb_id <= {n}'''.format( -# col=col, -# complexity=geom_complexity, -# schema='cdb_observatory.' if USE_SCHEMA else '', -# api_method=api_method, -# args=ARGS[api_method, normalization].format(geom, boundary), -# n=rows) -# start = time() -# query(stmt) -# end = time() -# qps = (rows / (end - start)) -# results.append({ -# 'rows': rows, -# 'qps': qps, -# 'stmt': stmt -# }) -# print rows, ': ', qps, ' QPS' -# -# if 'OBS_RECORD_TEST' in os.environ: -# record({ -# 'geom_complexity': geom_complexity, -# 'api_method': api_method, -# 'normalization': normalization, -# 'geom': geom -# }, results) +@parameterized([ + ('simple', '_OBS_GetGeometryScores', 'NULL', 1), + ('simple', '_OBS_GetGeometryScores', 'NULL', 500), + ('simple', '_OBS_GetGeometryScores', 'NULL', 3000), + + ('complex', '_OBS_GetGeometryScores', 'NULL', 1), + ('complex', '_OBS_GetGeometryScores', 'NULL', 500), + ('complex', '_OBS_GetGeometryScores', 'NULL', 3000), + + ('country_simple', '_OBS_GetGeometryScores', 'NULL', 1), + ('country_simple', '_OBS_GetGeometryScores', 'NULL', 500), + ('country_simple', '_OBS_GetGeometryScores', 'NULL', 5000), + + ('country_complex', '_OBS_GetGeometryScores', 'NULL', 1), + ('country_complex', '_OBS_GetGeometryScores', 'NULL', 500), + ('country_complex', '_OBS_GetGeometryScores', 'NULL', 5000), +]) +def test_getgeometryscores_performance(geom_complexity, api_method, filters, target_geoms): + print api_method, geom_complexity, filters, target_geoms + + rownums = (1, 5, 10, ) if 'complex' in geom_complexity else (5, 25, 50,) + results = [] + for rows in rownums: + stmt = '''SELECT {schema}{api_method}(geom, {filters}, {target_geoms}) + FROM obs_perftest_{complexity} + WHERE cartodb_id <= {n}'''.format( + complexity=geom_complexity, + schema='cdb_observatory.' if USE_SCHEMA else '', + api_method=api_method, + filters=filters, + target_geoms=target_geoms, + n=rows) + start = time() + query(stmt) + end = time() + qps = (rows / (end - start)) + results.append({ + 'rows': rows, + 'qps': qps, + 'stmt': stmt + }) + print rows, ': ', qps, ' QPS' + + if 'OBS_RECORD_TEST' in os.environ: + record({ + 'geom_complexity': geom_complexity, + 'api_method': api_method, + 'filters': filters, + 'target_geoms': target_geoms + }, results) + +@parameterized([ + ('simple', 'OBS_GetMeasureByID', None, 'us.census.tiger.census_tract', None), + ('complex', 'OBS_GetMeasureByID', None, 'us.census.tiger.county', None), + + ('simple', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'), + ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'), + ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'), + ('simple', 'OBS_GetMeasure', 'area', 'point', 'NULL'), + ('simple', 'OBS_GetMeasure', 'area', 'geom', 'NULL'), + ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'), + ('simple', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'), + ('simple', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'), + ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'), + ('simple', 'OBS_GetCategory', None, 'point', 'NULL'), + ('simple', 'OBS_GetCategory', None, 'geom', 'NULL'), + ('simple', 'OBS_GetCategory', None, 'offset_geom', 'NULL'), + + ('simple', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"), + ('simple', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"), + + ('complex', 'OBS_GetMeasure', 'predenominated', 'point', 'NULL'), + ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', 'NULL'), + ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', 'NULL'), + ('complex', 'OBS_GetMeasure', 'area', 'point', 'NULL'), + ('complex', 'OBS_GetMeasure', 'area', 'geom', 'NULL'), + ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', 'NULL'), + ('complex', 'OBS_GetMeasure', 'denominator', 'point', 'NULL'), + ('complex', 'OBS_GetMeasure', 'denominator', 'geom', 'NULL'), + ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', 'NULL'), + ('complex', 'OBS_GetCategory', None, 'point', 'NULL'), + ('complex', 'OBS_GetCategory', None, 'geom', 'NULL'), + ('complex', 'OBS_GetCategory', None, 'offset_geom', 'NULL'), + + ('complex', 'OBS_GetMeasure', 'predenominated', 'point', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'predenominated', 'geom', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'predenominated', 'offset_geom', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'area', 'point', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'area', 'geom', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'area', 'offset_geom', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'denominator', 'point', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'denominator', 'geom', "'us.census.tiger.county'"), + ('complex', 'OBS_GetMeasure', 'denominator', 'offset_geom', "'us.census.tiger.county'"), + ('complex', 'OBS_GetCategory', None, 'point', "'us.census.tiger.census_tract'"), + ('complex', 'OBS_GetCategory', None, 'geom', "'us.census.tiger.census_tract'"), + ('complex', 'OBS_GetCategory', None, 'offset_geom', "'us.census.tiger.census_tract'"), +]) +def test_getmeasure_performance(geom_complexity, api_method, normalization, geom, boundary): + print api_method, geom_complexity, normalization, geom, boundary + col = 'measure' if 'measure' in api_method.lower() else 'category' + results = [] + + rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (5, 25, 50, ) + for rows in rownums: + stmt = '''UPDATE obs_perftest_{complexity} + SET {col} = {schema}{api_method}({args}) + WHERE cartodb_id <= {n}'''.format( + col=col, + complexity=geom_complexity, + schema='cdb_observatory.' if USE_SCHEMA else '', + api_method=api_method, + args=ARGS[api_method, normalization].format(geom, boundary), + n=rows) + start = time() + query(stmt) + end = time() + qps = (rows / (end - start)) + results.append({ + 'rows': rows, + 'qps': qps, + 'stmt': stmt + }) + print rows, ': ', qps, ' QPS' + + if 'OBS_RECORD_TEST' in os.environ: + record({ + 'geom_complexity': geom_complexity, + 'api_method': api_method, + 'normalization': normalization, + 'boundary': boundary, + 'geom': geom + }, results) @parameterized([ @@ -289,7 +290,7 @@ def test_getmeasure_split_performance(geom_complexity, normalization, geom, boun print geom_complexity, normalization, geom, boundary results = [] - rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (100, 500, 1000) + rownums = (1, 5, 10, ) if geom_complexity == 'complex' else (10, 50, 100) for rows in rownums: stmt = ''' with data as (