From 921ef46eb86aab037fa5f6d2ff17cdc90792da0c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 16 May 2017 15:59:34 +0200 Subject: [PATCH 1/5] Use contrib_regression as obs backend #366 --- server/extension/test/expected/00_install_test.out | 2 +- server/extension/test/sql/00_install_test.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 2f68fdd..5bf0320 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -37,7 +37,7 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null (1 row) -SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); +SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 31b8660..e2e7c96 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -14,7 +14,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); -SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); +SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) From c653914694d18abc92f4185a2cd67407c777f6d4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 16 May 2017 16:18:42 +0200 Subject: [PATCH 2/5] Add test for empty table response #366 --- .../test/expected/366_empty_table_test.out | 5 ++ .../test/sql/366_empty_table_test.sql | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 server/extension/test/expected/366_empty_table_test.out create mode 100644 server/extension/test/sql/366_empty_table_test.sql diff --git a/server/extension/test/expected/366_empty_table_test.out b/server/extension/test/expected/366_empty_table_test.out new file mode 100644 index 0000000..53903cf --- /dev/null +++ b/server/extension/test/expected/366_empty_table_test.out @@ -0,0 +1,5 @@ +\set ECHO none + id | data +----+------ +(0 rows) + diff --git a/server/extension/test/sql/366_empty_table_test.sql b/server/extension/test/sql/366_empty_table_test.sql new file mode 100644 index 0000000..021c186 --- /dev/null +++ b/server/extension/test/sql/366_empty_table_test.sql @@ -0,0 +1,46 @@ +\set ECHO none +\set VERBOSITY verbose +SET client_min_messages TO error; + +-- Set configuration for a user 'foo' +DO $$ + import json + from cartodb_services.config import ServiceConfiguration + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration('observatory', 'foo', None) + service_config.user.set('soft_obs_general_limit', True) + service_config.user.set('period_end_date', '20170516') + +$$ LANGUAGE plpythonu; + + +-- Mock Observatory backend function +CREATE SCHEMA cdb_observatory; +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetData(geomvals geomval[], params JSON, merge BOOLEAN DEFAULT TRUE) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ +BEGIN + -- this will return an empty set + RAISE NOTICE 'Mocked OBS_GetData()'; +END; +$$ LANGUAGE plpgsql; +GRANT USAGE ON SCHEMA cdb_observatory TO geocoder_api; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api; + + +-- Test it +SELECT * FROM cdb_dataservices_server.OBS_GetData( + 'foo', + NULL, + '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::public._geomval, + '[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": ' + '"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments' + 'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries' + 'in some states and situations, to allow for census-tract-to-governmental-unit relationships where the governmental boundaries tend to remain unchanged between censuses. State and county boundaries always are census tract boundaries in the standard census geographic hierarchy. Tribal census tracts are a unique geographic entity defined within federally recognized American Indian reservations and off-reservation trust lands and can cross state and county boundaries. Tribal census tracts may be completely different from the census tracts and block groups defined by state and county (see “Tribal Census Tract”).", "denom_description": "The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.", "max_timespan_rank": null, "numer_description": "The number of people identifying as white, non-Hispanic in each' + 'geography.", "geom_t_description": null, "denom_t_description": null, "numer_t_description": null, "geom_geomref_colname": "geoid", "denom_geomref_colname": "geoid", "numer_geomref_colname": "geoid"}]'::json, + true); From b50637d36f68c8a761d031b45a5ec09ff8b7618a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 16 May 2017 16:23:13 +0200 Subject: [PATCH 3/5] Fix return type of OBS_GetData #366 --- server/extension/sql/110_data_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index e578b28..fb9dc70 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -619,7 +619,7 @@ RETURNS TABLE ( quota_service.increment_success_service_use(len(result)) return result else: - return None + return [] except BaseException as e: import sys quota_service.increment_failed_service_use(len(geomvals)) From 4ef60833441120b13d902140924fad2faaf48ca6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 16 May 2017 17:19:40 +0200 Subject: [PATCH 4/5] Test for OBS_GetData(geomrefs TEXT[], params JSON) #366 --- .../test/expected/366_empty_table_test.out | 4 ++++ .../test/sql/366_empty_table_test.sql | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/server/extension/test/expected/366_empty_table_test.out b/server/extension/test/expected/366_empty_table_test.out index 53903cf..15bdfc5 100644 --- a/server/extension/test/expected/366_empty_table_test.out +++ b/server/extension/test/expected/366_empty_table_test.out @@ -3,3 +3,7 @@ ----+------ (0 rows) + id | data +----+------ +(0 rows) + diff --git a/server/extension/test/sql/366_empty_table_test.sql b/server/extension/test/sql/366_empty_table_test.sql index 021c186..d601c6c 100644 --- a/server/extension/test/sql/366_empty_table_test.sql +++ b/server/extension/test/sql/366_empty_table_test.sql @@ -44,3 +44,22 @@ SELECT * FROM cdb_dataservices_server.OBS_GetData( 'in some states and situations, to allow for census-tract-to-governmental-unit relationships where the governmental boundaries tend to remain unchanged between censuses. State and county boundaries always are census tract boundaries in the standard census geographic hierarchy. Tribal census tracts are a unique geographic entity defined within federally recognized American Indian reservations and off-reservation trust lands and can cross state and county boundaries. Tribal census tracts may be completely different from the census tracts and block groups defined by state and county (see “Tribal Census Tract”).", "denom_description": "The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates.", "max_timespan_rank": null, "numer_description": "The number of people identifying as white, non-Hispanic in each' 'geography.", "geom_t_description": null, "denom_t_description": null, "numer_t_description": null, "geom_geomref_colname": "geoid", "denom_geomref_colname": "geoid", "numer_geomref_colname": "geoid"}]'::json, true); + + + +-- Mock another observatory backend function (overloaded, different params) +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetData(geomrefs TEXT[], params JSON) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ +BEGIN + -- this will return an empty set + RAISE NOTICE 'Mocked OBS_GetData()'; +END; +$$ LANGUAGE plpgsql; +GRANT USAGE ON SCHEMA cdb_observatory TO geocoder_api; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api; + +-- Test it +SELECT * FROM cdb_dataservices_server.OBS_GetData('foo', NULL, '{bar, baz}'::TEXT[], '[]'::JSON); From 59a8cafc74cb2dbd0d9f564b33377c21795bef7a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 16 May 2017 17:20:16 +0200 Subject: [PATCH 5/5] Fix another empty table case #366 --- server/extension/sql/110_data_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index fb9dc70..0bed8c0 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -678,7 +678,7 @@ RETURNS TABLE ( quota_service.increment_success_service_use(len(result)) return result else: - return None + return [] except BaseException as e: import sys quota_service.increment_failed_service_use(len(geomrefs))