From 56de523498c548f318d3354ba478f5cb0418ae66 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 12 Jan 2017 04:03:01 +0000 Subject: [PATCH 1/9] add wrappers for getmeta, getdata(geoms) and getdata(ids) --- .../sql/110_data_observatory_augmentation.sql | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index d3d1968..163dc93 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -567,3 +567,178 @@ RETURNS NUMERIC AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeta', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + quota_service.increment_success_service_use() + return result[0]['meta'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; From 8204bb3451f4e9a80e4974d41f5f059145ef8c82 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 12 Jan 2017 04:03:33 +0000 Subject: [PATCH 2/9] fix error-causing raise (which obscures real error) --- .../cartodb_services/cartodb_services/tools/redis_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 2b77531..7cbc375 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -41,7 +41,7 @@ class RedisDBConfig: key) conf = self._db_conn.execute(conf_query)[0]['conf'] if conf is None: - raise "There is no redis configuration defined" + raise Exception("There is no redis configuration defined") else: params = json.loads(conf) self._host = params['redis_host'] From e81f005ce9fd9ef5565c73c777092d2ecd252bc1 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 12 Jan 2017 21:22:45 +0000 Subject: [PATCH 3/9] adding server tests, interface to generate client sql code --- client/renderer/interface.yaml | 34 ++++++++++++++++++- .../expected/100_data_observatory_test.out | 34 +++++++++++++++++++ .../test/sql/100_data_observatory_test.sql | 21 ++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 3b9ff7c..fa7c257 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -261,6 +261,38 @@ - { name: boundary_id, type: text} - { name: time_span, type: text, default: 'NULL'} +- name: obs_getdata + return_type: TABLE(id int, data json) + multi_row: true + multi_field: true + table_fields: + - { name: id, type: integer } + - { name: data, type: json } + params: + - { name: geomvals, type: "geomval[]" } + - { name: params, type: json } + - { name: merge, type: boolean, default: true } + +- name: obs_getdata + return_type: TABLE(id text, data json) + multi_row: true + multi_field: true + table_fields: + - { name: id, type: text } + - { name: data, type: json } + params: + - { name: geomrefs, type: "text[]" } + - { name: params, type: json } + +- name: obs_getmeta + return_type: json + params: + - { name: geom_ref, type: "Geometry(Geometry, 4326)" } + - { name: params, type: json } + - { name: max_timespan_rank, type: integer, default: 'NULL' } + - { name: max_score_rank, type: integer, default: 'NULL' } + - { name: target_geoms, type: integer, default: 'NULL' } + - name: obs_getcategory return_type: text params: @@ -388,4 +420,4 @@ return_type: BOOLEAN params: - { name: service, type: TEXT } - - { name: input_size, type: NUMERIC } \ No newline at end of file + - { name: input_size, type: NUMERIC } diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 23cd4e1..7fc17a2 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -141,6 +141,40 @@ SELECT exists(SELECT * t (1 row) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getmeta' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, text[], json'); + exists +-------- + t +(1 row) + + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql index 945448d..caabb11 100644 --- a/server/extension/test/sql/100_data_observatory_test.sql +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -89,6 +89,27 @@ SELECT exists(SELECT * AND proname = 'obs_getboundarybyid' AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getmeta' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, text[], json'); + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) From 1f225e0978ad1f595592cd19550967887d9e8b1b Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 19 Jan 2017 20:08:34 +0000 Subject: [PATCH 4/9] fix extraneous newline in expectation --- server/extension/test/expected/100_data_observatory_test.out | 1 - 1 file changed, 1 deletion(-) diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 7fc17a2..f509633 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -174,7 +174,6 @@ SELECT exists(SELECT * t (1 row) - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) From bfd7df8c8a019a585f2dac9401f1a7095c03dc0e Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 19 Jan 2017 20:29:50 +0000 Subject: [PATCH 5/9] tmp commit --- client/test/sql/90_data_observatory_test.sql | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/client/test/sql/90_data_observatory_test.sql b/client/test/sql/90_data_observatory_test.sql index 4f6bd7a..cfda606 100644 --- a/client/test/sql/90_data_observatory_test.sql +++ b/client/test/sql/90_data_observatory_test.sql @@ -194,6 +194,30 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms; + RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( geomrefs text[], params JSON) +RETURNS TABLE ( id TEXT, data JSON) AS $$ +DECLARE + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, id, data; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( geomvals geomval[], params JSON) +RETURNS TABLE ( id TEXT, data JSON) AS $$ +DECLARE + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, id, data; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; + -- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); @@ -214,3 +238,7 @@ SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326 SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); SELECT obs_search('total_pop'::text); SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +select cdb_observatory.obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], cdb_observatory.obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', + From 8094e79d0b1b06cd8ec4b5335d26664e99082b03 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 19 Jan 2017 21:12:44 +0000 Subject: [PATCH 6/9] fix some bugs in client test --- client/test/sql/90_data_observatory_test.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/test/sql/90_data_observatory_test.sql b/client/test/sql/90_data_observatory_test.sql index cfda606..5bd749b 100644 --- a/client/test/sql/90_data_observatory_test.sql +++ b/client/test/sql/90_data_observatory_test.sql @@ -194,15 +194,15 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL) RETURNS JSON AS $$ BEGIN RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms; - RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON + RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( geomrefs text[], params JSON) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(username text, orgname text, geomrefs text[], params JSON) RETURNS TABLE ( id TEXT, data JSON) AS $$ DECLARE RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, id, data; @@ -210,7 +210,7 @@ DECLARE END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( geomvals geomval[], params JSON) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(username text, orgname text, geomvals geomval[], params JSON) RETURNS TABLE ( id TEXT, data JSON) AS $$ DECLARE RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, id, data; @@ -240,5 +240,5 @@ SELECT obs_search('total_pop'::text); SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); -select cdb_observatory.obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], cdb_observatory.obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', From f0d96541b6faddb0ed320f1b517f665e8b42eea1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 26 Jan 2017 14:39:09 +0100 Subject: [PATCH 7/9] Fixed client tests --- .../expected/90_data_observatory_test.out | 52 +++++++++++++++++++ client/test/sql/90_data_observatory_test.sql | 18 +++---- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/client/test/expected/90_data_observatory_test.out b/client/test/expected/90_data_observatory_test.out index 573bee2..f7d0831 100644 --- a/client/test/expected/90_data_observatory_test.out +++ b/client/test/expected/90_data_observatory_test.out @@ -173,6 +173,27 @@ BEGIN RETURN QUERY SELECT '0106000020E61000000100000001030000000100000019000000DD96C805677C52C0CBD8D0CDFE5A44407A19C5724B7C52C033BF9A03045B4440FF59F3E32F7C52C0CB290131095B4440247EC51A2E7C52C0548B8862F25A44401FF5D72B2C7C52C0CB811E6ADB5A444049F086342A7C52C09CC3B5DAC35A444044679945287C52C01F680586AC5A444012D90759167C52C09430D3F6AF5A44405698BED7107C52C04759BF99985A4440DEE522BE137C52C00664AF777F5A4440F92EA52E197C52C09BA73AE4665A4440C558A65F227C52C0D80DDB16655A4440373465A71F7C52C065A71FD4455A44404AED45B41D7C52C0785DBF60375A444083A3E4D5397C52C09946938B315A4440F2B4FCC0557C52C01FF5D72B2C5A444026C286A7577C52C02BDD5D67435A44405BCF108E597C52C0C651B9895A5A444030D461855B7C52C0D1393FC5715A444065E1EB6B5D7C52C078280AF4895A44403AE63C635F7C52C07D1F0E12A25A44403F6F2A52617C52C05983F755B95A44405C3AE63C637C52C04852D2C3D05A444049810530657C52C0FAEE5696E85A4440DD96C805677C52C0CBD8D0CDFE5A4440'::geometry as the_geom, '36047049300'::text as geom_refs; END; $$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms; + RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); NOTICE: cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, , 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group) @@ -337,3 +358,34 @@ CONTEXT: PL/pgSQL function obs_getavailableboundaries(geometry,text) line 16 at (us.census.tiger.place,"Incorporated places are those reported to the Census Bureau as legally in existence as of January 1, 2010, as reported in the latest Boundary and Annexation Survey (BAS), under the laws of their respective states.",2014,obs_7c9493c41fa8f4bd178ab993ea3d5891c1977667) (1 row) +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, , 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], 1, 1, 1000) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)" +PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement + obs_getmeta +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}] +(1 row) + +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, , 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}], 1, 1, 1000) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)" +PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement +NOTICE: cdb_dataservices_client._obs_getdata(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, , {36047}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]) +CONTEXT: PL/pgSQL function obs_getdata(text[],json) line 16 at RETURN QUERY + obs_getdata +-------------------------------------------- + (36047,"[{""value"" : 10349.1547875017}]") +(1 row) + +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); +NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, , 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], , , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)" +PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement +NOTICE: cdb_dataservices_client._obs_getdata(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, , {"(0101000020E61000009A999999997952C09A99999999594440,1)"}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}], t) +CONTEXT: PL/pgSQL function obs_getdata(geomval[],json,boolean) line 16 at RETURN QUERY + obs_getdata +-------------------------------------------- + (36047,"[{""value"" : 10349.1547875017}]") +(1 row) + diff --git a/client/test/sql/90_data_observatory_test.sql b/client/test/sql/90_data_observatory_test.sql index 5bd749b..0f052b4 100644 --- a/client/test/sql/90_data_observatory_test.sql +++ b/client/test/sql/90_data_observatory_test.sql @@ -202,18 +202,18 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(username text, orgname text, geomrefs text[], params JSON) -RETURNS TABLE ( id TEXT, data JSON) AS $$ -DECLARE - RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, id, data; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params; RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; END; $$ LANGUAGE 'plpgsql'; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(username text, orgname text, geomvals geomval[], params JSON) -RETURNS TABLE ( id TEXT, data JSON) AS $$ -DECLARE - RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, id, data; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge; RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; END; $$ LANGUAGE 'plpgsql'; @@ -240,5 +240,5 @@ SELECT obs_search('total_pop'::text); SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); -SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); From b4fa0dd4fc6f91b3b1b038e9447bd8eaf59cc397 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 26 Jan 2017 17:46:36 +0100 Subject: [PATCH 8/9] Added integration tests for the new DO functions --- .../test_data_observatory_functions.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 0461f7b..5224269 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -290,3 +290,35 @@ class TestDataObservatoryFunctions(TestCase): IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: assert_equal(e.message[0], "The api_key must be provided") + + def test_if_obs_get_meta_is_ok(self): + query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]', 1, 1, 1000) as metadata LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) + result = IntegrationTestHelper.execute_query(self.sql_api_url, query) + assert_not_equal(result['metadata'], None) + + def test_if_obs_get_meta_without_api_key_raise_error(self): + query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]', 1, 1, 1000) LIMIT 1;" + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided") + + def test_if_obs_get_data_is_ok(self): + query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]', 1, 1, 1000)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) + query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]')) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) + result_1 = IntegrationTestHelper.execute_query(self.sql_api_url, query_1) + assert_not_equal(result_1['data_id'], None) + result_2 = IntegrationTestHelper.execute_query(self.sql_api_url, query_2) + assert_not_equal(result_2['data_id'], None) + + def test_if_obs_get_data_without_api_key_raise_error(self): + query_1 = "SELECT id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]', 1, 1, 1000))" + query_2 = "SELECT id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]'));" + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query_1) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided") + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query_2) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided") From b2f9a1003550e87dacf5b0a720509027e88b3ec1 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 1 Feb 2017 09:53:35 +0100 Subject: [PATCH 9/9] Change timespan for getboundaryid to 2015 --- test/integration/test_data_observatory_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 5224269..f9dca29 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -160,12 +160,12 @@ class TestDataObservatoryFunctions(TestCase): assert_equal(e.message[0], "The api_key must be provided") def test_if_obs_get_boundary_id_is_ok(self): - query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) + query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) result = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(result['boundary_id'], None) def test_if_obs_get_boundary_id_without_api_key_raise_error(self): - query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;" + query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;" try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: