diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 3b9ff7c..fa7c257 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -261,6 +261,38 @@ - { name: boundary_id, type: text} - { name: time_span, type: text, default: 'NULL'} +- name: obs_getdata + return_type: TABLE(id int, data json) + multi_row: true + multi_field: true + table_fields: + - { name: id, type: integer } + - { name: data, type: json } + params: + - { name: geomvals, type: "geomval[]" } + - { name: params, type: json } + - { name: merge, type: boolean, default: true } + +- name: obs_getdata + return_type: TABLE(id text, data json) + multi_row: true + multi_field: true + table_fields: + - { name: id, type: text } + - { name: data, type: json } + params: + - { name: geomrefs, type: "text[]" } + - { name: params, type: json } + +- name: obs_getmeta + return_type: json + params: + - { name: geom_ref, type: "Geometry(Geometry, 4326)" } + - { name: params, type: json } + - { name: max_timespan_rank, type: integer, default: 'NULL' } + - { name: max_score_rank, type: integer, default: 'NULL' } + - { name: target_geoms, type: integer, default: 'NULL' } + - name: obs_getcategory return_type: text params: @@ -388,4 +420,4 @@ return_type: BOOLEAN params: - { name: service, type: TEXT } - - { name: input_size, type: NUMERIC } \ No newline at end of file + - { name: input_size, type: NUMERIC } diff --git a/client/test/expected/90_data_observatory_test.out b/client/test/expected/90_data_observatory_test.out index 573bee2..f7d0831 100644 --- a/client/test/expected/90_data_observatory_test.out +++ b/client/test/expected/90_data_observatory_test.out @@ -173,6 +173,27 @@ BEGIN RETURN QUERY SELECT '0106000020E61000000100000001030000000100000019000000DD96C805677C52C0CBD8D0CDFE5A44407A19C5724B7C52C033BF9A03045B4440FF59F3E32F7C52C0CB290131095B4440247EC51A2E7C52C0548B8862F25A44401FF5D72B2C7C52C0CB811E6ADB5A444049F086342A7C52C09CC3B5DAC35A444044679945287C52C01F680586AC5A444012D90759167C52C09430D3F6AF5A44405698BED7107C52C04759BF99985A4440DEE522BE137C52C00664AF777F5A4440F92EA52E197C52C09BA73AE4665A4440C558A65F227C52C0D80DDB16655A4440373465A71F7C52C065A71FD4455A44404AED45B41D7C52C0785DBF60375A444083A3E4D5397C52C09946938B315A4440F2B4FCC0557C52C01FF5D72B2C5A444026C286A7577C52C02BDD5D67435A44405BCF108E597C52C0C651B9895A5A444030D461855B7C52C0D1393FC5715A444065E1EB6B5D7C52C078280AF4895A44403AE63C635F7C52C07D1F0E12A25A44403F6F2A52617C52C05983F755B95A44405C3AE63C637C52C04852D2C3D05A444049810530657C52C0FAEE5696E85A4440DD96C805677C52C0CBD8D0CDFE5A4440'::geometry as the_geom, '36047049300'::text as geom_refs; END; $$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms; + RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; -- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); NOTICE: cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, , 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group) @@ -337,3 +358,34 @@ CONTEXT: PL/pgSQL function obs_getavailableboundaries(geometry,text) line 16 at (us.census.tiger.place,"Incorporated places are those reported to the Census Bureau as legally in existence as of January 1, 2010, as reported in the latest Boundary and Annexation Survey (BAS), under the laws of their respective states.",2014,obs_7c9493c41fa8f4bd178ab993ea3d5891c1977667) (1 row) +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, , 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], 1, 1, 1000) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)" +PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement + obs_getmeta +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}] +(1 row) + +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, , 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}], 1, 1, 1000) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)" +PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement +NOTICE: cdb_dataservices_client._obs_getdata(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, , {36047}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]) +CONTEXT: PL/pgSQL function obs_getdata(text[],json) line 16 at RETURN QUERY + obs_getdata +-------------------------------------------- + (36047,"[{""value"" : 10349.1547875017}]") +(1 row) + +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); +NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, , 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], , , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)" +PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement +NOTICE: cdb_dataservices_client._obs_getdata(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, , {"(0101000020E61000009A999999997952C09A99999999594440,1)"}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}], t) +CONTEXT: PL/pgSQL function obs_getdata(geomval[],json,boolean) line 16 at RETURN QUERY + obs_getdata +-------------------------------------------- + (36047,"[{""value"" : 10349.1547875017}]") +(1 row) + diff --git a/client/test/sql/90_data_observatory_test.sql b/client/test/sql/90_data_observatory_test.sql index 4f6bd7a..0f052b4 100644 --- a/client/test/sql/90_data_observatory_test.sql +++ b/client/test/sql/90_data_observatory_test.sql @@ -194,6 +194,30 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms; + RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true) +RETURNS TABLE (id TEXT, data JSON) AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge; + RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data; +END; +$$ LANGUAGE 'plpgsql'; + -- Exercise the public and the proxied function SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text); SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); @@ -214,3 +238,7 @@ SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326 SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); SELECT obs_search('total_pop'::text); SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326)); +SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000); +SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000)); +SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]')); + diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index d3d1968..163dc93 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -567,3 +567,178 @@ RETURNS NUMERIC AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomvals geomval[], + params JSON, + merge BOOLEAN DEFAULT True) +RETURNS TABLE ( + id INT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"]) + result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge]) + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData( + username TEXT, + orgname TEXT, + geomrefs TEXT[], + params JSON) +RETURNS TABLE ( + id TEXT, + data JSON +) AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getdata', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"]) + result = plpy.execute(obs_plan, [username, orgname, geomrefs, params]) + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + exc_info = sys.exc_info() + logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2])) + logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetData') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta( + username TEXT, + orgname TEXT, + geom Geometry(Geometry, 4326), + params JSON, + max_timespan_rank INTEGER DEFAULT NULL, + max_score_rank INTEGER DEFAULT NULL, + target_geoms INTEGER DEFAULT NULL) +RETURNS JSON AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + with metrics('obs_getmeta', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"]) + result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms]) + if result: + quota_service.increment_success_service_use() + return result[0]['meta'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeta') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 23cd4e1..f509633 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -141,6 +141,39 @@ SELECT exists(SELECT * t (1 row) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getmeta' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, text[], json'); + exists +-------- + t +(1 row) + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql index 945448d..caabb11 100644 --- a/server/extension/test/sql/100_data_observatory_test.sql +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -89,6 +89,27 @@ SELECT exists(SELECT * AND proname = 'obs_getboundarybyid' AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getmeta' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdata' + AND oidvectortypes(p.proargtypes) = 'text, text, text[], json'); + SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 2b77531..7cbc375 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -41,7 +41,7 @@ class RedisDBConfig: key) conf = self._db_conn.execute(conf_query)[0]['conf'] if conf is None: - raise "There is no redis configuration defined" + raise Exception("There is no redis configuration defined") else: params = json.loads(conf) self._host = params['redis_host'] diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 0461f7b..f9dca29 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -160,12 +160,12 @@ class TestDataObservatoryFunctions(TestCase): assert_equal(e.message[0], "The api_key must be provided") def test_if_obs_get_boundary_id_is_ok(self): - query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) + query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key']) result = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(result['boundary_id'], None) def test_if_obs_get_boundary_id_without_api_key_raise_error(self): - query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;" + query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;" try: IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: @@ -290,3 +290,35 @@ class TestDataObservatoryFunctions(TestCase): IntegrationTestHelper.execute_query(self.sql_api_url, query) except Exception as e: assert_equal(e.message[0], "The api_key must be provided") + + def test_if_obs_get_meta_is_ok(self): + query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]', 1, 1, 1000) as metadata LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) + result = IntegrationTestHelper.execute_query(self.sql_api_url, query) + assert_not_equal(result['metadata'], None) + + def test_if_obs_get_meta_without_api_key_raise_error(self): + query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]', 1, 1, 1000) LIMIT 1;" + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided") + + def test_if_obs_get_data_is_ok(self): + query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]', 1, 1, 1000)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) + query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]')) LIMIT 1;&api_key={0}".format(self.env_variables['api_key']) + result_1 = IntegrationTestHelper.execute_query(self.sql_api_url, query_1) + assert_not_equal(result_1['data_id'], None) + result_2 = IntegrationTestHelper.execute_query(self.sql_api_url, query_2) + assert_not_equal(result_2['data_id'], None) + + def test_if_obs_get_data_without_api_key_raise_error(self): + query_1 = "SELECT id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]', 1, 1, 1000))" + query_2 = "SELECT id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]'));" + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query_1) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided") + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query_2) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided")