cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #339 from CartoDB/obs-getdata-getmeta-wrappers

Browse Source 
Obs_GetData OBS_GetMeta wrappers
This commit is contained in:
Mario de Frutos
2017-02-01 10:05:52 +01:00
committed by GitHub
parent c550a5d4e9 b2f9a10035
commit c364c7f5ed
8 changed files with 377 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
client/renderer/interface.yaml
View File

@@ -261,6 +261,38 @@
- { name: boundary_id, type: text}
- { name: time_span, type: text, default: 'NULL'}
- name: obs_getdata
return_type: TABLE(id int, data json)
multi_row: true
multi_field: true
table_fields:
- { name: id, type: integer }
- { name: data, type: json }
params:
- { name: geomvals, type: "geomval[]" }
- { name: params, type: json }
- { name: merge, type: boolean, default: true }
- name: obs_getdata
return_type: TABLE(id text, data json)
multi_row: true
multi_field: true
table_fields:
- { name: id, type: text }
- { name: data, type: json }
params:
- { name: geomrefs, type: "text[]" }
- { name: params, type: json }
- name: obs_getmeta
return_type: json
params:
- { name: geom_ref, type: "Geometry(Geometry, 4326)" }
- { name: params, type: json }
- { name: max_timespan_rank, type: integer, default: 'NULL' }
- { name: max_score_rank, type: integer, default: 'NULL' }
- { name: target_geoms, type: integer, default: 'NULL' }
- name: obs_getcategory
return_type: text
params:
@@ -388,4 +420,4 @@
return_type: BOOLEAN
params:
- { name: service, type: TEXT }
- { name: input_size, type: NUMERIC }
- { name: input_size, type: NUMERIC }

52
client/test/expected/90_data_observatory_test.out
View File

@@ -173,6 +173,27 @@ BEGIN
RETURN QUERY SELECT '0106000020E61000000100000001030000000100000019000000DD96C805677C52C0CBD8D0CDFE5A44407A19C5724B7C52C033BF9A03045B4440FF59F3E32F7C52C0CB290131095B4440247EC51A2E7C52C0548B8862F25A44401FF5D72B2C7C52C0CB811E6ADB5A444049F086342A7C52C09CC3B5DAC35A444044679945287C52C01F680586AC5A444012D90759167C52C09430D3F6AF5A44405698BED7107C52C04759BF99985A4440DEE522BE137C52C00664AF777F5A4440F92EA52E197C52C09BA73AE4665A4440C558A65F227C52C0D80DDB16655A4440373465A71F7C52C065A71FD4455A44404AED45B41D7C52C0785DBF60375A444083A3E4D5397C52C09946938B315A4440F2B4FCC0557C52C01FF5D72B2C5A444026C286A7577C52C02BDD5D67435A44405BCF108E597C52C0C651B9895A5A444030D461855B7C52C0D1393FC5715A444065E1EB6B5D7C52C078280AF4895A44403AE63C635F7C52C07D1F0E12A25A44403F6F2A52617C52C05983F755B95A44405C3AE63C637C52C04852D2C3D05A444049810530657C52C0FAEE5696E85A4440DD96C805677C52C0CBD8D0CDFE5A4440'::geometry as the_geom, '36047049300'::text as geom_refs;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL)
RETURNS JSON AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms;
RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json)
RETURNS TABLE (id TEXT, data JSON) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params;
RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true)
RETURNS TABLE (id TEXT, data JSON) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge;
RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text);
NOTICE: cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group)
@@ -337,3 +358,34 @@ CONTEXT: PL/pgSQL function obs_getavailableboundaries(geometry,text) line 16 at
(us.census.tiger.place,"Incorporated places are those reported to the Census Bureau as legally in existence as of January 1, 2010, as reported in the latest Boundary and Annexation Survey (BAS), under the laws of their respective states.",2014,obs_7c9493c41fa8f4bd178ab993ea3d5891c1977667)
(1 row)
SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000);
NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, <NULL>, 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], 1, 1, 1000)
CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)"
PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement
obs_getmeta
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]
(1 row)
SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000));
NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, <NULL>, 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}], 1, 1, 1000)
CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)"
PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement
NOTICE: cdb_dataservices_client._obs_getdata(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, <NULL>, {36047}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}])
CONTEXT: PL/pgSQL function obs_getdata(text[],json) line 16 at RETURN QUERY
obs_getdata
--------------------------------------------
(36047,"[{""value"" : 10349.1547875017}]")
(1 row)
SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]'));
NOTICE: cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, <NULL>, 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)"
PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement
NOTICE: cdb_dataservices_client._obs_getdata(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, <NULL>, {"(0101000020E61000009A999999997952C09A99999999594440,1)"}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}], t)
CONTEXT: PL/pgSQL function obs_getdata(geomval[],json,boolean) line 16 at RETURN QUERY
obs_getdata
--------------------------------------------
(36047,"[{""value"" : 10349.1547875017}]")
(1 row)

28
client/test/sql/90_data_observatory_test.sql
View File

@@ -194,6 +194,30 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL)
RETURNS JSON AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms;
RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json)
RETURNS TABLE (id TEXT, data JSON) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params;
RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true)
RETURNS TABLE (id TEXT, data JSON) AS $$
BEGIN
RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge;
RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text);
SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text);
@@ -214,3 +238,7 @@ SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326
SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326));
SELECT obs_search('total_pop'::text);
SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326));
SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000);
SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000));
SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]'));

175
server/extension/sql/110_data_observatory_augmentation.sql
View File

@@ -567,3 +567,178 @@ RETURNS NUMERIC AS $$
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
username TEXT,
orgname TEXT,
geomvals geomval[],
params JSON,
merge BOOLEAN DEFAULT True)
RETURNS TABLE (
id INT,
data JSON
) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
username TEXT,
orgname TEXT,
geomvals geomval[],
params JSON,
merge BOOLEAN DEFAULT True)
RETURNS TABLE (
id INT,
data JSON
) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdata', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetData')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
username TEXT,
orgname TEXT,
geomrefs TEXT[],
params JSON)
RETURNS TABLE (
id TEXT,
data JSON
) AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
username TEXT,
orgname TEXT,
geomrefs TEXT[],
params JSON)
RETURNS TABLE (
id TEXT,
data JSON
) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdata', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
exc_info = sys.exc_info()
logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2]))
logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetData')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
username TEXT,
orgname TEXT,
geom Geometry(Geometry, 4326),
params JSON,
max_timespan_rank INTEGER DEFAULT NULL,
max_score_rank INTEGER DEFAULT NULL,
target_geoms INTEGER DEFAULT NULL)
RETURNS JSON AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta(
username TEXT,
orgname TEXT,
geom Geometry(Geometry, 4326),
params JSON,
max_timespan_rank INTEGER DEFAULT NULL,
max_score_rank INTEGER DEFAULT NULL,
target_geoms INTEGER DEFAULT NULL)
RETURNS JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getmeta', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
if result:
quota_service.increment_success_service_use()
return result[0]['meta']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeta')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

33
server/extension/test/expected/100_data_observatory_test.out
View File

@@ -141,6 +141,39 @@ SELECT exists(SELECT *
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeta'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, text[], json');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

21
server/extension/test/sql/100_data_observatory_test.sql
View File

@@ -89,6 +89,27 @@ SELECT exists(SELECT *
AND proname = 'obs_getboundarybyid'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getmeta'
AND oidvectortypes(p.proargtypes) = 'text, text, geometry, json, integer, integer, integer');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, geomval[], json, boolean');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_dataservices_server'
AND proname = 'obs_getdata'
AND oidvectortypes(p.proargtypes) = 'text, text, text[], json');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

2
server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py
View File

@@ -41,7 +41,7 @@ class RedisDBConfig:
key)
conf = self._db_conn.execute(conf_query)[0]['conf']
if conf is None:
raise "There is no redis configuration defined"
raise Exception("There is no redis configuration defined")
else:
params = json.loads(conf)
self._host = params['redis_host']

36
test/integration/test_data_observatory_functions.py
View File

@@ -160,12 +160,12 @@ class TestDataObservatoryFunctions(TestCase):
assert_equal(e.message[0], "The api_key must be provided")
def test_if_obs_get_boundary_id_is_ok(self):
query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;&api_key={0}".format(self.env_variables['api_key'])
query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key'])
result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
assert_not_equal(result['boundary_id'], None)
def test_if_obs_get_boundary_id_without_api_key_raise_error(self):
query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;"
query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;"
try:
IntegrationTestHelper.execute_query(self.sql_api_url, query)
except Exception as e:
@@ -290,3 +290,35 @@ class TestDataObservatoryFunctions(TestCase):
IntegrationTestHelper.execute_query(self.sql_api_url, query)
except Exception as e:
assert_equal(e.message[0], "The api_key must be provided")
def test_if_obs_get_meta_is_ok(self):
query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]', 1, 1, 1000) as metadata LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
assert_not_equal(result['metadata'], None)
def test_if_obs_get_meta_without_api_key_raise_error(self):
query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]', 1, 1, 1000) LIMIT 1;"
try:
IntegrationTestHelper.execute_query(self.sql_api_url, query)
except Exception as e:
assert_equal(e.message[0], "The api_key must be provided")
def test_if_obs_get_data_is_ok(self):
query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]', 1, 1, 1000)) LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]')) LIMIT 1;&api_key={0}".format(self.env_variables['api_key'])
result_1 = IntegrationTestHelper.execute_query(self.sql_api_url, query_1)
assert_not_equal(result_1['data_id'], None)
result_2 = IntegrationTestHelper.execute_query(self.sql_api_url, query_2)
assert_not_equal(result_2['data_id'], None)
def test_if_obs_get_data_without_api_key_raise_error(self):
query_1 = "SELECT id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]', 1, 1, 1000))"
query_2 = "SELECT id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{\"numer_id\": \"us.census.acs.B01003001\"}]'));"
try:
IntegrationTestHelper.execute_query(self.sql_api_url, query_1)
except Exception as e:
assert_equal(e.message[0], "The api_key must be provided")
try:
IntegrationTestHelper.execute_query(self.sql_api_url, query_2)
except Exception as e:
assert_equal(e.message[0], "The api_key must be provided")