diff --git a/client/cdb_dataservices_client--0.4.0--0.5.0.sql b/client/cdb_dataservices_client--0.4.0--0.5.0.sql index ac927b7..d200158 100644 --- a/client/cdb_dataservices_client--0.4.0--0.5.0.sql +++ b/client/cdb_dataservices_client--0.4.0--0.5.0.sql @@ -9,9 +9,9 @@ -- and should also be the only ones with SECURITY DEFINER CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot (geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ DECLARE - ret json; + username text; orgname text; BEGIN @@ -24,8 +24,8 @@ BEGIN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - SELECT cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level) INTO ret; - RETURN ret; + RETURN QUERY + SELECT * FROM cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; @@ -37,9 +37,9 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER; -- and should also be the only ones with SECURITY DEFINER CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot (geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ DECLARE - ret json; + username text; orgname text; BEGIN @@ -52,8 +52,8 @@ BEGIN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - SELECT cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level) INTO ret; - RETURN ret; + RETURN QUERY + SELECT * FROM cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level); END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; @@ -452,7 +452,7 @@ $$ LANGUAGE 'plpgsql' SECURITY DEFINER; CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot (username text, organization_name text, geom geometry(Geometry, 4326), time_span text DEFAULT NULL, geometry_level text DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ CONNECT cdb_dataservices_client._server_conn_str(); SELECT cdb_dataservices_server.obs_getdemographicsnapshot (username, organization_name, geom, time_span, geometry_level); @@ -460,7 +460,7 @@ RETURNS json AS $$ $$ LANGUAGE plproxy; CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot (username text, organization_name text, geom geometry(Geometry, 4326), geometry_level text DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ CONNECT cdb_dataservices_client._server_conn_str(); SELECT cdb_dataservices_server.obs_getsegmentsnapshot (username, organization_name, geom, geometry_level); diff --git a/client/cdb_dataservices_client--0.5.0--0.4.0.sql b/client/cdb_dataservices_client--0.5.0--0.4.0.sql index b8a01be..c412da1 100644 --- a/client/cdb_dataservices_client--0.5.0--0.4.0.sql +++ b/client/cdb_dataservices_client--0.5.0--0.4.0.sql @@ -1,6 +1,25 @@ --DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.4.0'" to load this file. \quit +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot(geometry); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot(geometry); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot(geometry); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot(geometry); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid(text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius(geometry, numeric, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius(geometry, numeric, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory(geometry, text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation(geometry); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search(text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries(geometry); + DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_get_demographic_snapshot(geometry); DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_get_segment_snapshot(geometry); DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getdemographicsnapshot(geometry); diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 7a3ace5..bb0a213 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -103,14 +103,16 @@ - { name: geometry_level, type: text, default: 'NULL' } - name: obs_getdemographicsnapshot - return_type: json + return_type: SETOF JSON + multi_row: true params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: time_span, type: "text", default: 'NULL' } - { name: geometry_level, type: text, default: 'NULL' } - name: obs_getsegmentsnapshot - return_type: json + return_type: SETOF JSON + multi_row: true params: - { name: geom, type: "geometry(Geometry, 4326)" } - { name: geometry_level, type: text, default: 'NULL' } diff --git a/client/test/expected/90_data_observatory_test.out b/client/test/expected/90_data_observatory_test.out index 5e8f8b2..dcbed19 100644 --- a/client/test/expected/90_data_observatory_test.out +++ b/client/test/expected/90_data_observatory_test.out @@ -184,8 +184,7 @@ PL/pgSQL function obs_get_segment_snapshot(geometry,text) line 16 at SQL stateme SELECT obs_getdemographicsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013', '"us.census.tiger".block_group'::text); NOTICE: cdb_dataservices_client._obs_getdemographicsnapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdemographicsnapshot invoked with params (test_user, , 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group) -CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getdemographicsnapshot(username, orgname, geom, time_span, geometry_level)" -PL/pgSQL function obs_getdemographicsnapshot(geometry,text,text) line 16 at SQL statement +CONTEXT: PL/pgSQL function obs_getdemographicsnapshot(geometry,text,text) line 16 at RETURN QUERY obs_getdemographicsnapshot ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null} @@ -193,8 +192,7 @@ PL/pgSQL function obs_getdemographicsnapshot(geometry,text,text) line 16 at SQL SELECT obs_getsegmentsnapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text); NOTICE: cdb_dataservices_client._obs_getsegmentsnapshot(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getsegmentsnapshot invoked with params (test_user, , 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, "us.census.tiger".block_group) -CONTEXT: SQL statement "SELECT cdb_dataservices_client._obs_getsegmentsnapshot(username, orgname, geom, geometry_level)" -PL/pgSQL function obs_getsegmentsnapshot(geometry,text) line 16 at SQL statement +CONTEXT: PL/pgSQL function obs_getsegmentsnapshot(geometry,text) line 16 at RETURN QUERY obs_getsegmentsnapshot ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"total_pop":9516.27915900609,"male_pop":6152.51885204623,"female_pop":3363.76030695986,"median_age":28.8,"white_pop":5301.51624447348,"black_pop":149.500458087105,"asian_pop":230.000704749392,"hispanic_pop":3835.26175169611,"amerindian_pop":0,"other_race_pop":0,"two_or_more_races_pop":0,"not_hispanic_pop":5681.01740730998,"households":3323.51018362871,"pop_25_years_over":7107.02177675621,"high_school_diploma":1040.753188991,"less_one_year_college":69.0002114248176,"one_year_more_college":793.502431385402,"associates_degree":327.751004267883,"bachelors_degree":2742.7584041365,"masters_degree":931.502854235037,"median_income":66304,"gini_index":0.3494,"income_per_capita":28291,"housing_units":3662.76122313407,"vacant_housing_units":339.251039505353,"vacant_housing_units_for_rent":120.750369993431,"vacant_housing_units_for_sale":0,"median_rent":1764,"percent_income_spent_on_rent":35.3,"owner_occupied_housing_units":339.251039505353,"million_dollar_housing_units":0,"mortgaged_housing_units":224.250687130657,"commuters_16_over":6549.27006773893,"commute_less_10_mins":327.751004267883,"commute_10_14_mins":28.750088093674,"commute_15_19_mins":201.250616655718,"commute_20_24_mins":621.001902823358,"commute_25_29_mins":373.751145217762,"commute_30_34_mins":1851.5056732326,"commute_35_44_mins":1414.50433420876,"commute_45_59_mins":1115.50341803455,"commute_60_more_mins":615.251885204623,"aggregate_travel_time_to_work":null,"income_less_10000":57.500176187348,"income_10000_14999":0,"income_15000_19999":212.750651893187,"income_20000_24999":408.251250930171,"income_25000_29999":0,"income_30000_34999":155.25047570584,"income_35000_39999":109.250334755961,"income_40000_44999":92.0002818997568,"income_45000_49999":63.2501938060828,"income_50000_59999":184.000563799514,"income_60000_74999":621.001902823358,"income_75000_99999":552.001691398541,"income_100000_124999":327.751004267883,"income_125000_149999":333.501021886618,"income_150000_199999":126.500387612166,"income_200000_or_more":null,"land_area":null} diff --git a/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql b/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql index 3cd3197..b37b11d 100644 --- a/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql +++ b/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql @@ -1,6 +1,8 @@ --DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.8.0'" to load this file. \quit +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetDemographicSnapshot(text, text, geometry(Geometry, 4326), text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetSegmentSnapshot(text, text, geometry(Geometry, 4326), text); CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) RETURNS boolean AS $$ @@ -16,17 +18,16 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - return result -$$ LANGUAGE plpythonu; +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( username TEXT, @@ -34,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( geom geometry(Geometry, 4326), time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -50,11 +51,15 @@ RETURNS json AS $$ obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) quota_service.increment_success_service_use() - return result[0]['snapshot'] + return resp else: quota_service.increment_empty_service_use() - return None + return [] except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() @@ -66,23 +71,22 @@ RETURNS json AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - obs_plan = plpy.prepare("SELECT cdb_dataservices_server.OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - return result -$$ LANGUAGE plpythonu; +RETURNS SETOF json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -95,14 +99,18 @@ RETURNS json AS $$ plpy.error('You have reached the limit of your quota') try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) quota_service.increment_success_service_use() - return result[0]['snapshot'] + return resp else: quota_service.increment_empty_service_use() - return None + return [] except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() diff --git a/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql b/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql index e8a9f91..c171819 100644 --- a/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql +++ b/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql @@ -3,7 +3,9 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.4'" to load this file. \quit DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_config(text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetDemographicSnapshot(text, text, geometry(Geometry, 4326), text, text); DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetDemographicSnapshot(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetSegmentSnapshot(text, text, geometry(Geometry, 4326), text); DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetSegmentSnapshot(TEXT, TEXT, geometry(Geometry, 4326), TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT); @@ -34,79 +36,24 @@ DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPointsByGeometry(TEXT, T DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetPointsByPointAndRadius(TEXT, TEXT, geometry(Point, 4326), NUMERIC, TEXT, TEXT, TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPointsByPointAndRadius(TEXT, TEXT, geometry(Point, 4326), NUMERIC, TEXT, TEXT, TEXT); -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), time_span TEXT DEFAULT '2009 - 2013', geometry_level TEXT DEFAULT '"us.census.tiger".block_group') RETURNS json AS $$ - from cartodb_services.metrics import QuotaService - import json + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) - if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') - - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT '"us.census.tiger".block_group') RETURNS json AS $$ - from cartodb_services.metrics import QuotaService - import json - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) - if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') - - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu; \ No newline at end of file + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; \ No newline at end of file diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index bf57fcd..7b43e79 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -22,30 +22,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( geom geometry(Geometry, 4326), time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF json AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); + SELECT * FROM cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); $$ LANGUAGE plproxy; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT NULL, - geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - return result -$$ LANGUAGE plpythonu; - CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -61,11 +49,15 @@ RETURNS json AS $$ obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) quota_service.increment_success_service_use() - return result[0]['snapshot'] + return resp else: quota_service.increment_empty_service_use() - return None + return [] except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() @@ -82,28 +74,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( orgname TEXT, geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF json AS $$ CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); + SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); $$ LANGUAGE plproxy; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ - obs_plan = plpy.prepare("SELECT cdb_dataservices_server.OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - return result -$$ LANGUAGE plpythonu; - CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( username TEXT, orgname TEXT, geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) -RETURNS json AS $$ +RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -116,14 +97,18 @@ RETURNS json AS $$ plpy.error('You have reached the limit of your quota') try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) quota_service.increment_success_service_use() - return result[0]['snapshot'] + return resp else: quota_service.increment_empty_service_use() - return None + return [] except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 914991f..0cc1ac8 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -1,25 +1,3 @@ -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'obs_get_demographic_snapshot' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'obs_get_segment_snapshot' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); - exists --------- - t -(1 row) - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql index 227e10b..e8b9d85 100644 --- a/server/extension/test/sql/100_data_observatory_test.sql +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -1,17 +1,3 @@ -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'obs_get_demographic_snapshot' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'obs_get_segment_snapshot' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); - SELECT exists(SELECT * FROM pg_proc p INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) diff --git a/test/integration/test_data_observatory_functions.py b/test/integration/test_data_observatory_functions.py index 982fa66..1ebe647 100644 --- a/test/integration/test_data_observatory_functions.py +++ b/test/integration/test_data_observatory_functions.py @@ -15,19 +15,11 @@ class TestDataObservatoryFunctions(TestCase): ) def test_if_get_demographic_snapshot_is_ok(self): - query = "SELECT obs_get_demographic_snapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) - result = IntegrationTestHelper.execute_query(self.sql_api_url, query) - assert_not_equal(result['snapshot'], None) query = "SELECT obs_GetDemographicSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) result = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(result['snapshot'], None) def test_if_get_demographic_snapshot_without_api_key_raise_error(self): - query = "SELECT obs_get_demographic_snapshot(CDB_LatLng(40.704512, -73.936669));" - try: - IntegrationTestHelper.execute_query(self.sql_api_url, query) - except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") query = "SELECT obs_GetDemographicSnapshot(CDB_LatLng(40.704512, -73.936669));" try: IntegrationTestHelper.execute_query(self.sql_api_url, query) @@ -35,19 +27,11 @@ class TestDataObservatoryFunctions(TestCase): assert_equal(e.message[0], "The api_key must be provided") def test_if_get_segment_snapshot_is_ok(self): - query = "SELECT obs_get_segment_snapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) - result = IntegrationTestHelper.execute_query(self.sql_api_url, query) - assert_not_equal(result['snapshot'], None) query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669)) as snapshot;&api_key={0}".format(self.env_variables['api_key']) result = IntegrationTestHelper.execute_query(self.sql_api_url, query) assert_not_equal(result['snapshot'], None) def test_if_get_segment_snapshot_without_api_key_raise_error(self): - query = "SELECT obs_get_segment_snapshot(CDB_LatLng(40.704512, -73.936669));" - try: - IntegrationTestHelper.execute_query(self.sql_api_url, query) - except Exception as e: - assert_equal(e.message[0], "The api_key must be provided") query = "SELECT OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669));" try: IntegrationTestHelper.execute_query(self.sql_api_url, query)