From c7c9e6e284a59f11c2ebfecdec795421eecf50c6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 11 May 2016 11:50:21 +0200 Subject: [PATCH] Server side new data observatory functions (v0.8.0) --- server/extension/Makefile | 4 +- .../cdb_dataservices_server--0.7.4--0.8.0.sql | 862 ++++++++++++++++++ .../cdb_dataservices_server--0.8.0--0.7.4.sql | 112 +++ .../extension/cdb_dataservices_server.control | 2 +- .../cdb_dataservices_server--0.7.3--0.7.4.sql | 0 .../cdb_dataservices_server--0.7.4--0.7.3.sql | 0 .../cdb_dataservices_server--0.7.4.sql | 0 server/extension/sql/110_data_observatory.sql | 115 --- .../sql/110_data_observatory_augmentation.sql | 397 ++++++++ .../sql/115_data_observatory_exploration.sql | 108 +++ .../sql/120_data_observatory_geometries.sql | 374 ++++++++ server/extension/sql/15_config_helper.sql | 14 + .../expected/100_data_observatory_test.out | 176 ++++ .../test/sql/100_data_observatory_test.sql | 114 ++- .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 76 +- .../cartodb_services/metrics/quota.py | 9 +- server/lib/python/cartodb_services/setup.py | 2 +- .../cartodb_services/test/test_config.py | 2 +- .../cartodb_services/test/test_helper.py | 2 +- 20 files changed, 2216 insertions(+), 155 deletions(-) create mode 100644 server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.7.3--0.7.4.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.7.4--0.7.3.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.7.4.sql (100%) delete mode 100644 server/extension/sql/110_data_observatory.sql create mode 100644 server/extension/sql/110_data_observatory_augmentation.sql create mode 100644 server/extension/sql/115_data_observatory_exploration.sql create mode 100644 server/extension/sql/120_data_observatory_geometries.sql diff --git a/server/extension/Makefile b/server/extension/Makefile index 0acc452..1c869fc 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql) # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) \ - cdb_dataservices_server--0.7.3--0.7.4.sql \ - cdb_dataservices_server--0.7.4--0.7.3.sql + cdb_dataservices_server--0.7.4--0.8.0.sql \ + cdb_dataservices_server--0.8.0--0.7.4.sql REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) TEST_DIR = test/ diff --git a/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql b/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql new file mode 100644 index 0000000..2b7bf8b --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.7.4--0.8.0.sql @@ -0,0 +1,862 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.8.0'" to load this file. \quit + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + obs_plan = plpy.prepare("SELECT cdb_dataservices_server.OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetMeasure: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetCategory: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetUSCensusMeasure: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetUSCensusCategory: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetPopulation: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4) as search;", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_Search: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetAvailableBoundaries: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundary: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use obs_search: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundaryById: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundariesByGeometry: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundariesByPointAndRadius: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetPointsByGeometry: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetPointsByPointAndRadius: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + + diff --git a/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql b/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql new file mode 100644 index 0000000..e8a9f91 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.8.0--0.7.4.sql @@ -0,0 +1,112 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.4'" to load this file. \quit + +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_config(text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetDemographicSnapshot(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetSegmentSnapshot(TEXT, TEXT, geometry(Geometry, 4326), TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetCategory(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetCategory(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetUSCensusMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetUSCensusMeasure(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetUSCensusCategory(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetUSCensusCategory(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetPopulation(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPopulation(TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_Search(TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_Search(TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetAvailableBoundaries(TEXT, TEXT, geometry(Geometry, 4326), TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetAvailableBoundaries(TEXT, TEXT, geometry(Geometry, 4326), TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetBoundary(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetBoundary(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetBoundaryId(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetBoundaryId(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetBoundaryById(TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetBoundaryById(TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetBoundariesByGeometry(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetBoundariesByGeometry(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius(TEXT, TEXT, NUMERIC, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(TEXT, TEXT, NUMERIC, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetPointsByGeometry(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPointsByGeometry(TEXT, TEXT, geometry(Point, 4326), TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetPointsByPointAndRadius(TEXT, TEXT, geometry(Point, 4326), NUMERIC, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetPointsByPointAndRadius(TEXT, TEXT, geometry(Point, 4326), NUMERIC, TEXT, TEXT, TEXT); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT '2009 - 2013', + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index cc9db8e..19613d1 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.7.4' +default_version = '0.8.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.7.3--0.7.4.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.3--0.7.4.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.3--0.7.4.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.3--0.7.4.sql diff --git a/server/extension/cdb_dataservices_server--0.7.4--0.7.3.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.4--0.7.3.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.4--0.7.3.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.4--0.7.3.sql diff --git a/server/extension/cdb_dataservices_server--0.7.4.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.4.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.4.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.4.sql diff --git a/server/extension/sql/110_data_observatory.sql b/server/extension/sql/110_data_observatory.sql deleted file mode 100644 index 1f3075b..0000000 --- a/server/extension/sql/110_data_observatory.sql +++ /dev/null @@ -1,115 +0,0 @@ --- --- Observatory connection config --- --- The purpose of this function is provide to the PL/Proxy functions --- the connection string needed to connect with the current production database - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( - username TEXT, - orgname TEXT) -RETURNS text AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - - return user_obs_snapshot_config.connection_str -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT '2009 - 2013', - geometry_level TEXT DEFAULT '"us.census.tiger".block_group') -RETURNS json AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - time_span TEXT DEFAULT '2009 - 2013', - geometry_level TEXT DEFAULT '"us.census.tiger".block_group') -RETURNS json AS $$ - from cartodb_services.metrics import QuotaService - import json - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) - if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') - - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT '"us.census.tiger".block_group') -RETURNS json AS $$ - CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); - SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); -$$ LANGUAGE plproxy; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( - username TEXT, - orgname TEXT, - geom geometry(Geometry, 4326), - geometry_level TEXT DEFAULT '"us.census.tiger".block_group') -RETURNS json AS $$ - from cartodb_services.metrics import QuotaService - import json - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) - if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') - - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql new file mode 100644 index 0000000..793958d --- /dev/null +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -0,0 +1,397 @@ +-- +-- Observatory connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the current production database + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str( + username TEXT, + orgname TEXT) +RETURNS text AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + return user_obs_snapshot_config.connection_str +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetDemographicSnapshot(geom, time_span, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL, + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + obs_plan = plpy.prepare("SELECT cdb_dataservices_server.OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT NULL) +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetMeasure(geom, measure_id, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + measure_id TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetMeasure: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetCategory(geom, category_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + category_id TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetCategory: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusMeasure(geom, name, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetUSCensusMeasure: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetUSCensusCategory(geom, name, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + name TEXT, + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetUSCensusCategory: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetPopulation(geom, normalize, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + normalize TEXT DEFAULT 'area', + boundary_id TEXT DEFAULT NULL, + time_span TEXT DEFAULT NULL) +RETURNS NUMERIC AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetPopulation: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql new file mode 100644 index 0000000..fbf38cc --- /dev/null +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -0,0 +1,108 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_Search(search_term, relevant_boundary); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( + username TEXT, + orgname TEXT, + search_term TEXT, + relevant_boundary TEXT DEFAULT NULL) +RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_Search: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetAvailableBoundaries(geom, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT NULL) +RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetAvailableBoundaries: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql new file mode 100644 index 0000000..f61d8e1 --- /dev/null +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -0,0 +1,374 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundary(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundary: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_OBS_GetBoundaryId(geom, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS TEXT AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use obs_search: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT cdb_observatory.OBS_GetBoundaryById(geometry_id, boundary_id, time_span); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( + username TEXT, + orgname TEXT, + geometry_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL) +RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundaryById: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type text DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundariesByGeometry: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetBoundariesByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetBoundariesByPointAndRadius: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByGeometry(geom, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetPointsByGeometry: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + SELECT * FROM cdb_observatory.OBS_GetPointsByPointAndRadius(geom, radius, boundary_id, time_span, overlap_type); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius( + username TEXT, + orgname TEXT, + geom geometry(Point, 4326), + radius NUMERIC, + boundary_id TEXT, + time_span TEXT DEFAULT NULL, + overlap_type TEXT DEFAULT 'intersects') +RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import QuotaService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use OBS_GetPointsByPointAndRadius: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index eb038fe..68726ff 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -67,3 +67,17 @@ RETURNS boolean AS $$ GD[cache_key] = obs_snapshot_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out index 225382d..914991f 100644 --- a/server/extension/test/expected/100_data_observatory_test.out +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -20,3 +20,179 @@ SELECT exists(SELECT * t (1 row) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdemographicsnapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getsegmentsnapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getmeasure' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getcategory' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getuscensusmeasure' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getuscensuscategory' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getpopulation' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_search' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getavailableboundaries' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundary' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundaryid' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundarybyid' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundariesbygeometry' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundariesbypointandradius' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getpointsbygeometry' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getpointsbypointandradius' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql index 7621cb2..227e10b 100644 --- a/server/extension/test/sql/100_data_observatory_test.sql +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -10,4 +10,116 @@ SELECT exists(SELECT * INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) WHERE ns.nspname = 'cdb_dataservices_server' AND proname = 'obs_get_segment_snapshot' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); \ No newline at end of file + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getdemographicsnapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getsegmentsnapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getmeasure' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getcategory' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getuscensusmeasure' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getuscensuscategory' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getpopulation' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_search' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getavailableboundaries' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundary' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundaryid' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundarybyid' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundariesbygeometry' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getboundariesbypointandradius' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getpointsbygeometry' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_getpointsbypointandradius' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, numeric, text, text, text'); \ No newline at end of file diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 0807d55..078bbbd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig +from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 141aa90..b1f110f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -33,32 +33,11 @@ class ServiceConfig(object): def organization(self): return self._orgname - -class ObservatorySnapshotConfig(ServiceConfig): - - SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit' - QUOTA_KEY = 'obs_snapshot_quota' - PERIOD_END_DATE = 'period_end_date' +class DataObservatoryConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): - super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn, + super(DataObservatoryConfig, self).__init__(redis_connection, db_conn, username, orgname) - self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) - if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': - self._soft_limit = True - else: - self._soft_limit = False - # Mixed config between db and redis. If we don't update all the users - # in redis, we could use the db value as default - if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) - else: - self._monthly_quota = float(self._db_config.data_observatory_monthly_quota) - self._connection_str = self._db_config.data_observatory_connection_str - - @property - def service_type(self): - return 'obs_snapshot' @property def monthly_quota(self): @@ -76,6 +55,52 @@ class ObservatorySnapshotConfig(ServiceConfig): def connection_str(self): return self._connection_str +class ObservatorySnapshotConfig(DataObservatoryConfig): + + SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit' + QUOTA_KEY = 'obs_snapshot_quota' + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_connection, db_conn, username, orgname=None): + super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn, + username, orgname) + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_limit = True + else: + self._soft_limit = False + self._monthly_quota = 0 + if self.QUOTA_KEY in self._redis_config: + self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + self._connection_str = self._db_config.data_observatory_connection_str + + @property + def service_type(self): + return 'obs_snapshot' + +class ObservatoryConfig(DataObservatoryConfig): + + SOFT_LIMIT_KEY = 'soft_obs_general_limit' + QUOTA_KEY = 'obs_general_quota' + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_connection, db_conn, username, orgname=None): + super(ObservatoryConfig, self).__init__(redis_connection, db_conn, + username, orgname) + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_limit = True + else: + self._soft_limit = False + self._monthly_quota = 0 + if self.QUOTA_KEY in self._redis_config: + self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + self._connection_str = self._db_config.data_observatory_connection_str + + @property + def service_type(self): + return 'obs_general' + class RoutingConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' @@ -370,7 +395,6 @@ class ServicesDBConfig: raise ConfigException('Data Observatory configuration missing') else: do_conf = json.loads(do_conf_json) - self._data_observatory_monthly_quota = do_conf['monthly_quota'] if self._orgname and self._orgname in do_conf['connection']['whitelist']: self._data_observatory_connection_str = do_conf['connection']['staging'] elif self._username in do_conf['connection']['whitelist']: @@ -434,10 +458,6 @@ class ServicesDBConfig: def geocoder_log_path(self): return self._geocoder_log_path - @property - def data_observatory_monthly_quota(self): - return self._data_observatory_monthly_quota - @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 2460082..c627769 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -73,9 +73,9 @@ class QuotaChecker: elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() - elif re.match('obs_snapshot', + elif re.match('obs_*', self._user_service_config.service_type) is not None: - return self.__check_obs_snapshot_quota() + return self.__check_data_observatory_quota() else: return False @@ -118,13 +118,14 @@ class QuotaChecker: else: return False - def __check_obs_snapshot_quota(self): + def __check_data_observatory_quota(self): user_quota = self._user_service_config.monthly_quota + soft_limit = self._user_service_config.soft_limit today = date.today() service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) - if (user_quota > 0 and current_used <= user_quota): + if soft_limit or (user_quota > 0 and current_used <= user_quota): return True else: return False diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 5808b69..3254050 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.5.3', + version='0.6.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index 8d980d2..4a639ec 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -49,7 +49,7 @@ class TestConfig(TestCase): end_date=yesterday) do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, 'test_user') - assert do_config.monthly_quota == 100000 + assert do_config.monthly_quota == 0 assert do_config.soft_limit is False assert do_config.period_end_date.date() == yesterday.date() diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 41d58e4..e13fa65 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -61,4 +61,4 @@ def _plpy_execute_side_effect(*args, **kwargs): elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": - return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'}] + return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]