From 4407ff630e8c46174b5b777b4e6c153b4c8667d7 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 25 Apr 2016 08:58:49 +0200 Subject: [PATCH] Change data observatory config file to segragate more in order to prepare for future observatory elements --- server/extension/Makefile | 4 +- .../cdb_dataservices_server--0.7.0--0.7.1.sql | 94 +++++++++++++++++++ .../cdb_dataservices_server--0.7.1--0.7.0.sql | 94 +++++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- .../cdb_dataservices_server--0.6.2--0.7.0.sql | 0 .../cdb_dataservices_server--0.7.0--0.6.2.sql | 0 .../cdb_dataservices_server--0.7.0.sql | 0 server/extension/sql/110_data_observatory.sql | 12 +-- server/extension/sql/15_config_helper.sql | 10 +- 9 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.6.2--0.7.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.7.0--0.6.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.7.0.sql (100%) diff --git a/server/extension/Makefile b/server/extension/Makefile index ca57d6e..f4500a5 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql) # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) \ - cdb_dataservices_server--0.6.2--0.7.0.sql \ - cdb_dataservices_server--0.7.0--0.6.2.sql + cdb_dataservices_server--0.7.0--0.7.1.sql \ + cdb_dataservices_server--0.7.1--0.7.0.sql REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) TEST_DIR = test/ diff --git a/server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql b/server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql new file mode 100644 index 0000000..0c4a66d --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql @@ -0,0 +1,94 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.1'" to load this file. \quit +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_data_observatory_config(text, text); +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT '2009 - 2013', + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql b/server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql new file mode 100644 index 0000000..c8b04ed --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql @@ -0,0 +1,94 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.0'" to load this file. \quit +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config(text, text); +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_data_observatory_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import DataObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = data_observatory_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT '2009 - 2013', + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + + quota_service = QuotaService(user_data_observatory_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_exception(type_, value_, traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + + quota_service = QuotaService(user_data_observatory_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 23be467..ad3e6d9 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.7.0' +default_version = '0.7.1' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.6.2--0.7.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.2--0.7.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.2--0.7.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.2--0.7.0.sql diff --git a/server/extension/cdb_dataservices_server--0.7.0--0.6.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.0--0.6.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.0--0.6.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.0--0.6.2.sql diff --git a/server/extension/cdb_dataservices_server--0.7.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.0.sql diff --git a/server/extension/sql/110_data_observatory.sql b/server/extension/sql/110_data_observatory.sql index b77b101..842a166 100644 --- a/server/extension/sql/110_data_observatory.sql +++ b/server/extension/sql/110_data_observatory.sql @@ -10,10 +10,10 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - quota_service = QuotaService(user_data_observatory_config, redis_conn) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') @@ -48,10 +48,10 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - quota_service = QuotaService(user_data_observatory_config, redis_conn) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 88b43c1..eb038fe 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -54,16 +54,16 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) RETURNS boolean AS $$ - cache_key = "user_data_observatory_config_{0}".format(username) + cache_key = "user_obs_snapshot_config_{0}".format(username) if cache_key in GD: return False else: - from cartodb_services.metrics import DataObservatoryConfig + from cartodb_services.metrics import ObservatorySnapshotConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = data_observatory_config + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config return True $$ LANGUAGE plpythonu SECURITY DEFINER;