From 3c5325c1d7714b50aa058ccb144ac1650fa2898d Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 19 Apr 2016 18:45:45 +0200 Subject: [PATCH] Server data observatory functions --- server/extension/sql/110_data_observatory.sql | 76 +++++++++++++++++++ server/extension/sql/15_config_helper.sql | 18 ++++- .../test/expected/00_install_test.out | 7 ++ .../expected/100_data_observatory_test.out | 22 ++++++ server/extension/test/sql/00_install_test.sql | 2 + .../test/sql/100_data_observatory_test.sql | 13 ++++ .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 35 +++++++++ .../cartodb_services/metrics/quota.py | 14 ++++ server/lib/python/cartodb_services/setup.py | 2 +- .../cartodb_services/test/test_helper.py | 2 + .../test/test_quota_service.py | 26 ++++++- 12 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 server/extension/sql/110_data_observatory.sql create mode 100644 server/extension/test/expected/100_data_observatory_test.out create mode 100644 server/extension/test/sql/100_data_observatory_test.sql diff --git a/server/extension/sql/110_data_observatory.sql b/server/extension/sql/110_data_observatory.sql new file mode 100644 index 0000000..b4394a1 --- /dev/null +++ b/server/extension/sql/110_data_observatory.sql @@ -0,0 +1,76 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT '2009 - 2013', + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + + quota_service = QuotaService(user_data_observatory_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + + quota_service = QuotaService(user_data_observatory_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index b521767..88b43c1 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -1,4 +1,3 @@ --- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_geocoder_config_{0}".format(username) @@ -13,7 +12,6 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; --- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_internal_geocoder_config_{0}".format(username) @@ -28,7 +26,6 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; --- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_isolines_routing_config_{0}".format(username) @@ -43,7 +40,6 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; --- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_routing_config_{0}".format(username) @@ -57,3 +53,17 @@ RETURNS boolean AS $$ GD[cache_key] = routing_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_data_observatory_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import DataObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = data_observatory_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 9c217f6..6e7916d 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers; CREATE EXTENSION plpythonu; CREATE EXTENSION cartodb; CREATE EXTENSION cdb_geocoder; +CREATE EXTENSION observatory VERSION 'dev'; -- Install the extension CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db @@ -37,6 +38,12 @@ SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null (1 row) +SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}'); + cdb_conf_setconf +------------------ + +(1 row) + -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ diff --git a/server/extension/test/expected/100_data_observatory_test.out b/server/extension/test/expected/100_data_observatory_test.out new file mode 100644 index 0000000..225382d --- /dev/null +++ b/server/extension/test/expected/100_data_observatory_test.out @@ -0,0 +1,22 @@ +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_get_demographic_snapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_get_segment_snapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index efd55f5..da9c8af 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -4,6 +4,7 @@ CREATE EXTENSION schema_triggers; CREATE EXTENSION plpythonu; CREATE EXTENSION cartodb; CREATE EXTENSION cdb_geocoder; +CREATE EXTENSION observatory VERSION 'dev'; -- Install the extension CREATE EXTENSION cdb_dataservices_server; @@ -14,6 +15,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); +SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"monthly_quota": 10000}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) diff --git a/server/extension/test/sql/100_data_observatory_test.sql b/server/extension/test/sql/100_data_observatory_test.sql new file mode 100644 index 0000000..7621cb2 --- /dev/null +++ b/server/extension/test/sql/100_data_observatory_test.sql @@ -0,0 +1,13 @@ +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_get_demographic_snapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'obs_get_segment_snapshot' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text'); \ No newline at end of file diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index af7bed5..c357ade 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException +from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, DataObservatoryConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 784de75..9b1b332 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -32,6 +32,28 @@ class ServiceConfig(object): def organization(self): return self._orgname +class DataObservatoryConfig(ServiceConfig): + + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_connection, db_conn, username, orgname=None): + super(DataObservatoryConfig, self).__init__(redis_connection, db_conn, + username, orgname) + self._monthly_quota = self._db_config.data_observatory_monthly_quota + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + + @property + def service_type(self): + return 'data_observatory' + + @property + def monthly_quota(self): + return self._monthly_quota + + @property + def period_end_date(self): + return self._period_end_date + class RoutingConfig(ServiceConfig): @@ -315,6 +337,7 @@ class ServicesDBConfig: self._get_here_config() self._get_mapzen_config() self._get_logger_config() + self._get_data_observatory_config() def _get_here_config(self): heremaps_conf_json = self._get_conf('heremaps_conf') @@ -340,6 +363,14 @@ class ServicesDBConfig: self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] + def _get_data_observatory_config(self): + do_conf_json = self._get_conf('data_observatory_conf') + if not do_conf_json: + raise ConfigException('Data Observatory configuration missing') + else: + do_conf = json.loads(do_conf_json) + self._data_observatory_monthly_quota = do_conf['monthly_quota'] + def _get_logger_config(self): logger_conf_json = self._get_conf('logger_conf') if not logger_conf_json: @@ -396,6 +427,10 @@ class ServicesDBConfig: def geocoder_log_path(self): return self._geocoder_log_path + @property + def data_observatory_monthly_quota(self): + return self._data_observatory_monthly_quota + class ServicesRedisConfig: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 238fb4e..b1d1137 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -73,6 +73,9 @@ class QuotaChecker: elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() + elif re.match('data_observatory', + self._user_service_config.service_type) is not None: + return self.__check_data_observatory_quota() else: return False @@ -114,3 +117,14 @@ class QuotaChecker: return True else: return False + + def __check_data_observatory_quota(self): + user_quota = self._user_service_config.monthly_quota + today = date.today() + service_type = self._user_service_config.service_type + current_used = self._user_service.used_quota(service_type, today) + + if (user_quota > 0 and current_used <= user_quota): + return True + else: + return False diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 562de35..6e218e8 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.5', + version='0.5.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 0fd2fd3..000d260 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -51,3 +51,5 @@ def _plpy_execute_side_effect(*args, **kwargs): return [{'conf': '{"routing": {"api_key": "valhalla-Z61FWEs", "monthly_quota": 1500000}, "geocoder": {"api_key": "search-d744tp0", "monthly_quota": 1500000}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] + elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": + return [{'conf': '{"monthly_quota": 100000}'}] diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 8f4414b..bcdacf8 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,7 +1,7 @@ import test_helper from mockredis import MockRedis from cartodb_services.metrics import QuotaService -from cartodb_services.metrics import GeocoderConfig, RoutingConfig +from cartodb_services.metrics import GeocoderConfig, RoutingConfig, DataObservatoryConfig from unittest import TestCase from nose.tools import assert_raises from datetime import datetime, date @@ -109,6 +109,20 @@ class TestQuotaService(TestCase): qs.increment_success_service_use(amount=1500000) assert qs.check_user_quota() is False + def test_should_check_user_data_observatory_quota_correctly(self): + qs = self.__build_data_observatory_quota_service('test_user') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=100000) + assert qs.check_user_quota() is False + + def test_should_check_org_data_observatory_quota_correctly(self): + qs = self.__build_data_observatory_quota_service('test_user', orgname='testorg') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=100000) + assert qs.check_user_quota() is False + def __prepare_quota_service(self, username, quota, service, orgname, soft_limit, end_date): test_helper.build_redis_user_config(self.redis_conn, username, @@ -139,3 +153,13 @@ class TestQuotaService(TestCase): routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) + + def __build_data_observatory_quota_service(self, username, quota=100, + service='data_observatory', orgname=None, + soft_limit=False, + end_date=datetime.today()): + self.__prepare_quota_service(username, quota, service, orgname, + soft_limit, end_date) + do_config = DataObservatoryConfig(self.redis_conn, self._plpy_mock, + username, orgname) + return QuotaService(do_config, redis_connection=self.redis_conn)