From 27be704bd642b3c58325f7297eb55c40de938778 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 18 Oct 2016 17:26:43 +0200 Subject: [PATCH 01/12] Changes needed to read quota from redis #281 Use per-user/org quota from redis, if present. Otherwise default to the quota stored in server database (as it did to date). --- .../cartodb_services/metrics/config.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index d6e7271..dc5a930 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -116,6 +116,7 @@ class RoutingConfig(ServiceConfig): ROUTING_PROVIDER_KEY = 'routing_provider' MAPZEN_PROVIDER = 'mapzen' DEFAULT_PROVIDER = 'mapzen' + QUOTA_KEY = 'mapzen_routing_quota' def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, db_conn, @@ -124,7 +125,7 @@ class RoutingConfig(ServiceConfig): if not self._routing_provider: self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key - self._monthly_quota = self._db_config.mapzen_routing_monthly_quota + self._set_monthly_quota() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property @@ -144,6 +145,19 @@ class RoutingConfig(ServiceConfig): def period_end_date(self): return self._period_end_date + def _set_monthly_quota(self): + self._monthly_quota = self._get_effective_monthly_quota() + + def _get_effective_monthly_quota(self): + quota_from_redis = self._redis_config[self.QUOTA_KEY] + if quota_from_redis and quota_from_redis <> '': + return int(quota_from_redis) + else: + return self._db_config.mapzen_routing_monthly_quota + + + + class IsolinesRoutingConfig(ServiceConfig): @@ -547,6 +561,7 @@ class ServicesRedisConfig: GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' QUOTA_KEY = 'geocoding_quota' ISOLINES_QUOTA_KEY = 'here_isolines_quota' + ROUTING_QUOTA_KEY = 'mapzen_routing_quota' OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota' OBS_GENERAL_QUOTA_KEY = 'obs_general_quota' PERIOD_END_DATE = 'period_end_date' @@ -574,6 +589,9 @@ class ServicesRedisConfig: if self.ROUTING_PROVIDER_KEY not in user_config: user_config[self.ROUTING_PROVIDER_KEY] = '' + # Mapzen routing quota might be not present + user_config[self.ROUTING_QUOTA_KEY] = user_config.get(self.ROUTING_QUOTA_KEY) + if orgname: self.__get_organization_config(orgname, user_config) @@ -587,6 +605,7 @@ class ServicesRedisConfig: else: user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] + user_config[self.ROUTING_QUOTA_KEY] = org_config.get(self.ROUTING_QUOTA_KEY) if self.OBS_SNAPSHOT_QUOTA_KEY in org_config: user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY] if self.OBS_GENERAL_QUOTA_KEY in org_config: From bbd9b6b98e3e74176bb7ce306687bdce1d27a07f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 18 Oct 2016 17:35:36 +0200 Subject: [PATCH 02/12] Add soft limit to config object #281 --- .../cartodb_services/metrics/config.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index dc5a930..b4fcf2e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -117,6 +117,7 @@ class RoutingConfig(ServiceConfig): MAPZEN_PROVIDER = 'mapzen' DEFAULT_PROVIDER = 'mapzen' QUOTA_KEY = 'mapzen_routing_quota' + SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, db_conn, @@ -126,6 +127,7 @@ class RoutingConfig(ServiceConfig): self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key self._set_monthly_quota() + self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property @@ -145,6 +147,11 @@ class RoutingConfig(ServiceConfig): def period_end_date(self): return self._period_end_date + @property + def soft_limit(self): + return self._soft_limit + + def _set_monthly_quota(self): self._monthly_quota = self._get_effective_monthly_quota() @@ -155,7 +162,11 @@ class RoutingConfig(ServiceConfig): else: return self._db_config.mapzen_routing_monthly_quota - + def _set_soft_limit(self): + if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_limit = True + else: + self._soft_limit = False From 72998c324a8a4cf7851402fd68b0f1475c89e808 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 18 Oct 2016 17:40:13 +0200 Subject: [PATCH 03/12] Simplify the code for regularity's sake #281 --- .../cartodb_services/cartodb_services/metrics/config.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index b4fcf2e..796c447 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -156,7 +156,7 @@ class RoutingConfig(ServiceConfig): self._monthly_quota = self._get_effective_monthly_quota() def _get_effective_monthly_quota(self): - quota_from_redis = self._redis_config[self.QUOTA_KEY] + quota_from_redis = self._redis_config.get(self.QUOTA_KEY) if quota_from_redis and quota_from_redis <> '': return int(quota_from_redis) else: @@ -600,9 +600,6 @@ class ServicesRedisConfig: if self.ROUTING_PROVIDER_KEY not in user_config: user_config[self.ROUTING_PROVIDER_KEY] = '' - # Mapzen routing quota might be not present - user_config[self.ROUTING_QUOTA_KEY] = user_config.get(self.ROUTING_QUOTA_KEY) - if orgname: self.__get_organization_config(orgname, user_config) @@ -616,7 +613,8 @@ class ServicesRedisConfig: else: user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] - user_config[self.ROUTING_QUOTA_KEY] = org_config.get(self.ROUTING_QUOTA_KEY) + if self.ROUTING_QUOTA_KEY in org_config: + user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY] if self.OBS_SNAPSHOT_QUOTA_KEY in org_config: user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY] if self.OBS_GENERAL_QUOTA_KEY in org_config: From aaff5564ec9a13863291ec527b84f31d50250f6a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 18 Oct 2016 17:44:51 +0200 Subject: [PATCH 04/12] Use soft_limit for `__check_routing_quota` #281 --- .../python/cartodb_services/cartodb_services/metrics/quota.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 13aefc7..16c1df7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -115,8 +115,9 @@ class QuotaChecker: today = date.today() service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) + soft_limit = self._user_service_config.soft_limit - if (user_quota > 0 and current_used <= user_quota): + if soft_limit or (user_quota > 0 and current_used <= user_quota): return True else: return False From 2b1b1c981f5441e36d060cf583d0ad56edc13f23 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 19 Oct 2016 17:41:43 +0200 Subject: [PATCH 05/12] Add routing to UserMetricsService.used_quota --- .../python/cartodb_services/cartodb_services/metrics/user.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 30bd5a1..1c31aae 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -8,6 +8,7 @@ class UserMetricsService: SERVICE_GEOCODER_NOKIA = 'geocoder_here' SERVICE_GEOCODER_CACHE = 'geocoder_cache' SERVICE_HERE_ISOLINES = 'here_isolines' + SERVICE_MAPZEN_ROUTING = 'routing_mapzen' DAY_OF_MONTH_ZERO_PADDED = '%d' def __init__(self, user_geocoder_config, redis_connection): @@ -19,6 +20,8 @@ class UserMetricsService: def used_quota(self, service_type, date): if service_type == self.SERVICE_HERE_ISOLINES: return self.__used_isolines_quota(service_type, date) + elif service_type == self.SERVICE_MAPZEN_ROUTING: + return self.__used_routing_quota(service_type, date) else: return self.__used_geocoding_quota(service_type, date) From 247034c21e1d14d8f6eb0d1fa1579c931f7edaa7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 19 Oct 2016 18:11:37 +0200 Subject: [PATCH 06/12] Some tests for RoutingConfig #281 --- .../cartodb_services/test/metrics/__init__.py | 0 .../test/metrics/test_config.py | 50 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/metrics/__init__.py create mode 100644 server/lib/python/cartodb_services/test/metrics/test_config.py diff --git a/server/lib/python/cartodb_services/test/metrics/__init__.py b/server/lib/python/cartodb_services/test/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py new file mode 100644 index 0000000..62e3301 --- /dev/null +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -0,0 +1,50 @@ +from unittest import TestCase +from mockredis import MockRedis +from cartodb_services.metrics import RoutingConfig +from ..test_helper import build_plpy_mock + +class TestRoutingConfig(TestCase): + + def setUp(self): + self._redis_conn = MockRedis() + self._db_conn = build_plpy_mock() + self._username = 'my_test_user' + self._user_key = "rails:users:{0}".format(self._username) + self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') + + def test_should_pick_quota_from_server_by_default(self): + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.monthly_quota == 1500000 + + def test_should_pick_quota_from_redis_if_present(self): + self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.monthly_quota == 1000 + + def test_org_quota_overrides_user_quota(self): + self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) + orgname = 'my_test_org' + orgname_key = "rails:orgs:{0}".format(orgname) + self._redis_conn.hset(orgname_key, 'period_end_date', '2016-05-31') + self._redis_conn.hset(orgname_key, 'mapzen_routing_quota', 5000) + + # TODO: these are not too relevant for the routing config + self._redis_conn.hset(orgname_key, 'geocoding_quota', 0) + self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0) + + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.monthly_quota == 5000 + + + def test_should_have_soft_limit_false_by_default(self): + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.soft_limit == False + + def test_can_set_soft_limit_in_user_conf(self): + self._redis_conn.hset(self._user_key, 'soft_mapzen_routing_limit', True) + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.soft_limit == True From 0c49107f967592d9c834f8b9a1b740b8160c7e8c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 20 Oct 2016 13:13:30 +0200 Subject: [PATCH 07/12] Some tests for the QuotaChecker #281 --- .../test/metrics/test_quota.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/metrics/test_quota.py diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py new file mode 100644 index 0000000..924ccd7 --- /dev/null +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -0,0 +1,66 @@ +from unittest import TestCase +from mockredis import MockRedis +from ..test_helper import build_plpy_mock +from cartodb_services.metrics.quota import QuotaChecker +from cartodb_services.metrics import RoutingConfig +from datetime import datetime + + +class RoutingConfigMock(object): + + def __init__(self, **kwargs): + self.__dict__ = kwargs + + +class TestQuotaChecker(TestCase): + + USERNAME = 'my_test_user' + PERIOD_END_DATE = datetime(2016,10,20) + SERVICE_TYPE = 'routing_mapzen' + REDIS_KEY = 'user:{0}:{1}:success_responses:{2}{3}'.format( + USERNAME, + SERVICE_TYPE, + PERIOD_END_DATE.year, + PERIOD_END_DATE.month + ) + + def test_routing_quota_check_passes_when_enough_quota(self): + user_service_config = RoutingConfigMock( + username = self.USERNAME, + organization = None, + service_type = self.SERVICE_TYPE, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = False + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 999) + assert QuotaChecker(user_service_config, redis_conn).check() == True + + def test_routing_quota_check_fails_when_quota_exhausted(self): + user_service_config = RoutingConfigMock( + username = self.USERNAME, + organization = None, + service_type = self.SERVICE_TYPE, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = False + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 1001) + checker = QuotaChecker(user_service_config, redis_conn) + assert checker.check() == False + + def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self): + user_service_config = RoutingConfigMock( + username = self.USERNAME, + organization = None, + service_type = self.SERVICE_TYPE, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = True + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 1001) + checker = QuotaChecker(user_service_config, redis_conn) + assert checker.check() == True From e1a7d1751cf48b562e28cc3f04f23ee9eccd2969 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 20 Oct 2016 15:36:36 +0200 Subject: [PATCH 08/12] Fix corner case: used_quota == monthly_quota #281 --- .../cartodb_services/metrics/quota.py | 8 ++++---- .../test/metrics/test_quota.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 16c1df7..ea54508 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -93,7 +93,7 @@ class QuotaChecker: current_used = self._user_service.used_quota(service_type, today) soft_geocoding_limit = self._user_service_config.soft_geocoding_limit - if soft_geocoding_limit or (user_quota > 0 and current_used <= user_quota): + if soft_geocoding_limit or (user_quota > 0 and current_used < user_quota): return True else: return False @@ -105,7 +105,7 @@ class QuotaChecker: current_used = self._user_service.used_quota(service_type, today) soft_isolines_limit = self._user_service_config.soft_isolines_limit - if soft_isolines_limit or (user_quota > 0 and current_used <= user_quota): + if soft_isolines_limit or (user_quota > 0 and current_used < user_quota): return True else: return False @@ -117,7 +117,7 @@ class QuotaChecker: current_used = self._user_service.used_quota(service_type, today) soft_limit = self._user_service_config.soft_limit - if soft_limit or (user_quota > 0 and current_used <= user_quota): + if soft_limit or (user_quota > 0 and current_used < user_quota): return True else: return False @@ -129,7 +129,7 @@ class QuotaChecker: service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) - if soft_limit or (user_quota > 0 and current_used <= user_quota): + if soft_limit or (user_quota > 0 and current_used < user_quota): return True else: return False diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py index 924ccd7..8dfff65 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_quota.py +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -51,6 +51,24 @@ class TestQuotaChecker(TestCase): checker = QuotaChecker(user_service_config, redis_conn) assert checker.check() == False + def test_routing_quota_check_fails_right_in_the_limit(self): + """ + I have 1000 credits and I just spent 1000 today. I should not pass + the check to perform the 1001th routing operation. + """ + user_service_config = RoutingConfigMock( + username = self.USERNAME, + organization = None, + service_type = self.SERVICE_TYPE, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = False + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 1000) + checker = QuotaChecker(user_service_config, redis_conn) + assert checker.check() == False + def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self): user_service_config = RoutingConfigMock( username = self.USERNAME, From cc2ab1bc0ce12689cd83d72e1a1de3d8f0716a02 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 21 Oct 2016 15:08:14 +0200 Subject: [PATCH 09/12] Fix test: make it independent of current date #281 --- .../test/metrics/test_quota.py | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py index 8dfff65..c56756c 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_quota.py +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -14,40 +14,41 @@ class RoutingConfigMock(object): class TestQuotaChecker(TestCase): - USERNAME = 'my_test_user' - PERIOD_END_DATE = datetime(2016,10,20) - SERVICE_TYPE = 'routing_mapzen' - REDIS_KEY = 'user:{0}:{1}:success_responses:{2}{3}'.format( - USERNAME, - SERVICE_TYPE, - PERIOD_END_DATE.year, - PERIOD_END_DATE.month - ) + def setUp(self): + self.username = 'my_test_user' + self.period_end_date = datetime.today() + self.service_type = 'routing_mapzen' + self.redis_key = 'user:{0}:{1}:success_responses:{2}{3}'.format( + self.username, + self.service_type, + self.period_end_date.year, + self.period_end_date.month + ) def test_routing_quota_check_passes_when_enough_quota(self): user_service_config = RoutingConfigMock( - username = self.USERNAME, + username = self.username, organization = None, - service_type = self.SERVICE_TYPE, + service_type = self.service_type, monthly_quota = 1000, period_end_date = datetime.today(), soft_limit = False ) redis_conn = MockRedis() - redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 999) + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 999) assert QuotaChecker(user_service_config, redis_conn).check() == True def test_routing_quota_check_fails_when_quota_exhausted(self): user_service_config = RoutingConfigMock( - username = self.USERNAME, + username = self.username, organization = None, - service_type = self.SERVICE_TYPE, + service_type = self.service_type, monthly_quota = 1000, period_end_date = datetime.today(), soft_limit = False ) redis_conn = MockRedis() - redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 1001) + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001) checker = QuotaChecker(user_service_config, redis_conn) assert checker.check() == False @@ -57,28 +58,28 @@ class TestQuotaChecker(TestCase): the check to perform the 1001th routing operation. """ user_service_config = RoutingConfigMock( - username = self.USERNAME, + username = self.username, organization = None, - service_type = self.SERVICE_TYPE, + service_type = self.service_type, monthly_quota = 1000, period_end_date = datetime.today(), soft_limit = False ) redis_conn = MockRedis() - redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 1000) + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1000) checker = QuotaChecker(user_service_config, redis_conn) assert checker.check() == False def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self): user_service_config = RoutingConfigMock( - username = self.USERNAME, + username = self.username, organization = None, - service_type = self.SERVICE_TYPE, + service_type = self.service_type, monthly_quota = 1000, period_end_date = datetime.today(), soft_limit = True ) redis_conn = MockRedis() - redis_conn.zincrby(self.REDIS_KEY, self.PERIOD_END_DATE.day, 1001) + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001) checker = QuotaChecker(user_service_config, redis_conn) assert checker.check() == True From 8e02c64aeb50875b6384823801717d5461b16591 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 21 Oct 2016 15:17:35 +0200 Subject: [PATCH 10/12] Tests for UserMetricsService, mapzen routing #281 --- .../test/metrics/test_user.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 server/lib/python/cartodb_services/test/metrics/test_user.py diff --git a/server/lib/python/cartodb_services/test/metrics/test_user.py b/server/lib/python/cartodb_services/test/metrics/test_user.py new file mode 100644 index 0000000..88b8e65 --- /dev/null +++ b/server/lib/python/cartodb_services/test/metrics/test_user.py @@ -0,0 +1,30 @@ +from unittest import TestCase +from cartodb_services.metrics import UserMetricsService +import datetime +from mockredis import MockRedis + +class UserGeocoderConfig(object): + + def __init__(self, **kwargs): + self.__dict__ = kwargs + + +class TestUserMetricsService(TestCase): + + def setUp(self): + user_geocoder_config = UserGeocoderConfig( + username = 'my_test_user', + organization = None, + period_end_date = datetime.date.today() + ) + redis_conn = MockRedis() + self.user_metrics_service = UserMetricsService(user_geocoder_config, redis_conn) + + + def test_routing_used_quota_zero_when_no_usage(self): + assert self.user_metrics_service.used_quota(UserMetricsService.SERVICE_MAPZEN_ROUTING, datetime.date.today()) == 0 + + def test_routing_used_quota_counts_usages(self): + self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'success_responses') + self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'empty_responses') + assert self.user_metrics_service.used_quota('routing_mapzen', datetime.date.today()) == 2 From db80d389e0bcb3eae80f9a736d275715ce7d5fe7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 21 Oct 2016 15:38:56 +0200 Subject: [PATCH 11/12] Do not require org geocoder and isolines quota to be present --- .../cartodb_services/cartodb_services/metrics/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 796c447..8c739fa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -611,8 +611,10 @@ class ServicesRedisConfig: if not org_config: raise ConfigException("""There is no organization config available. Please check your configuration.'""") else: - user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] - user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] + if self.QUOTA_KEY in org_config: + user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] + if self.ISOLINES_QUOTA_KEY in org_config: + user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] if self.ROUTING_QUOTA_KEY in org_config: user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY] if self.OBS_SNAPSHOT_QUOTA_KEY in org_config: From d4ac2eb5e697dd85deda377c58ee36cf51e9f5ec Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 21 Oct 2016 15:40:20 +0200 Subject: [PATCH 12/12] Tests for ServicesRedisConfig (mapzen quota) #281 --- .../test/metrics/test_config.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 62e3301..a9142f0 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -1,6 +1,6 @@ from unittest import TestCase from mockredis import MockRedis -from cartodb_services.metrics import RoutingConfig +from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig from ..test_helper import build_plpy_mock class TestRoutingConfig(TestCase): @@ -48,3 +48,20 @@ class TestRoutingConfig(TestCase): orgname = None config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) assert config.soft_limit == True + + +class TestServicesRedisConfig(TestCase): + def test_it_picks_mapzen_routing_quota_from_redis(self): + redis_conn = MockRedis() + redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42) + redis_config = ServicesRedisConfig(redis_conn).build('my_username', None) + assert 'mapzen_routing_quota' in redis_config + assert int(redis_config['mapzen_routing_quota']) == 42 + + def test_org_quota_overrides_user_quota(self): + redis_conn = MockRedis() + redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42) + redis_conn.hset('rails:orgs:acme', 'mapzen_routing_quota', 31415) + redis_config = ServicesRedisConfig(redis_conn).build('my_username', 'acme') + assert 'mapzen_routing_quota' in redis_config + assert int(redis_config['mapzen_routing_quota']) == 31415