diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index d6e7271..8c739fa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -116,6 +116,8 @@ class RoutingConfig(ServiceConfig): ROUTING_PROVIDER_KEY = 'routing_provider' MAPZEN_PROVIDER = 'mapzen' DEFAULT_PROVIDER = 'mapzen' + QUOTA_KEY = 'mapzen_routing_quota' + SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, db_conn, @@ -124,7 +126,8 @@ class RoutingConfig(ServiceConfig): if not self._routing_provider: self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key - self._monthly_quota = self._db_config.mapzen_routing_monthly_quota + self._set_monthly_quota() + self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property @@ -144,6 +147,28 @@ class RoutingConfig(ServiceConfig): def period_end_date(self): return self._period_end_date + @property + def soft_limit(self): + return self._soft_limit + + + def _set_monthly_quota(self): + self._monthly_quota = self._get_effective_monthly_quota() + + def _get_effective_monthly_quota(self): + quota_from_redis = self._redis_config.get(self.QUOTA_KEY) + if quota_from_redis and quota_from_redis <> '': + return int(quota_from_redis) + else: + return self._db_config.mapzen_routing_monthly_quota + + def _set_soft_limit(self): + if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_limit = True + else: + self._soft_limit = False + + class IsolinesRoutingConfig(ServiceConfig): @@ -547,6 +572,7 @@ class ServicesRedisConfig: GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' QUOTA_KEY = 'geocoding_quota' ISOLINES_QUOTA_KEY = 'here_isolines_quota' + ROUTING_QUOTA_KEY = 'mapzen_routing_quota' OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota' OBS_GENERAL_QUOTA_KEY = 'obs_general_quota' PERIOD_END_DATE = 'period_end_date' @@ -585,8 +611,12 @@ class ServicesRedisConfig: if not org_config: raise ConfigException("""There is no organization config available. Please check your configuration.'""") else: - user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] - user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] + if self.QUOTA_KEY in org_config: + user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] + if self.ISOLINES_QUOTA_KEY in org_config: + user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] + if self.ROUTING_QUOTA_KEY in org_config: + user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY] if self.OBS_SNAPSHOT_QUOTA_KEY in org_config: user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY] if self.OBS_GENERAL_QUOTA_KEY in org_config: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 13aefc7..ea54508 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -93,7 +93,7 @@ class QuotaChecker: current_used = self._user_service.used_quota(service_type, today) soft_geocoding_limit = self._user_service_config.soft_geocoding_limit - if soft_geocoding_limit or (user_quota > 0 and current_used <= user_quota): + if soft_geocoding_limit or (user_quota > 0 and current_used < user_quota): return True else: return False @@ -105,7 +105,7 @@ class QuotaChecker: current_used = self._user_service.used_quota(service_type, today) soft_isolines_limit = self._user_service_config.soft_isolines_limit - if soft_isolines_limit or (user_quota > 0 and current_used <= user_quota): + if soft_isolines_limit or (user_quota > 0 and current_used < user_quota): return True else: return False @@ -115,8 +115,9 @@ class QuotaChecker: today = date.today() service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) + soft_limit = self._user_service_config.soft_limit - if (user_quota > 0 and current_used <= user_quota): + if soft_limit or (user_quota > 0 and current_used < user_quota): return True else: return False @@ -128,7 +129,7 @@ class QuotaChecker: service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) - if soft_limit or (user_quota > 0 and current_used <= user_quota): + if soft_limit or (user_quota > 0 and current_used < user_quota): return True else: return False diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 30bd5a1..1c31aae 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -8,6 +8,7 @@ class UserMetricsService: SERVICE_GEOCODER_NOKIA = 'geocoder_here' SERVICE_GEOCODER_CACHE = 'geocoder_cache' SERVICE_HERE_ISOLINES = 'here_isolines' + SERVICE_MAPZEN_ROUTING = 'routing_mapzen' DAY_OF_MONTH_ZERO_PADDED = '%d' def __init__(self, user_geocoder_config, redis_connection): @@ -19,6 +20,8 @@ class UserMetricsService: def used_quota(self, service_type, date): if service_type == self.SERVICE_HERE_ISOLINES: return self.__used_isolines_quota(service_type, date) + elif service_type == self.SERVICE_MAPZEN_ROUTING: + return self.__used_routing_quota(service_type, date) else: return self.__used_geocoding_quota(service_type, date) diff --git a/server/lib/python/cartodb_services/test/metrics/__init__.py b/server/lib/python/cartodb_services/test/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py new file mode 100644 index 0000000..a9142f0 --- /dev/null +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -0,0 +1,67 @@ +from unittest import TestCase +from mockredis import MockRedis +from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig +from ..test_helper import build_plpy_mock + +class TestRoutingConfig(TestCase): + + def setUp(self): + self._redis_conn = MockRedis() + self._db_conn = build_plpy_mock() + self._username = 'my_test_user' + self._user_key = "rails:users:{0}".format(self._username) + self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') + + def test_should_pick_quota_from_server_by_default(self): + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.monthly_quota == 1500000 + + def test_should_pick_quota_from_redis_if_present(self): + self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.monthly_quota == 1000 + + def test_org_quota_overrides_user_quota(self): + self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000) + orgname = 'my_test_org' + orgname_key = "rails:orgs:{0}".format(orgname) + self._redis_conn.hset(orgname_key, 'period_end_date', '2016-05-31') + self._redis_conn.hset(orgname_key, 'mapzen_routing_quota', 5000) + + # TODO: these are not too relevant for the routing config + self._redis_conn.hset(orgname_key, 'geocoding_quota', 0) + self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0) + + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.monthly_quota == 5000 + + + def test_should_have_soft_limit_false_by_default(self): + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.soft_limit == False + + def test_can_set_soft_limit_in_user_conf(self): + self._redis_conn.hset(self._user_key, 'soft_mapzen_routing_limit', True) + orgname = None + config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname) + assert config.soft_limit == True + + +class TestServicesRedisConfig(TestCase): + def test_it_picks_mapzen_routing_quota_from_redis(self): + redis_conn = MockRedis() + redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42) + redis_config = ServicesRedisConfig(redis_conn).build('my_username', None) + assert 'mapzen_routing_quota' in redis_config + assert int(redis_config['mapzen_routing_quota']) == 42 + + def test_org_quota_overrides_user_quota(self): + redis_conn = MockRedis() + redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42) + redis_conn.hset('rails:orgs:acme', 'mapzen_routing_quota', 31415) + redis_config = ServicesRedisConfig(redis_conn).build('my_username', 'acme') + assert 'mapzen_routing_quota' in redis_config + assert int(redis_config['mapzen_routing_quota']) == 31415 diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py new file mode 100644 index 0000000..c56756c --- /dev/null +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -0,0 +1,85 @@ +from unittest import TestCase +from mockredis import MockRedis +from ..test_helper import build_plpy_mock +from cartodb_services.metrics.quota import QuotaChecker +from cartodb_services.metrics import RoutingConfig +from datetime import datetime + + +class RoutingConfigMock(object): + + def __init__(self, **kwargs): + self.__dict__ = kwargs + + +class TestQuotaChecker(TestCase): + + def setUp(self): + self.username = 'my_test_user' + self.period_end_date = datetime.today() + self.service_type = 'routing_mapzen' + self.redis_key = 'user:{0}:{1}:success_responses:{2}{3}'.format( + self.username, + self.service_type, + self.period_end_date.year, + self.period_end_date.month + ) + + def test_routing_quota_check_passes_when_enough_quota(self): + user_service_config = RoutingConfigMock( + username = self.username, + organization = None, + service_type = self.service_type, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = False + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 999) + assert QuotaChecker(user_service_config, redis_conn).check() == True + + def test_routing_quota_check_fails_when_quota_exhausted(self): + user_service_config = RoutingConfigMock( + username = self.username, + organization = None, + service_type = self.service_type, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = False + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001) + checker = QuotaChecker(user_service_config, redis_conn) + assert checker.check() == False + + def test_routing_quota_check_fails_right_in_the_limit(self): + """ + I have 1000 credits and I just spent 1000 today. I should not pass + the check to perform the 1001th routing operation. + """ + user_service_config = RoutingConfigMock( + username = self.username, + organization = None, + service_type = self.service_type, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = False + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1000) + checker = QuotaChecker(user_service_config, redis_conn) + assert checker.check() == False + + def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self): + user_service_config = RoutingConfigMock( + username = self.username, + organization = None, + service_type = self.service_type, + monthly_quota = 1000, + period_end_date = datetime.today(), + soft_limit = True + ) + redis_conn = MockRedis() + redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001) + checker = QuotaChecker(user_service_config, redis_conn) + assert checker.check() == True diff --git a/server/lib/python/cartodb_services/test/metrics/test_user.py b/server/lib/python/cartodb_services/test/metrics/test_user.py new file mode 100644 index 0000000..88b8e65 --- /dev/null +++ b/server/lib/python/cartodb_services/test/metrics/test_user.py @@ -0,0 +1,30 @@ +from unittest import TestCase +from cartodb_services.metrics import UserMetricsService +import datetime +from mockredis import MockRedis + +class UserGeocoderConfig(object): + + def __init__(self, **kwargs): + self.__dict__ = kwargs + + +class TestUserMetricsService(TestCase): + + def setUp(self): + user_geocoder_config = UserGeocoderConfig( + username = 'my_test_user', + organization = None, + period_end_date = datetime.date.today() + ) + redis_conn = MockRedis() + self.user_metrics_service = UserMetricsService(user_geocoder_config, redis_conn) + + + def test_routing_used_quota_zero_when_no_usage(self): + assert self.user_metrics_service.used_quota(UserMetricsService.SERVICE_MAPZEN_ROUTING, datetime.date.today()) == 0 + + def test_routing_used_quota_counts_usages(self): + self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'success_responses') + self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'empty_responses') + assert self.user_metrics_service.used_quota('routing_mapzen', datetime.date.today()) == 2