cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use redis or db data observatory quota value

Browse Source
This commit is contained in:
Mario de Frutos
2016-04-22 16:43:28 +02:00
parent 976697c81e
commit a9c420ba7b
8 changed files with 108 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@@ -1,3 +1,3 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, DataObservatoryConfig
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig
from quota import QuotaService
from user import UserMetricsService

60
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -32,19 +32,31 @@ class ServiceConfig(object):
def organization(self):
return self._orgname
class DataObservatoryConfig(ServiceConfig):
class ObservatorySnapshotConfig(ServiceConfig):
SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
QUOTA_KEY = 'obs_snapshot_quota'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._monthly_quota = self._db_config.data_observatory_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
# Mixed config between db and redis. If we don't update all the users
# in redis, we could use the db value as default
if self.QUOTA_KEY in self._redis_config:
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
else:
self._monthly_quota = float(self._db_config.data_observatory_monthly_quota)
@property
def service_type(self):
return 'data_observatory'
return 'obs_snapshot'
@property
def monthly_quota(self):
@@ -54,6 +66,10 @@ class DataObservatoryConfig(ServiceConfig):
def period_end_date(self):
return self._period_end_date
@property
def soft_limit(self):
return self._soft_limit
class RoutingConfig(ServiceConfig):
@@ -200,28 +216,6 @@ class GeocoderConfig(ServiceConfig):
self.__parse_config(filtered_config, self._db_config)
self.__check_config(filtered_config)
def __get_user_config(self, username, orgname):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
if orgname:
self.__get_organization_config(orgname, user_config)
return user_config
def __get_organization_config(self, orgname, user_config):
org_config = self._redis_connection.hgetall(
"rails:orgs:{0}".format(orgname))
if not org_config:
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
@@ -438,24 +432,24 @@ class ServicesRedisConfig:
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
QUOTA_KEY = 'geocoding_quota'
ISOLINES_QUOTA_KEY = 'here_isolines_quota'
OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_conn):
self._redis_connection = redis_conn
def build(self, username, password):
return self.__get_user_config(username, password)
def build(self, username, orgname):
return self.__get_user_config(username, orgname)
def __get_user_config(self, username, orgname):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
if orgname:
self.__get_organization_config(orgname, user_config)
elif orgname:
self.__get_organization_config(orgname, user_config)
return user_config
return user_config
def __get_organization_config(self, orgname, user_config):
org_config = self._redis_connection.hgetall(
@@ -465,6 +459,8 @@ class ServicesRedisConfig:
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]

6
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -73,9 +73,9 @@ class QuotaChecker:
elif re.match('routing_mapzen',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('data_observatory',
elif re.match('obs_snapshot',
self._user_service_config.service_type) is not None:
return self.__check_data_observatory_quota()
return self.__check_obs_snapshot_quota()
else:
return False
@@ -118,7 +118,7 @@ class QuotaChecker:
else:
return False
def __check_data_observatory_quota(self):
def __check_obs_snapshot_quota(self):
user_quota = self._user_service_config.monthly_quota
today = date.today()
service_type = self._user_service_config.service_type

2
server/lib/python/cartodb_services/setup.py
View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.5.0',
version='0.5.1',
description='CartoDB Services API Python Library',

29
server/lib/python/cartodb_services/test/test_config.py
View File

@@ -1,5 +1,5 @@
import test_helper
from cartodb_services.metrics import GeocoderConfig, ConfigException
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
@@ -32,10 +32,29 @@ class TestConfig(TestCase):
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_return_config_for_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user',
do_quota=100, soft_do_limit=True,
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
'test_user')
assert do_config.monthly_quota == 100
assert do_config.soft_limit is True
assert do_config.period_end_date.date() == yesterday.date()
def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user',
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
'test_user')
assert do_config.monthly_quota == 100000
assert do_config.soft_limit is False
assert do_config.period_end_date.date() == yesterday.date()
def test_should_raise_exception_when_missing_parameters(self):
plpy_mock = test_helper.build_plpy_mock(empty=True)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, plpy_mock, 'test_user',
None)
assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
plpy_mock, 'test_user', None)

17
server/lib/python/cartodb_services/test/test_helper.py
View File

@@ -4,6 +4,7 @@ from mock import Mock
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
service="heremaps", isolines_quota=0,
do_quota=None, soft_do_limit=None,
end_date=datetime.today()):
user_redis_name = "rails:users:{0}".format(username)
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
@@ -11,23 +12,31 @@ def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
if do_quota:
redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota)
if soft_do_limit:
redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit',
soft_do_limit)
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
def build_redis_org_config(redis_conn, orgname, quota=100, isolines_quota=0,
def build_redis_org_config(redis_conn, orgname, quota=100, service="heremaps",
isolines_quota=0, do_quota=None,
end_date=datetime.today()):
org_redis_name = "rails:orgs:{0}".format(orgname)
redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
redis_conn.hset(org_redis_name, 'here_isolines_quota', isolines_quota)
if do_quota:
redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota)
redis_conn.hset(org_redis_name, 'period_end_date', end_date)
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
def increment_geocoder_uses(redis_conn, username, orgname=None,
date=date.today(), service='geocoder_here',
metric='success_responses', amount=20):
def increment_service_uses(redis_conn, username, orgname=None,
date=date.today(), service='geocoder_here',
metric='success_responses', amount=20):
prefix = 'org' if orgname else 'user'
entity_name = orgname if orgname else username
yearmonth = date.strftime('%Y%m')

61
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, DataObservatoryConfig
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
@@ -28,40 +28,40 @@ class TestQuotaService(TestCase):
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org',
soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
test_helper.increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
@@ -109,29 +109,34 @@ class TestQuotaService(TestCase):
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_data_observatory_quota_correctly(self):
qs = self.__build_data_observatory_quota_service('test_user')
def test_should_check_user_obs_snapshot_quota_correctly(self):
qs = self.__build_obs_snapshot_quota_service('test_user')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=100000)
assert qs.check_user_quota() is False
def test_should_check_org_data_observatory_quota_correctly(self):
qs = self.__build_data_observatory_quota_service('test_user', orgname='testorg')
def test_should_check_org_obs_snapshot_quota_correctly(self):
qs = self.__build_obs_snapshot_quota_service('test_user',
orgname='testorg')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=100000)
assert qs.check_user_quota() is False
def __prepare_quota_service(self, username, quota, service, orgname,
soft_limit, end_date):
soft_limit, do_quota, soft_do_limit, end_date):
test_helper.build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
soft_do_limit=soft_do_limit,
do_quota=do_quota,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
quota=quota, service=service,
do_quota=do_quota,
end_date=end_date)
self._plpy_mock = test_helper.build_plpy_mock()
def __build_geocoder_quota_service(self, username, quota=100,
@@ -139,27 +144,27 @@ class TestQuotaService(TestCase):
soft_limit=False,
end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, end_date)
soft_limit, 0, False, end_date)
geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock,
username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
def __build_routing_quota_service(self, username, quota=100,
service='routing_mapzen', orgname=None,
soft_limit=False,
end_date=datetime.today()):
def __build_routing_quota_service(self, username, service='routing_mapzen',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, end_date)
soft_limit, 0, False, end_date)
routing_config = RoutingConfig(self.redis_conn, self._plpy_mock,
username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn)
def __build_data_observatory_quota_service(self, username, quota=100,
service='data_observatory', orgname=None,
def __build_obs_snapshot_quota_service(self, username, quota=100,
service='obs_snapshot',
orgname=None,
soft_limit=False,
end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, end_date)
do_config = DataObservatoryConfig(self.redis_conn, self._plpy_mock,
self.__prepare_quota_service(username, 0, service, orgname, False,
quota, soft_limit, end_date)
do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock,
username, orgname)
return QuotaService(do_config, redis_connection=self.redis_conn)

10
server/lib/python/cartodb_services/test/test_user_service.py
View File

@@ -18,15 +18,15 @@ class TestUserService(TestCase):
def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=400)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_user_not_amount_in_used_quota_for_month_should_be_0(self):