cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Server side new data observatory functions (v0.8.0)

Browse Source
This commit is contained in:
Mario de Frutos
2016-05-11 11:50:21 +02:00
parent 64d2afb536
commit c7c9e6e284
20 changed files with 2216 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@@ -1,3 +1,3 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService

76
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -33,32 +33,11 @@ class ServiceConfig(object):
def organization(self):
return self._orgname
class ObservatorySnapshotConfig(ServiceConfig):
SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
QUOTA_KEY = 'obs_snapshot_quota'
PERIOD_END_DATE = 'period_end_date'
class DataObservatoryConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
# Mixed config between db and redis. If we don't update all the users
# in redis, we could use the db value as default
if self.QUOTA_KEY in self._redis_config:
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
else:
self._monthly_quota = float(self._db_config.data_observatory_monthly_quota)
self._connection_str = self._db_config.data_observatory_connection_str
@property
def service_type(self):
return 'obs_snapshot'
@property
def monthly_quota(self):
@@ -76,6 +55,52 @@ class ObservatorySnapshotConfig(ServiceConfig):
def connection_str(self):
return self._connection_str
class ObservatorySnapshotConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
QUOTA_KEY = 'obs_snapshot_quota'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
self._monthly_quota = 0
if self.QUOTA_KEY in self._redis_config:
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
self._connection_str = self._db_config.data_observatory_connection_str
@property
def service_type(self):
return 'obs_snapshot'
class ObservatoryConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_general_limit'
QUOTA_KEY = 'obs_general_quota'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(ObservatoryConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
self._monthly_quota = 0
if self.QUOTA_KEY in self._redis_config:
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
self._connection_str = self._db_config.data_observatory_connection_str
@property
def service_type(self):
return 'obs_general'
class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
@@ -370,7 +395,6 @@ class ServicesDBConfig:
raise ConfigException('Data Observatory configuration missing')
else:
do_conf = json.loads(do_conf_json)
self._data_observatory_monthly_quota = do_conf['monthly_quota']
if self._orgname and self._orgname in do_conf['connection']['whitelist']:
self._data_observatory_connection_str = do_conf['connection']['staging']
elif self._username in do_conf['connection']['whitelist']:
@@ -434,10 +458,6 @@ class ServicesDBConfig:
def geocoder_log_path(self):
return self._geocoder_log_path
@property
def data_observatory_monthly_quota(self):
return self._data_observatory_monthly_quota
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str

9
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -73,9 +73,9 @@ class QuotaChecker:
elif re.match('routing_mapzen',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('obs_snapshot',
elif re.match('obs_*',
self._user_service_config.service_type) is not None:
return self.__check_obs_snapshot_quota()
return self.__check_data_observatory_quota()
else:
return False
@@ -118,13 +118,14 @@ class QuotaChecker:
else:
return False
def __check_obs_snapshot_quota(self):
def __check_data_observatory_quota(self):
user_quota = self._user_service_config.monthly_quota
soft_limit = self._user_service_config.soft_limit
today = date.today()
service_type = self._user_service_config.service_type
current_used = self._user_service.used_quota(service_type, today)
if (user_quota > 0 and current_used <= user_quota):
if soft_limit or (user_quota > 0 and current_used <= user_quota):
return True
else:
return False

2
server/lib/python/cartodb_services/setup.py
View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.5.3',
version='0.6.0',
description='CartoDB Services API Python Library',

2
server/lib/python/cartodb_services/test/test_config.py
View File

@@ -49,7 +49,7 @@ class TestConfig(TestCase):
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
'test_user')
assert do_config.monthly_quota == 100000
assert do_config.monthly_quota == 0
assert do_config.soft_limit is False
assert do_config.period_end_date.date() == yesterday.date()

2
server/lib/python/cartodb_services/test/test_helper.py
View File

@@ -61,4 +61,4 @@ def _plpy_execute_side_effect(*args, **kwargs):
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'}]
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]