cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove DO Python code and SQL tests

Browse Source
This commit is contained in:
Jesús Arroyo Torrens
2021-03-05 10:44:09 +01:00
parent 610398f531
commit edbefad6b0
29 changed files with 11 additions and 2252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@@ -1,4 +1,4 @@
from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException
from cartodb_services.metrics.quota import QuotaService
from cartodb_services.metrics.user import UserMetricsService
from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

72
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -57,57 +57,6 @@ class ServiceConfig(object):
return None
class DataObservatoryConfig(ServiceConfig):
METRICS_LOG_KEY = 'do_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
username, orgname)
@property
def monthly_quota(self):
return self._monthly_quota
@property
def period_end_date(self):
return self._period_end_date
@property
def soft_limit(self):
return self._soft_limit
@property
def connection_str(self):
return self._connection_str
@property
def provider(self):
return 'data observatory'
class ObservatoryConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_general_limit'
QUOTA_KEY = 'obs_general_quota'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(ObservatoryConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
self._connection_str = self._db_config.data_observatory_connection_str
@property
def service_type(self):
return 'obs_general'
class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
@@ -624,7 +573,6 @@ class ServicesDBConfig:
self._get_mapbox_config()
self._get_tomtom_config()
self._get_geocodio_config()
self._get_data_observatory_config()
def _get_server_config(self):
server_config_json = self._get_conf('server_conf')
@@ -716,19 +664,6 @@ class ServicesDBConfig:
self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
def _get_data_observatory_config(self):
do_conf_json = self._get_conf('data_observatory_conf')
if not do_conf_json:
raise ConfigException('Data Observatory configuration missing')
else:
do_conf = json.loads(do_conf_json)
if self._orgname and self._orgname in do_conf['connection']['whitelist']:
self._data_observatory_connection_str = do_conf['connection']['staging']
elif self._username in do_conf['connection']['whitelist']:
self._data_observatory_connection_str = do_conf['connection']['staging']
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_conf(self, key):
try:
sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
@@ -897,10 +832,6 @@ class ServicesDBConfig:
def geocodio_geocoder_service_params(self):
return self._geocodio_geocoder_service_params
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str
@property
def logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
@@ -917,7 +848,6 @@ class ServicesRedisConfig:
QUOTA_KEY = 'geocoding_quota'
ISOLINES_QUOTA_KEY = 'here_isolines_quota'
ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
PERIOD_END_DATE = 'period_end_date'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
ISOLINES_PROVIDER_KEY = 'isolines_provider'
@@ -960,8 +890,6 @@ class ServicesRedisConfig:
user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
if self.ROUTING_QUOTA_KEY in org_config:
user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
if self.OBS_GENERAL_QUOTA_KEY in org_config:
user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
if self.PERIOD_END_DATE in org_config:
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
if self.GOOGLE_GEOCODER_CLIENT_ID in org_config:

2
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@@ -127,8 +127,6 @@ class MetricsServiceLoggerFactory:
return MetricsGenericLogger(service_config, logger)
elif re.search('_isolines$', service_config.service_type):
return MetricsIsolinesLogger(service_config, logger)
elif re.search('^obs_*', service_config.service_type):
return MetricsGenericLogger(service_config, logger)
else:
return None

18
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -78,9 +78,6 @@ class QuotaChecker:
elif re.match('^routing_',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('^obs_',
self._user_service_config.service_type) is not None:
return self.__check_data_observatory_quota()
else:
return False
@@ -123,18 +120,3 @@ class QuotaChecker:
return True
else:
return False
def __check_data_observatory_quota(self):
user_quota = self._user_service_config.monthly_quota
soft_limit = self._user_service_config.soft_limit
today = date.today()
service_type = self._user_service_config.service_type
current_used = self._user_service.used_quota(service_type, today)
# Quick workaround so we don't take into account numer of credits
# spent for users that have defined the quota.
# See https://github.com/CartoDB/bigmetadata/issues/215
if soft_limit or (user_quota > 0):
return True
else:
return False

16
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@@ -26,7 +26,6 @@ class UserMetricsService:
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
SERVICE_TOMTOM_ROUTING = 'routing_tomtom'
SERVICE_OBSERVATORY = 'obs_general'
DAY_OF_MONTH_ZERO_PADDED = '%d'
def __init__(self, user_geocoder_config, redis_connection):
@@ -45,8 +44,6 @@ class UserMetricsService:
self.SERVICE_MAPBOX_ROUTING,
self.SERVICE_TOMTOM_ROUTING]:
return self.__used_routing_quota(service_type, date)
elif service_type == self.SERVICE_OBSERVATORY:
return self.__used_observatory_quota(service_type, date)
else:
return self.__used_geocoding_quota(service_type, date)
@@ -97,19 +94,6 @@ class UserMetricsService:
return current_use
def __used_observatory_quota(self, service_type, date):
date_from, date_to = self.__current_billing_cycle()
current_use = 0
success_responses = self.get_metrics(service_type,
'success_responses', date_from,
date_to)
empty_responses = self.get_metrics(service_type,
'empty_responses', date_from,
date_to)
current_use += (success_responses + empty_responses)
return current_use
def increment_service_use(self, service_type, metric, date=date.today(),
amount=1):
""" Increment the services uses in monthly and daily basis"""

2
server/lib/python/cartodb_services/setup.py
View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.23.5',
version='0.23.6',
description='CartoDB Services API Python Library',

96
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@@ -344,102 +344,6 @@ class TestRoutingConfig(TestCase):
assert config.soft_limit == True
class TestDataObservatoryUserConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
plpy_mock_config()
def test_should_return_config_for_obs_config(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=100, end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 100
assert do_config.soft_limit is False
assert do_config.period_end_date.date() == yesterday.date()
def test_should_return_0_if_quota_is_0_in_redis(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=0, end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 0
def test_should_return_0_if_quota_is_empty_in_redis(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota='', end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 0
def test_should_return_true_if_soft_limit_is_true_in_redis(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=0, soft_limit=True, end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.soft_limit is True
def test_should_return_true_if_soft_limit_is_empty_string_in_redis(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=0, soft_limit='', end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.soft_limit is False
class TestDataObservatoryOrgConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
plpy_mock_config()
def test_should_return_organization_config(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=100, end_date=yesterday)
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
quota=200, end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org')
assert do_config.monthly_quota == 200
assert do_config.period_end_date.date() == yesterday.date()
def test_should_return_quota_0_for_0_in_org_quota_config(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=100)
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
quota=0, end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org')
assert do_config.monthly_quota == 0
def test_should_return_quota_0_for_empty_in_org_quota_config(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=100)
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
quota='', end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org')
assert do_config.monthly_quota == 0
def test_should_return_user_config_when_org_quota_is_not_defined(self):
yesterday = datetime.today() - timedelta(days=1)
build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
quota=100)
build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
quota=None, end_date=yesterday)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org')
assert do_config.monthly_quota == 100
class TestServicesRedisConfig(TestCase):
def test_it_picks_mapzen_routing_quota_from_redis(self):
redis_conn = MockRedis()

7
server/lib/python/cartodb_services/test/test_helper.py
View File

@@ -27,9 +27,6 @@ def build_redis_user_config(redis_conn, username, service, quota=100,
redis_conn.hset(user_redis_name, 'routing_provider', provider)
redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota))
redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower())
elif service is 'data_observatory':
redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota))
redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower())
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
@@ -53,9 +50,6 @@ def build_redis_org_config(redis_conn, orgname, service, quota=100,
redis_conn.hset(org_redis_name, 'routing_provider', provider)
if quota is not None:
redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota))
elif service is 'data_observatory':
if quota is not None:
redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota))
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
@@ -80,6 +74,5 @@ def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}])

22
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@@ -178,17 +178,6 @@ class TestQuotaService(TestCase):
qs.increment_isolines_service_use(amount=1500000)
assert qs.check_user_quota() is False
# Quick workaround so we don't take into account numer of credits
# spent for users that have defined the quota.
# See https://github.com/CartoDB/bigmetadata/issues/215
def test_should_check_org_obs_quota_correctly(self):
qs = self.__build_obs_quota_service('test_user',
orgname='testorg')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=100000)
assert qs.check_user_quota() is True
def __prepare_quota_service(self, username, service, quota, provider,
orgname, soft_limit, end_date):
build_redis_user_config(self.redis_conn, username, service,
@@ -226,14 +215,3 @@ class TestQuotaService(TestCase):
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(isolines_config, redis_connection=self.redis_conn)
def __build_obs_quota_service(self, username, quota=100,
provider='obs_general',
orgname=None,
soft_limit=False,
end_date=datetime.today()):
self.__prepare_quota_service(username, 'data_observatory', quota,
None, orgname, soft_limit, end_date)
do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(do_config, redis_connection=self.redis_conn)