6c5ca97468
ServiceManager class has been introduced to handle service configuration at SQL level (with a LegacyServiceManager alternative for non-refactored services). These new classes take the responsibility of rate limits and quota checking. Tests have been added for ServiceManager and rate limits, but currently they check only the limits configuration since Lua support would be needed to use rratelimit with MockRedis.
219 lines
13 KiB
Python
219 lines
13 KiB
Python
from test_helper import *
|
|
from unittest import TestCase
|
|
from mock import Mock, MagicMock, patch
|
|
from nose.tools import assert_raises, assert_not_equal, assert_equal
|
|
|
|
from datetime import datetime, date
|
|
from cartodb_services.tools import ServiceManager, LegacyServiceManager
|
|
from mockredis import MockRedis
|
|
import cartodb_services
|
|
from cartodb_services.metrics import GeocoderConfig
|
|
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
|
|
from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
|
|
from cartodb_services.tools import RateLimitExceeded
|
|
|
|
LUA_AVAILABLE_FOR_MOCKREDIS = False
|
|
|
|
class MockRedisWithVersionInfo(MockRedis):
|
|
def info(self):
|
|
return {'redis_version': '3.0.2'}
|
|
|
|
class TestServiceManager(TestCase):
|
|
|
|
def setUp(self):
|
|
plpy_mock_config()
|
|
cartodb_services.init(plpy_mock, _GD={})
|
|
self.username = 'test_user'
|
|
self.orgname = 'test_org'
|
|
self.redis_conn = MockRedisWithVersionInfo()
|
|
build_redis_user_config(self.redis_conn, self.username, 'geocoding')
|
|
build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
|
|
self.environment = 'production'
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
|
|
|
|
def check_rate_limit(self, service_manager, n, active=True):
|
|
if LUA_AVAILABLE_FOR_MOCKREDIS:
|
|
for _ in xrange(n):
|
|
service_manager.check()
|
|
if active:
|
|
with assert_raises(RateLimitExceeded):
|
|
service_manager.check()
|
|
else:
|
|
service_manager.check()
|
|
else:
|
|
# rratelimit doesn't work with MockRedis because it needs Lua support
|
|
# so, we'll simply perform some sanity check on the configuration of the rate limiter
|
|
if active:
|
|
assert_equal(service_manager.rate_limiter._config.is_limited(), True)
|
|
assert_equal(service_manager.rate_limiter._config.limit, n)
|
|
else:
|
|
assert not service_manager.rate_limiter._config.is_limited()
|
|
|
|
def test_legacy_service_manager(self):
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
service_manager.check()
|
|
assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
|
|
|
|
def test_service_manager(self):
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
service_manager.check()
|
|
assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
|
|
|
|
def test_no_rate_limit_by_default(self):
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
self.check_rate_limit(service_manager, 3, False)
|
|
|
|
def test_no_legacy_rate_limit_by_default(self):
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
self.check_rate_limit(service_manager, 3, False)
|
|
|
|
def test_legacy_server_rate_limit(self):
|
|
rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
self.check_rate_limit(service_manager, 3)
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
|
|
|
def test_server_rate_limit(self):
|
|
rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
self.check_rate_limit(service_manager, 3)
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
|
|
|
def test_user_rate_limit(self):
|
|
user_redis_name = "rails:users:{0}".format(self.username)
|
|
rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
|
|
|
def test_legacy_user_rate_limit(self):
|
|
user_redis_name = "rails:users:{0}".format(self.username)
|
|
rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
|
|
|
def test_org_rate_limit(self):
|
|
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
|
rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
|
|
|
def test_legacy_org_rate_limit(self):
|
|
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
|
rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
|
|
|
def test_user_rate_limit_precedence_over_org(self):
|
|
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
|
org_rate_limits = '{"limit":"1000","period":3600}'
|
|
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
|
user_redis_name = "rails:users:{0}".format(self.username)
|
|
user_rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
|
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
|
|
|
def test_org_rate_limit_precedence_over_server(self):
|
|
server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
|
|
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
|
org_rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
|
with patch.object(RedisConnectionBuilder,'get') as get_fn:
|
|
get_fn.return_value = self.redis_conn
|
|
service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|
|
|
|
def test_legacy_user_rate_limit_precedence_over_org(self):
|
|
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
|
org_rate_limits = '{"limit":"1000","period":3600}'
|
|
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
|
user_redis_name = "rails:users:{0}".format(self.username)
|
|
user_rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
|
self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
|
|
|
|
def test_legacy_org_rate_limit_precedence_over_server(self):
|
|
server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
|
|
org_redis_name = "rails:orgs:{0}".format(self.orgname)
|
|
org_rate_limits = '{"limit":"3","period":3600}'
|
|
self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
|
|
config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen')
|
|
config_cache = {
|
|
'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
|
|
'user_geocoder_config_test_user' : config,
|
|
'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
|
|
}
|
|
service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
|
|
self.check_rate_limit(service_manager, 3)
|
|
self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
|
|
plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
|