diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 521eb9f..fd85d6d 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -72,34 +72,20 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager from cartodb_services.here import HereMapsGeocoder - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - from cartodb_services.tools import RateLimiter - from cartodb_services.refactor.config.rate_limits import RateLimitsConfig - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + # from cartodb_services.metrics import QuotaService + # from cartodb_services.tools import Logger,LoggerConfig + # from cartodb_services.tools import RateLimiter + # from cartodb_services.refactor.config.rate_limits import RateLimitsConfig plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - rate_limits_config = RateLimitsConfig('geocoder', - username, - user_geocoder_config.rate_limit.get('limit'), - user_geocoder_config.rate_limit.get('period')) - rate_limiter = RateLimiter(rate_limits_config, redis_conn) - if not rate_limiter.check(): - raise Exception('Rate limit exceeded') - - # -- Check the quota - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + service_manager.check() try: - geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger) + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, logger) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_service_use() @@ -107,15 +93,15 @@ RETURNS Geometry AS $$ point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return None except BaseException as e: import sys - quota_service.increment_failed_service_use() + service_manager.quota_service.increment_failed_service_use() logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to geocode street point using here maps') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) @@ -154,64 +140,48 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - import cartodb_services - cartodb_services.init(plpy, GD) + from cartodb_services.tools import ServiceManager from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger - from cartodb_services.tools import RateLimiter - from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder - from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder - from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory - from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory - from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory - from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory - from cartodb_services.refactor.config.rate_limits import RateLimitsConfigBuilder + # from cartodb_services.metrics import QuotaService + # from cartodb_services.tools import Logger + # from cartodb_services.tools import RateLimiter + # from cartodb_services.refactor.tools.logger import LoggerConfigBuilder + # from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder + # from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory + # from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory + # from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + # from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + # from cartodb_services.refactor.config.rate_limits import RateLimitsConfigBuilder - server_config_backend = ServerConfigBackendFactory().get() - environment = ServerEnvironmentBuilder(server_config_backend).get() - user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() - org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + import cartodb_services + cartodb_services.init(plpy, GD) - logger_config = LoggerConfigBuilder(environment, server_config_backend).get() - logger = Logger(logger_config) - - mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() - rate_limit_config = RateLimiterConfigBuilder(server_config_backend, user_config_backend, org_config_backend, service='geocoder' user=username, org=orgname).get() - - redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() - - rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection) - if not rate_limiter.check(): - raise Exception('Rate limit exceeded') - - quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + service_manager.check() try: - geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, logger) country_iso3 = None if country: - country_iso3 = country_to_iso3(country) + untry_iso3 = country_to_iso3(country) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state_province=state_province, country=country_iso3, search_type='address') if coordinates: - quota_service.increment_success_service_use() + service_manager.quota_service.quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return None except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to geocode street point using mapzen') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index a3ff3bb..d4328a1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -523,15 +523,15 @@ class ServicesDBConfig: self._data_observatory_connection_str = do_conf['connection']['production'] def _get_rate_limits_config(self): - self._rate_limits = self._get_conf('rate_limits', default={}) + self._rate_limits = json.loads(self._get_conf('rate_limits', default='{}')) - def _get_conf(self, key, default='raise'): + def _get_conf(self, key, default=KeyError): try: sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) conf = self._db_conn.execute(sql, 1) return conf[0]['conf'] except Exception as e: - if (default != 'raise'): + if (default != KeyError): return default raise ConfigException("Error trying to get config for {0}: {1}".format(key, e)) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py index 9a2b6de..80d92a0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py @@ -50,14 +50,15 @@ class RateLimitsConfigBuilder(object): def get(self): # Order of precedence is user_conf, org_conf, server_conf - rate_limit_key = "{0}_rate_limit".format(service) + rate_limit_key = "{0}_rate_limit".format(self._service) rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None) if (rate_limit_json): rate_limit = rate_limit_json and json.loads(rate_limit_json) else: - rate_limit = self._server_conf.get('rate_limits', {}).get(service, {}) + rate_limit = self._server_conf.get('rate_limits', {}).get(self._service, {}) return RateLimitsConfig(self._service, self._username, - self._limit, - self._period) + rate_limit.get('limit', None), + rate_limit.get('period', None)) + diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py index 46dcd42..cb69b6f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py @@ -6,6 +6,6 @@ class ConfigBackendInterface(object): __metaclass__ = abc.ABCMeta @abc.abstractmethod - def get(self, key): + def get(self, key, default=None): """Return a value based on the key supplied from some storage""" pass diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py index 0661626..148b970 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py @@ -5,8 +5,8 @@ class InMemoryConfigStorage(ConfigBackendInterface): def __init__(self, config_hash={}): self._config_hash = config_hash - def get(self, key): + def get(self, key, default=None): try: return self._config_hash[key] except KeyError: - return None + return default diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py index b9c11e6..c080e13 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py @@ -2,5 +2,5 @@ from ..core.interfaces import ConfigBackendInterface class NullConfigStorage(ConfigBackendInterface): - def get(self, key): - return None + def get(self, key, default=None): + return default diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index cdb2e57..1e1f33c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -9,11 +9,13 @@ class RedisConfigStorage(ConfigBackendInterface): self._config_key = config_key self._data = None - def get(self, key): + def get(self, key, default=KeyError): if not self._data: self._data = self._connection.hgetall(self._config_key) - return self._data[key] - + if (default == KeyError): + return self._data[key] + else: + return self._data.get(key, default) class RedisUserConfigStorageBuilder(object): def __init__(self, redis_connection, username): diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 50137ca..1d3603c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -4,11 +4,18 @@ from ..core.interfaces import ConfigBackendInterface class InDbServerConfigStorage(ConfigBackendInterface): - def get(self, key): + def get(self, key, default=None): sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key) rows = cartodb_services.plpy.execute(sql, 1) - json_output = rows[0]['conf'] - if json_output: + json_output = None + try: + json_output = rows[0]['conf'] + except IndexError: + pass + if (json_output): return json.loads(json_output) else: - return None + if (default == KeyError): + raise KeyError + else: + return default diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index 7bea3e2..0658841 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -2,3 +2,6 @@ from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine from log import Logger, LoggerConfig +from rate_limiter import RateLimiter +from exceptions import RateLimitExceeded +from service_manager import ServiceManager, LegacyServiceManager diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/tools/exceptions.py new file mode 100644 index 0000000..f8b79c1 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/exceptions.py @@ -0,0 +1,3 @@ +class RateLimitExceeded(Exception): + def __str__(self): + return repr('Rate limit exceeded') diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py b/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py index 16cf3f2..3410519 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py @@ -4,13 +4,14 @@ class RateLimiter: def __init__(self, rate_limits_config, redis_connection): self._config = rate_limits_config + self._limiter = None if (self._config.is_limited()): - self._limiter = Limiter(rdis_connection, + self._limiter = Limiter(redis_connection, action=self._config.service, limit=self._config.limit, period=self._config.period) - def check(): + def check(self): ok = True if (self._limiter): ok = self._limiter.checked_insert(self._config.username) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py new file mode 100644 index 0000000..03c8fb4 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py @@ -0,0 +1,70 @@ +from exceptions import RateLimitExceeded + +class ServiceManagerBase: + def check(self): + if not self.rate_limiter.check(): + raise RateLimitExceeded() + if not self.quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + @property + def config(self): + return self.config + + @property + def quota_service(self): + return self.quota_service + + @property + def logger(self): + return self.logger + +from cartodb_services.metrics import QuotaService +from cartodb_services.tools import Logger,LoggerConfig +from cartodb_services.tools import RateLimiter +from cartodb_services.refactor.config.rate_limits import RateLimitsConfig + +class LegacyServiceManager(ServiceManagerBase): + + def __init__(self, service, username, orgname, gd): + redis_conn = gd["redis_connection_{0}".format(username)]['redis_metrics_connection'] + self.config = gd["user_{0}_config_{1}".format(service, username)] + logger_config = gd["logger_config"] + self.logger = Logger(logger_config) + + rate_limits_config = RateLimitsConfig(service, + username, + self.config.rate_limit.get('limit'), + self.config.rate_limit.get('period')) + self.rate_limiter = RateLimiter(rate_limits_config, redis_conn) + self.quota_service = QuotaService(self.config, redis_conn) + +from cartodb_services.metrics import QuotaService +from cartodb_services.tools import Logger +from cartodb_services.tools import RateLimiter +from cartodb_services.refactor.tools.logger import LoggerConfigBuilder +from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder +from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory +from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory +from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory +from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory +from cartodb_services.refactor.config.rate_limits import RateLimitsConfigBuilder + +class ServiceManager(ServiceManagerBase): + + def __init__(self, service, config_builder, username, orgname): + server_config_backend = ServerConfigBackendFactory().get() + environment = ServerEnvironmentBuilder(server_config_backend).get() + user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() + org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + + logger_config = LoggerConfigBuilder(environment, server_config_backend).get() + self.logger = Logger(logger_config) + + self.config = config_builder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + rate_limit_config = RateLimitsConfigBuilder(server_config_backend, user_config_backend, org_config_backend, service=service, user=username, org=orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() + + self.rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection) + self.quota_service = QuotaService(self.config, redis_metrics_connection) diff --git a/server/lib/python/cartodb_services/test/test_servicemanager.py b/server/lib/python/cartodb_services/test/test_servicemanager.py new file mode 100644 index 0000000..b7874ab --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_servicemanager.py @@ -0,0 +1,218 @@ +from test_helper import * +from unittest import TestCase +from mock import Mock, MagicMock, patch +from nose.tools import assert_raises, assert_not_equal, assert_equal + +from datetime import datetime, date +from cartodb_services.tools import ServiceManager, LegacyServiceManager +from mockredis import MockRedis +import cartodb_services +from cartodb_services.metrics import GeocoderConfig +from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder +from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder +from cartodb_services.tools import RateLimitExceeded + +LUA_AVAILABLE_FOR_MOCKREDIS = False + +class MockRedisWithVersionInfo(MockRedis): + def info(self): + return {'redis_version': '3.0.2'} + +class TestServiceManager(TestCase): + + def setUp(self): + plpy_mock_config() + cartodb_services.init(plpy_mock, _GD={}) + self.username = 'test_user' + self.orgname = 'test_org' + self.redis_conn = MockRedisWithVersionInfo() + build_redis_user_config(self.redis_conn, self.username, 'geocoding') + build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen') + self.environment = 'production' + plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}]) + + def check_rate_limit(self, service_manager, n, active=True): + if LUA_AVAILABLE_FOR_MOCKREDIS: + for _ in xrange(n): + service_manager.check() + if active: + with assert_raises(RateLimitExceeded): + service_manager.check() + else: + service_manager.check() + else: + # rratelimit doesn't work with MockRedis because it needs Lua support + # so, we'll simply perform some sanity check on the configuration of the rate limiter + if active: + assert_equal(service_manager.rate_limiter._config.is_limited(), True) + assert_equal(service_manager.rate_limiter._config.limit, n) + else: + assert not service_manager.rate_limiter._config.is_limited() + + def test_legacy_service_manager(self): + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + service_manager.check() + assert_equal(service_manager.config.service_type, 'geocoder_mapzen') + + def test_service_manager(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + service_manager.check() + assert_equal(service_manager.config.service_type, 'geocoder_mapzen') + + def test_no_rate_limit_by_default(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3, False) + + def test_no_legacy_rate_limit_by_default(self): + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3, False) + + def test_legacy_server_rate_limit(self): + rate_limits = '{"geocoder":{"limit":"3","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}]) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + + def test_server_rate_limit(self): + rate_limits = '{"geocoder":{"limit":"3","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}]) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + + def test_user_rate_limit(self): + user_redis_name = "rails:users:{0}".format(self.username) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_legacy_user_rate_limit(self): + user_redis_name = "rails:users:{0}".format(self.username) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_org_rate_limit(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + + def test_legacy_org_rate_limit(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + + def test_user_rate_limit_precedence_over_org(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"1000","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + user_redis_name = "rails:users:{0}".format(self.username) + user_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_org_rate_limit_precedence_over_server(self): + server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}]) + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + + def test_legacy_user_rate_limit_precedence_over_org(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"1000","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + user_redis_name = "rails:users:{0}".format(self.username) + user_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_legacy_org_rate_limit_precedence_over_server(self): + server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}]) + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])