Merge pull request #355 from CartoDB/346-user-rate-limits

Service rate limits

server/lib/python/cartodb_services/cartodb_services/config/__init__.py

@@ -0,0 +1,3 @@
+from service_configuration import ServiceConfiguration
+from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+from legacy_rate_limits import RateLimitsConfigLegacyBuilder

server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py

@@ -0,0 +1,46 @@
+import json
+from rate_limits import RateLimitsConfig
+
+class RateLimitsConfigLegacyBuilder(object):
+    """
+    Build a RateLimitsConfig object using the *legacy* configuration classes
+    """
+
+    def __init__(self, redis_connection, db_conn, service, username, orgname):
+        self._service = service
+        self._username = username
+        self._orgname = orgname
+        self._redis_connection = redis_connection
+        self._db_conn = db_conn
+
+    def get(self):
+        rate_limit = self.__get_rate_limit()
+        return RateLimitsConfig(self._service,
+                                self._username,
+                                rate_limit.get('limit', None),
+                                rate_limit.get('period', None))
+
+    def __get_rate_limit(self):
+        rate_limit = {}
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+        user_key = "rails:users:{0}".format(self._username)
+        rate_limit_json = self.__get_redis_config(user_key, rate_limit_key)
+        if not rate_limit_json and self._orgname:
+            org_key = "rails:orgs:{0}".format(self._orgname)
+            rate_limit_json = self.__get_redis_config(org_key, rate_limit_key)
+        if rate_limit_json:
+            rate_limit = json.loads(rate_limit_json)
+        else:
+            conf_key = 'rate_limits'
+            sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(conf_key)
+            try:
+                conf = self._db_conn.execute(sql, 1)[0]['conf']
+            except Exception:
+                conf = None
+            if conf:
+                rate_limit = json.loads(conf).get(self._service)
+        return rate_limit or {}
+
+    def __get_redis_config(self, basekey, param):
+        config = self._redis_connection.hgetall(basekey)
+        return config and config.get(param)

server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py

@@ -0,0 +1,113 @@
+import json
+
+from service_configuration import ServiceConfiguration
+
+class RateLimitsConfig(object):
+    """
+    Value object that represents the configuration needed to rate-limit services
+    """
+
+    def __init__(self,
+                 service,
+                 username,
+                 limit,
+                 period):
+        self._service = service
+        self._username = username
+        self._limit = limit and int(limit)
+        self._period = period and int(period)
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    # service this limit applies to
+    @property
+    def service(self):
+        return self._service
+
+    # user this limit applies to
+    @property
+    def username(self):
+        return self._username
+
+    # rate period in seconds
+    @property
+    def period(self):
+        return self._period
+
+    # rate limit in seconds
+    @property
+    def limit(self):
+        return self._limit
+
+    def is_limited(self):
+        return self._limit and self._limit > 0 and self._period and self._period > 0
+
+class RateLimitsConfigBuilder(object):
+    """
+    Build a rate limits configuration obtaining the parameters
+    from the user/org/server configuration.
+    """
+
+    def __init__(self, server_conf, user_conf, org_conf, service, username, orgname):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._service = service
+        self._username = username
+        self._orgname = orgname
+
+    def get(self):
+        # Order of precedence is user_conf, org_conf, server_conf
+
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+
+        rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None)
+        if (rate_limit_json):
+            rate_limit = rate_limit_json and json.loads(rate_limit_json)
+        else:
+            rate_limit = self._server_conf.get('rate_limits', {}).get(self._service, {})
+
+        return RateLimitsConfig(self._service,
+                                self._username,
+                                rate_limit.get('limit', None),
+                                rate_limit.get('period', None))
+
+
+class RateLimitsConfigSetter(object):
+
+    def __init__(self, service, username, orgname=None):
+        self._service = service
+        self._service_config = ServiceConfiguration(service, username, orgname)
+
+    def set_user_rate_limits(self, rate_limits_config):
+        # Note we allow copying a config from another user/service, so we
+        # ignore rate_limits:config.service and rate_limits:config.username
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+        if rate_limits_config.is_limited():
+            rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
+            rate_limit_json = json.dumps(rate_limit)
+            self._service_config.user.set(rate_limit_key, rate_limit_json)
+        else:
+            self._service_config.user.remove(rate_limit_key)
+
+    def set_org_rate_limits(self, rate_limits_config):
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+        if rate_limits_config.is_limited():
+            rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
+            rate_limit_json = json.dumps(rate_limit)
+            self._service_config.org.set(rate_limit_key, rate_limit_json)
+        else:
+            self._service_config.org.remove(rate_limit_key)
+
+    def set_server_rate_limits(self, rate_limits_config):
+        rate_limits = self._service_config.server.get('rate_limits', {})
+        if rate_limits_config.is_limited():
+            rate_limits[self._service] = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
+        else:
+            rate_limits.pop(self._service, None)
+        if rate_limits:
+            self._service_config.server.set('rate_limits', rate_limits)
+        else:
+            self._service_config.server.remove('rate_limits')
+

server/lib/python/cartodb_services/cartodb_services/config/service_configuration.py

@@ -0,0 +1,36 @@
+from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
+from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
+from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
+from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
+
+class ServiceConfiguration(object):
+    """
+    This class instantiates configuration backend objects for all the configuration levels of a service:
+    * environment
+    * server
+    * organization
+    * user
+    The configuration backends allow retrieval and modification of configuration parameters.
+    """
+
+    def __init__(self, service, username, orgname):
+        self._server_config_backend = ServerConfigBackendFactory().get()
+        self._environment = ServerEnvironmentBuilder(self._server_config_backend ).get()
+        self._user_config_backend = UserConfigBackendFactory(username, self._environment, self._server_config_backend ).get()
+        self._org_config_backend = OrgConfigBackendFactory(orgname, self._environment, self._server_config_backend ).get()
+
+    @property
+    def environment(self):
+        return self._environment
+
+    @property
+    def server(self):
+        return self._server_config_backend
+
+    @property
+    def user(self):
+        return self._user_config_backend
+
+    @property
+    def org(self):
+        return self._org_config_backend

server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py

@@ -6,6 +6,6 @@ class ConfigBackendInterface(object):
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
-    def get(self, key):
+    def get(self, key, default=None):
         """Return a value based on the key supplied from some storage"""
         pass

server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py

@@ -5,8 +5,8 @@ class InMemoryConfigStorage(ConfigBackendInterface):
     def __init__(self, config_hash={}):
         self._config_hash = config_hash
 
-    def get(self, key):
+    def get(self, key, default=None):
         try:
             return self._config_hash[key]
         except KeyError:
-            return None
+            return default

server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py

@@ -2,5 +2,5 @@ from ..core.interfaces import ConfigBackendInterface
 
 class NullConfigStorage(ConfigBackendInterface):
 
-    def get(self, key):
-        return None
+    def get(self, key, default=None):
+        return default

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py

@@ -9,11 +9,19 @@ class RedisConfigStorage(ConfigBackendInterface):
         self._config_key = config_key
         self._data = None
 
-    def get(self, key):
+    def get(self, key, default=KeyError):
         if not self._data:
             self._data = self._connection.hgetall(self._config_key)
-        return self._data[key]
+        if (default == KeyError):
+            return self._data[key]
+        else:
+            return self._data.get(key, default)
 
+    def set(self, key, value):
+        self._connection.hset(self._config_key, key, value)
+
+    def remove(self, key):
+        self._connection.hdel(self._config_key, key)
 
 class RedisUserConfigStorageBuilder(object):
     def __init__(self, redis_connection, username):

server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py

@@ -4,11 +4,28 @@ from ..core.interfaces import ConfigBackendInterface
 
 class InDbServerConfigStorage(ConfigBackendInterface):
 
-    def get(self, key):
+    def get(self, key, default=None):
         sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key)
         rows = cartodb_services.plpy.execute(sql, 1)
-        json_output = rows[0]['conf']
-        if json_output:
+        json_output = None
+        try:
+            json_output = rows[0]['conf']
+        except (IndexError, KeyError):
+            pass
+        if (json_output):
             return json.loads(json_output)
         else:
-            return None
+            if (default == KeyError):
+                raise KeyError
+            else:
+                return default
+
+    def set(self, key, config):
+        json_config = json.dumps(config)
+        quoted_config = cartodb_services.plpy.quote_nullable(json_config)
+        sql = "SELECT cdb_dataservices_server.cdb_conf_setconf('{0}', {1})".format(key, quoted_config)
+        cartodb_services.plpy.execute(sql)
+
+    def remove(self, key):
+        sql = "SELECT cdb_dataservices_server.cdb_conf_removeconf('{0}')".format(key)
+        cartodb_services.plpy.execute(sql)

server/lib/python/cartodb_services/cartodb_services/tools/__init__.py

@@ -2,3 +2,6 @@ from redis_tools import RedisConnection, RedisDBConfig
 from coordinates import Coordinate
 from polyline import PolyLine
 from log import Logger, LoggerConfig
+from rate_limiter import RateLimiter
+from service_manager import ServiceManager, RateLimitExceeded
+from legacy_service_manager import  LegacyServiceManager

server/lib/python/cartodb_services/cartodb_services/tools/legacy_service_manager.py

@@ -0,0 +1,23 @@
+from cartodb_services.metrics import QuotaService
+from cartodb_services.tools import Logger,LoggerConfig
+from cartodb_services.tools import RateLimiter
+from cartodb_services.config import RateLimitsConfigLegacyBuilder
+from cartodb_services.tools.service_manager import ServiceManagerBase
+import plpy
+
+class LegacyServiceManager(ServiceManagerBase):
+    """
+    This service manager relies on cached configuration (in gd) stored in *legacy* configuration objects
+    It's intended for use by the *legacy* configuration objects (in use prior to the configuration refactor).
+    """
+
+    def __init__(self, service, username, orgname, gd):
+        redis_conn = gd["redis_connection_{0}".format(username)]['redis_metrics_connection']
+        self.config = gd["user_{0}_config_{1}".format(service, username)]
+        logger_config = gd["logger_config"]
+        self.logger = Logger(logger_config)
+
+        self.quota_service = QuotaService(self.config, redis_conn)
+
+        rate_limit_config = RateLimitsConfigLegacyBuilder(redis_conn, plpy, service=service, username=username, orgname=orgname).get()
+        self.rate_limiter = RateLimiter(rate_limit_config, redis_conn)

server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py

@@ -0,0 +1,18 @@
+from rratelimit import Limiter
+
+class RateLimiter:
+
+    def __init__(self, rate_limits_config, redis_connection):
+        self._config = rate_limits_config
+        self._limiter = None
+        if (self._config.is_limited()):
+            self._limiter = Limiter(redis_connection,
+                                    action=self._config.service,
+                                    limit=self._config.limit,
+                                    period=self._config.period)
+
+    def check(self):
+        ok = True
+        if (self._limiter):
+            ok = self._limiter.checked_insert(self._config.username)
+        return ok

server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py

@@ -0,0 +1,70 @@
+from cartodb_services.metrics import QuotaService
+from cartodb_services.tools import Logger
+from cartodb_services.tools import RateLimiter
+from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
+from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
+from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
+
+class RateLimitExceeded(Exception):
+    def __str__(self):
+            return repr('Rate limit exceeded')
+
+class ServiceManagerBase:
+    """
+    A Service manager collects the configuration needed to use a service,
+    including thir-party services parameters.
+
+    This abstract class serves as the base for concrete service manager classes;
+    derived class must provide and initialize attributes for ``config``,
+    ``quota_service``, ``logger`` and ``rate_limiter`` (which can be None
+    for no limits).
+
+    It provides an `assert_within_limits` method to check quota and rate limits
+    which raises exceptions when limits are exceeded.
+
+    It exposes properties containing:
+
+    * ``config`` : a configuration object containing the configuration parameters for
+      a given service and provider.
+    * ``quota_service`` a QuotaService object to for quota accounting
+    * ``logger``
+
+    """
+
+    def assert_within_limits(self, quota=True, rate=True):
+        if rate and not self.rate_limiter.check():
+            raise RateLimitExceeded()
+        if quota and not self.quota_service.check_user_quota():
+            raise Exception('You have reached the limit of your quota')
+
+    @property
+    def config(self):
+        return self.config
+
+    @property
+    def quota_service(self):
+        return self.quota_service
+
+    @property
+    def logger(self):
+        return self.logger
+class ServiceManager(ServiceManagerBase):
+    """
+    This service manager delegates the configuration parameter details,
+    and the policies about configuration precedence to a configuration-builder class.
+    It uses the refactored configuration classes.
+    """
+
+    def __init__(self, service, config_builder, username, orgname):
+        service_config = ServiceConfiguration(service, username, orgname)
+
+        logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get()
+        self.logger = Logger(logger_config)
+
+        self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get()
+        rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
+
+        redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get()
+
+        self.rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection)
+        self.quota_service = QuotaService(self.config, redis_metrics_connection)

server/lib/python/cartodb_services/requirements.txt

@@ -4,7 +4,8 @@ python-dateutil==2.2
 googlemaps==2.4.2
 rollbar==0.13.2
 # Dependency for googlemaps package
-requests<=2.9.1
+requests==2.9.1
+rratelimit==0.0.4
 
 # Test
 mock==1.3.0

server/lib/python/cartodb_services/test/mock_plpy.py

@@ -16,7 +16,7 @@ class MockPlPy:
     def __init__(self):
         self._reset()
 
-    def _reset(self):
+    def _reset(self, log_executed_queries=False):
         self.infos = []
         self.notices = []
         self.debugs = []
@@ -28,11 +28,30 @@ class MockPlPy:
         self.results = []
         self.prepares = []
         self.results = {}
+        self._log_executed_queries = log_executed_queries
+        self._logged_queries = []
 
     def _define_result(self, query, result):
         pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
         self.results[pattern] = result
 
+    def _executed_queries(self):
+        if self._log_executed_queries:
+            return self._logged_queries
+        else:
+            raise Exception('Executed queries logging is not active')
+
+    def _has_executed_query(self, query):
+        pattern = re.compile(re.escape(query))
+        for executed_query in self._executed_queries():
+            if pattern.search(executed_query):
+                return True
+        return False
+
+    def _start_logging_executed_queries(self):
+        self._logged_queries = []
+        self._log_executed_queries = True
+
     def notice(self, msg):
         self.notices.append(msg)
 
@@ -47,7 +66,15 @@ class MockPlPy:
         return MockCursor(data)
 
     def execute(self, query, rows=1):
+        if self._log_executed_queries:
+            self._logged_queries.append(query)
         for pattern, result in self.results.iteritems():
             if pattern.search(query):
                 return result
         return []
+
+    def quote_nullable(self, value):
+        if value is None:
+            return 'NULL'
+        else:
+            return "'{0}'".format(value)

server/lib/python/cartodb_services/test/test_ratelimitsconfig.py

@@ -0,0 +1,124 @@
+from test_helper import *
+from unittest import TestCase
+from mock import Mock, MagicMock, patch
+from nose.tools import assert_raises, assert_not_equal, assert_equal
+
+from datetime import datetime, date
+from mockredis import MockRedis
+import cartodb_services
+
+from cartodb_services.tools import ServiceManager, LegacyServiceManager
+
+from cartodb_services.metrics import GeocoderConfig
+from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
+from cartodb_services.tools import RateLimitExceeded
+
+from cartodb_services.refactor.storage.redis_config import *
+from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
+from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
+from cartodb_services.config import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+
+class TestRateLimitsConfig(TestCase):
+
+    def setUp(self):
+        plpy_mock_config()
+        cartodb_services.init(plpy_mock, _GD={})
+        self.username = 'test_user'
+        self.orgname = 'test_org'
+        self.redis_conn = MockRedis()
+        build_redis_user_config(self.redis_conn, self.username, 'geocoding')
+        build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
+        self.environment = 'production'
+        plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+        basic_server_conf = {"server_conf": {"environment": "testing"},
+                                                     "mapzen_conf":
+                                                     {"geocoder":
+                                                      {"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}}
+                                                     }, "logger_conf": {}}
+        self.empty_server_config = InMemoryConfigStorage(basic_server_conf)
+        self.empty_redis_config = InMemoryConfigStorage({})
+        self.user_config = RedisUserConfigStorageBuilder(self.redis_conn, self.username).get()
+        self.org_config = RedisOrgConfigStorageBuilder(self.redis_conn, self.orgname).get()
+        self.server_config = ServerConfigBackendFactory().get()
+
+    def test_server_config(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+
+            # Write server level configuration
+            config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
+            config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
+            plpy_mock._start_logging_executed_queries()
+            config_setter.set_server_rate_limits(config)
+            assert plpy_mock._has_executed_query('cdb_conf_setconf(\'rate_limits\', \'{"geocoder": {"limit": 1234, "period": 86400}}\')')
+
+            # Re-read configuration
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
+            read_config = RateLimitsConfigBuilder(
+                server_conf=self.server_config,
+                user_conf=self.empty_redis_config,
+                org_conf=self.empty_redis_config,
+                service='geocoder',
+                username=self.username,
+                orgname=self.orgname
+            ).get()
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+            assert_equal(read_config, config)
+
+    def test_server_org_config(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+
+            server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
+            org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400)
+            config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
+
+            # Write server level configuration
+            config_setter.set_server_rate_limits(server_config)
+            # Override with org level configuration
+            config_setter.set_org_rate_limits(org_config)
+
+            # Re-read configuration
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
+            read_config = RateLimitsConfigBuilder(
+                server_conf=self.server_config,
+                user_conf=self.empty_redis_config,
+                org_conf=self.org_config,
+                service='geocoder',
+                username=self.username,
+                orgname=self.orgname
+            ).get()
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+            assert_equal(read_config, org_config)
+
+    def test_server_org_user_config(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+
+            server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
+            org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400)
+            user_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1236, period=86400)
+            config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
+
+            # Write server level configuration
+            config_setter.set_server_rate_limits(server_config)
+            # Override with org level configuration
+            config_setter.set_org_rate_limits(org_config)
+            # Override with user level configuration
+            config_setter.set_user_rate_limits(user_config)
+
+            # Re-read configuration
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
+            read_config = RateLimitsConfigBuilder(
+                server_conf=self.server_config,
+                user_conf=self.user_config,
+                org_conf=self.org_config,
+                service='geocoder',
+                username=self.username,
+                orgname=self.orgname
+            ).get()
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+            assert_equal(read_config, user_config)

server/lib/python/cartodb_services/test/test_servicemanager.py

@@ -0,0 +1,218 @@
+from test_helper import *
+from unittest import TestCase
+from mock import Mock, MagicMock, patch
+from nose.tools import assert_raises, assert_not_equal, assert_equal
+
+from datetime import datetime, date
+from cartodb_services.tools import ServiceManager, LegacyServiceManager
+from mockredis import MockRedis
+import cartodb_services
+from cartodb_services.metrics import GeocoderConfig
+from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
+from cartodb_services.tools import RateLimitExceeded
+
+LUA_AVAILABLE_FOR_MOCKREDIS = False
+
+class MockRedisWithVersionInfo(MockRedis):
+    def info(self):
+        return {'redis_version': '3.0.2'}
+
+class TestServiceManager(TestCase):
+
+    def setUp(self):
+        plpy_mock_config()
+        cartodb_services.init(plpy_mock, _GD={})
+        self.username = 'test_user'
+        self.orgname = 'test_org'
+        self.redis_conn = MockRedisWithVersionInfo()
+        build_redis_user_config(self.redis_conn, self.username, 'geocoding')
+        build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
+        self.environment = 'production'
+        plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+
+    def check_rate_limit(self, service_manager, n, active=True):
+        if LUA_AVAILABLE_FOR_MOCKREDIS:
+            for _ in xrange(n):
+                service_manager.assert_within_limits()
+            if active:
+                with assert_raises(RateLimitExceeded):
+                    service_manager.assert_within_limits()
+            else:
+                service_manager.assert_within_limits()
+        else:
+            # rratelimit doesn't work with MockRedis because it needs Lua support
+            # so, we'll simply perform some sanity check on the configuration of the rate limiter
+            if active:
+                assert_equal(service_manager.rate_limiter._config.is_limited(), True)
+                assert_equal(service_manager.rate_limiter._config.limit, n)
+            else:
+                assert not service_manager.rate_limiter._config.is_limited()
+
+    def test_legacy_service_manager(self):
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        service_manager.assert_within_limits()
+        assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
+
+    def test_service_manager(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            service_manager.assert_within_limits()
+            assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
+
+    def test_no_rate_limit_by_default(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3, False)
+
+    def test_no_legacy_rate_limit_by_default(self):
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3, False)
+
+    def test_legacy_server_rate_limit(self):
+        rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+
+    def test_server_rate_limit(self):
+        rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+
+    def test_user_rate_limit(self):
+        user_redis_name = "rails:users:{0}".format(self.username)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_legacy_user_rate_limit(self):
+        user_redis_name = "rails:users:{0}".format(self.username)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_org_rate_limit(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+
+    def test_legacy_org_rate_limit(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+
+    def test_user_rate_limit_precedence_over_org(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"1000","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        user_redis_name = "rails:users:{0}".format(self.username)
+        user_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_org_rate_limit_precedence_over_server(self):
+        server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+
+    def test_legacy_user_rate_limit_precedence_over_org(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"1000","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        user_redis_name = "rails:users:{0}".format(self.username)
+        user_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_legacy_org_rate_limit_precedence_over_server(self):
+        server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])