diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 91b87db..521eb9f 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -75,6 +75,8 @@ RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.tools import RateLimiter + from cartodb_services.refactor.config.rate_limits import RateLimitsConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] @@ -82,6 +84,15 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) + + rate_limits_config = RateLimitsConfig('geocoder', + username, + user_geocoder_config.rate_limit.get('limit'), + user_geocoder_config.rate_limit.get('period')) + rate_limiter = RateLimiter(rate_limits_config, redis_conn) + if not rate_limiter.check(): + raise Exception('Rate limit exceeded') + # -- Check the quota quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): @@ -115,7 +126,7 @@ RETURNS Geometry AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) @@ -149,6 +160,7 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger + from cartodb_services.tools import RateLimiter from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder @@ -156,6 +168,7 @@ RETURNS Geometry AS $$ from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory + from cartodb_services.refactor.config.rate_limits import RateLimitsConfigBuilder server_config_backend = ServerConfigBackendFactory().get() environment = ServerEnvironmentBuilder(server_config_backend).get() @@ -166,19 +179,14 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() + rate_limit_config = RateLimiterConfigBuilder(server_config_backend, user_config_backend, org_config_backend, service='geocoder' user=username, org=orgname).get() redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() - #-- e.g: RateLimit(service='geocoder', user=username, max_requests=2, period=60) - #-- rate_limiter = RateLimitBuilder(service='geocoder', user=username) - #-- How to pass the redis config along? - #-- rate_limiter_geocoder_config = RateLimiterUserConfigFactory(redis_metrics_connection, service='geocoder', user=username) - from rratelimit import Limiter - rate_limiter = Limiter(redis_metrics_connection, action='geocode', limit=2, period=60) - if not rate_limiter.checked_insert(username): + rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection) + if not rate_limiter.check(): raise Exception('Rate limit exceeded') - quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 14638f3..a3ff3bb 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -56,6 +56,13 @@ class ServiceConfig(object): else: return None + def _get_rate_limit(self, service): + rate_limit_key = "{0}_rate_limit".format(service) + rate_limit_json = self._redis_config.get(rate_limit_key, None) + if (rate_limit_json): + return rate_limit_json and json.loads(rate_limit_json) + else: + return self._db_config.rate_limits.get(service, {}) class DataObservatoryConfig(ServiceConfig): @@ -376,6 +383,8 @@ class GeocoderConfig(ServiceConfig): self._mapzen_api_key = db_config.mapzen_geocoder_api_key self._cost_per_hit = 0 + self._rate_limit = self._get_rate_limit('geocoder') + @property def service_type(self): if self._geocoder_provider == self.GOOGLE_GEOCODER: @@ -444,6 +453,9 @@ class GeocoderConfig(ServiceConfig): def provider(self): return self._geocoder_provider + @property + def rate_limit(self): + return self._rate_limit class ServicesDBConfig: @@ -458,6 +470,7 @@ class ServicesDBConfig: self._get_here_config() self._get_mapzen_config() self._get_data_observatory_config() + self._get_rate_limits_config() def _get_server_config(self): server_config_json = self._get_conf('server_conf') @@ -509,13 +522,17 @@ class ServicesDBConfig: else: self._data_observatory_connection_str = do_conf['connection']['production'] + def _get_rate_limits_config(self): + self._rate_limits = self._get_conf('rate_limits', default={}) - def _get_conf(self, key): + def _get_conf(self, key, default='raise'): try: sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) conf = self._db_conn.execute(sql, 1) return conf[0]['conf'] except Exception as e: + if (default != 'raise'): + return default raise ConfigException("Error trying to get config for {0}: {1}".format(key, e)) @property @@ -570,6 +587,10 @@ class ServicesDBConfig: def data_observatory_connection_str(self): return self._data_observatory_connection_str + @property + def rate_limits(self): + return self._rate_limits + @property def logger_config(self): logger_conf_json = self._get_conf('logger_conf') @@ -592,6 +613,7 @@ class ServicesRedisConfig: GEOCODER_PROVIDER_KEY = 'geocoder_provider' ISOLINES_PROVIDER_KEY = 'isolines_provider' ROUTING_PROVIDER_KEY = 'routing_provider' + GEOCODING_RATE_LIMIT_KEY = 'geocoder_rate_limit' def __init__(self, redis_conn): self._redis_connection = redis_conn @@ -646,3 +668,6 @@ class ServicesRedisConfig: user_config[self.ISOLINES_PROVIDER_KEY] = org_config[self.ISOLINES_PROVIDER_KEY] if self.ROUTING_PROVIDER_KEY in org_config: user_config[self.ROUTING_PROVIDER_KEY] = org_config[self.ROUTING_PROVIDER_KEY] + # for rate limit parameters, user config has precedence over organization + if self.GEOCODING_RATE_LIMIT_KEY in org_config and not self.GEOCODING_RATE_LIMIT_KEY in user_config: + user_config[self.GEOCODING_RATE_LIMIT_KEY] = org_config[self.GEOCODING_RATE_LIMIT_KEY] diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py new file mode 100644 index 0000000..9a2b6de --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py @@ -0,0 +1,63 @@ +class RateLimitsConfig(object): + """ + Value object that represents the configuration needed to rate-limit services + """ + + def __init__(self, + service, + username, + limit, + period): + self._service = service + self._username = username + self._limit = limit and int(limit) + self._period = period and int(period) + + # service this limit applies to + @property + def service(self): + return self._service + + # user this limit applies to + @property + def username(self): + return self._username + + # rate period in seconds + @property + def period(self): + return self._period + + # rate limit in seconds + @property + def limit(self): + return self._limit + + def is_limited(self): + return self._limit and self._limit > 0 and self._period and self._period > 0 + + +class RateLimitsConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, service, user, org): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._service = service + self._username = user + self._orgname = org + + def get(self): + # Order of precedence is user_conf, org_conf, server_conf + + rate_limit_key = "{0}_rate_limit".format(service) + rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None) + if (rate_limit_json): + rate_limit = rate_limit_json and json.loads(rate_limit_json) + else: + rate_limit = self._server_conf.get('rate_limits', {}).get(service, {}) + + return RateLimitsConfig(self._service, + self._username, + self._limit, + self._period) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py b/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py new file mode 100644 index 0000000..16cf3f2 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py @@ -0,0 +1,17 @@ +from rratelimit import Limiter + +class RateLimiter: + + def __init__(self, rate_limits_config, redis_connection): + self._config = rate_limits_config + if (self._config.is_limited()): + self._limiter = Limiter(rdis_connection, + action=self._config.service, + limit=self._config.limit, + period=self._config.period) + + def check(): + ok = True + if (self._limiter): + ok = self._limiter.checked_insert(self._config.username) + return ok