cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Geocoder, per user, cofigurable rate limits

Browse Source 
WIP, specially the GeocoderConfig part is flaky and ugly
This commit is contained in:
Javier Goizueta
2017-03-09 18:42:48 +01:00
parent 25ba9866ae
commit 250d384d06
4 changed files with 123 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
server/extension/sql/20_geocode_street.sql
View File

@@ -75,6 +75,8 @@ RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
from cartodb_services.tools import RateLimiter
from cartodb_services.refactor.config.rate_limits import RateLimitsConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
@@ -82,6 +84,15 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
rate_limits_config = RateLimitsConfig('geocoder',
username,
user_geocoder_config.rate_limit.get('limit'),
user_geocoder_config.rate_limit.get('period'))
rate_limiter = RateLimiter(rate_limits_config, redis_conn)
if not rate_limiter.check():
raise Exception('Rate limit exceeded')
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
@@ -115,7 +126,7 @@ RETURNS Geometry AS $$
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
@@ -149,6 +160,7 @@ RETURNS Geometry AS $$
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
from cartodb_services.tools import RateLimiter
from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
@@ -156,6 +168,7 @@ RETURNS Geometry AS $$
from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
from cartodb_services.refactor.config.rate_limits import RateLimitsConfigBuilder
server_config_backend = ServerConfigBackendFactory().get()
environment = ServerEnvironmentBuilder(server_config_backend).get()
@@ -166,19 +179,14 @@ RETURNS Geometry AS $$
logger = Logger(logger_config)
mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
rate_limit_config = RateLimiterConfigBuilder(server_config_backend, user_config_backend, org_config_backend, service='geocoder' user=username, org=orgname).get()
redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
#-- e.g: RateLimit(service='geocoder', user=username, max_requests=2, period=60)
#-- rate_limiter = RateLimitBuilder(service='geocoder', user=username)
#-- How to pass the redis config along?
#-- rate_limiter_geocoder_config = RateLimiterUserConfigFactory(redis_metrics_connection, service='geocoder', user=username)
from rratelimit import Limiter
rate_limiter = Limiter(redis_metrics_connection, action='geocode', limit=2, period=60)
if not rate_limiter.checked_insert(username):
rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection)
if not rate_limiter.check():
raise Exception('Rate limit exceeded')
quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')

27
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -56,6 +56,13 @@ class ServiceConfig(object):
else:
return None
def _get_rate_limit(self, service):
rate_limit_key = "{0}_rate_limit".format(service)
rate_limit_json = self._redis_config.get(rate_limit_key, None)
if (rate_limit_json):
return rate_limit_json and json.loads(rate_limit_json)
else:
return self._db_config.rate_limits.get(service, {})
class DataObservatoryConfig(ServiceConfig):
@@ -376,6 +383,8 @@ class GeocoderConfig(ServiceConfig):
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._cost_per_hit = 0
self._rate_limit = self._get_rate_limit('geocoder')
@property
def service_type(self):
if self._geocoder_provider == self.GOOGLE_GEOCODER:
@@ -444,6 +453,9 @@ class GeocoderConfig(ServiceConfig):
def provider(self):
return self._geocoder_provider
@property
def rate_limit(self):
return self._rate_limit
class ServicesDBConfig:
@@ -458,6 +470,7 @@ class ServicesDBConfig:
self._get_here_config()
self._get_mapzen_config()
self._get_data_observatory_config()
self._get_rate_limits_config()
def _get_server_config(self):
server_config_json = self._get_conf('server_conf')
@@ -509,13 +522,17 @@ class ServicesDBConfig:
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_rate_limits_config(self):
self._rate_limits = self._get_conf('rate_limits', default={})
def _get_conf(self, key):
def _get_conf(self, key, default='raise'):
try:
sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
conf = self._db_conn.execute(sql, 1)
return conf[0]['conf']
except Exception as e:
if (default != 'raise'):
return default
raise ConfigException("Error trying to get config for {0}: {1}".format(key, e))
@property
@@ -570,6 +587,10 @@ class ServicesDBConfig:
def data_observatory_connection_str(self):
return self._data_observatory_connection_str
@property
def rate_limits(self):
return self._rate_limits
@property
def logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
@@ -592,6 +613,7 @@ class ServicesRedisConfig:
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
ISOLINES_PROVIDER_KEY = 'isolines_provider'
ROUTING_PROVIDER_KEY = 'routing_provider'
GEOCODING_RATE_LIMIT_KEY = 'geocoder_rate_limit'
def __init__(self, redis_conn):
self._redis_connection = redis_conn
@@ -646,3 +668,6 @@ class ServicesRedisConfig:
user_config[self.ISOLINES_PROVIDER_KEY] = org_config[self.ISOLINES_PROVIDER_KEY]
if self.ROUTING_PROVIDER_KEY in org_config:
user_config[self.ROUTING_PROVIDER_KEY] = org_config[self.ROUTING_PROVIDER_KEY]
# for rate limit parameters, user config has precedence over organization
if self.GEOCODING_RATE_LIMIT_KEY in org_config and not self.GEOCODING_RATE_LIMIT_KEY in user_config:
user_config[self.GEOCODING_RATE_LIMIT_KEY] = org_config[self.GEOCODING_RATE_LIMIT_KEY]

63
server/lib/python/cartodb_services/cartodb_services/refactor/config/rate_limits.py Normal file
View File

@@ -0,0 +1,63 @@
class RateLimitsConfig(object):
"""
Value object that represents the configuration needed to rate-limit services
"""
def __init__(self,
service,
username,
limit,
period):
self._service = service
self._username = username
self._limit = limit and int(limit)
self._period = period and int(period)
# service this limit applies to
@property
def service(self):
return self._service
# user this limit applies to
@property
def username(self):
return self._username
# rate period in seconds
@property
def period(self):
return self._period
# rate limit in seconds
@property
def limit(self):
return self._limit
def is_limited(self):
return self._limit and self._limit > 0 and self._period and self._period > 0
class RateLimitsConfigBuilder(object):
def __init__(self, server_conf, user_conf, org_conf, service, user, org):
self._server_conf = server_conf
self._user_conf = user_conf
self._org_conf = org_conf
self._service = service
self._username = user
self._orgname = org
def get(self):
# Order of precedence is user_conf, org_conf, server_conf
rate_limit_key = "{0}_rate_limit".format(service)
rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None)
if (rate_limit_json):
rate_limit = rate_limit_json and json.loads(rate_limit_json)
else:
rate_limit = self._server_conf.get('rate_limits', {}).get(service, {})
return RateLimitsConfig(self._service,
self._username,
self._limit,
self._period)

17
server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py Normal file
View File

@@ -0,0 +1,17 @@
from rratelimit import Limiter
class RateLimiter:
def __init__(self, rate_limits_config, redis_connection):
self._config = rate_limits_config
if (self._config.is_limited()):
self._limiter = Limiter(rdis_connection,
action=self._config.service,
limit=self._config.limit,
period=self._config.period)
def check():
ok = True
if (self._limiter):
ok = self._limiter.checked_insert(self._config.username)
return ok