From 335d91b42de504b14621151c3db6bc7ae5e85398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 28 Feb 2018 13:11:18 +0100 Subject: [PATCH] separate rate limit logic from middleware and EVALSHA --- lib/cartodb/api/user_limits_api.js | 152 ++++++++++++++++++++ lib/cartodb/controllers/layergroup.js | 26 ++-- lib/cartodb/controllers/map.js | 8 +- lib/cartodb/controllers/named_maps.js | 17 ++- lib/cartodb/controllers/named_maps_admin.js | 14 +- lib/cartodb/middleware/rate-limit.js | 131 +---------------- lib/cartodb/server.js | 11 +- test/acceptance/rate-limit.test.js | 130 ++++++++++++++++- test/unit/cartodb/rate-limit.test.js | 3 +- 9 files changed, 329 insertions(+), 163 deletions(-) diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 0f4f0f4c..91125150 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -1,5 +1,8 @@ var step = require('step'); + +const RATE_LIMIT_REDIS_DB = 8; + /** * * @param metadataBackend @@ -11,6 +14,13 @@ function UserLimitsApi(metadataBackend, options) { this.metadataBackend = metadataBackend; this.options = options || {}; this.options.limits = this.options.limits || {}; + this.rateLimits = { + redisCommand: 'EVAL', + sha: null, + lua: getRateLimitLuaScript() + }; + + this.preprareRateLimit(); } module.exports = UserLimitsApi; @@ -77,3 +87,145 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call callback ); }; + +UserLimitsApi.prototype.preprareRateLimit = function () { + var self = this; + + if (this.options.limits.rateLimitsEnabled) { + this.metadataBackend.redisCmd( + RATE_LIMIT_REDIS_DB, + 'SCRIPT', + ['LOAD', getRateLimitLuaScript()], + function (err, sha) { + if (!err && sha) { + self.rateLimits.sha = sha; + self.rateLimits.redisCommand = 'EVALSHA'; + } + } + ); + } +}; + +UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) { + var self = this; + + let redisParams = [ + this.rateLimits.redisCommand === 'EVAL' ? this.rateLimits.lua : this.rateLimits.sha, + 2, + getStoreKey(user, endpointGroup), // KEY[1] + getStatusKey(user, endpointGroup) // KEY[2] + ]; + + this.metadataBackend.redisCmd( + RATE_LIMIT_REDIS_DB, + this.rateLimits.redisCommand, + redisParams, + function (err, rateLimits) { + if (err && err.name === 'ReplyError' && err.message === 'NOSCRIPT No matching script. Please use EVAL.') { + self.rateLimits.redisCommand = 'EVAL'; + return self.getRateLimit(user, endpointGroup, callback); + } + + let rateLimit; + if (!err) { + rateLimit = getLowerRateLimit(rateLimits); + } + + callback(err, rateLimit); + } + ); +}; + +/** + * Returns Redis key where the limits are saved by user and endpoint + * The value is a Redis hash: + * maxBurst (b): Integer (as string) + * countPerPeriod (c): Integer (as string) + * period (p): Integer (as string) + * @param {string} user + * @param {string} endpointGroup + */ +function getStoreKey(user, endpointGroup) { + return `limits:rate:store:${user}:${endpointGroup}`; +} + +/** + * Returns Redis key where the current state of the limit by user and endpoint + * This key is managed by redis-cell (CL.THROTTLE command) + * @param {string} user + * @param {string} endpointGroup + */ +function getStatusKey(user, endpointGroup) { + return `limits:rate:status:${user}:${endpointGroup}`; +} + +function getRateLimitLuaScript() { + return ` + local results = {} + local resultsCounter = 0 + + local limits = {} + local limitsArray = redis.call("LRANGE", KEYS[1], 0, -1) + + for i, v in ipairs(limitsArray) do + local rest = i % 3 + if rest ~= 0 then + limits[rest] = v + else + resultsCounter = resultsCounter + 1 + results[resultsCounter] = redis.call("CL.THROTTLE", KEYS[2], limits[1], limits[2], v) + end + end + + return results + `; +} + +/** + * Returns the inner rateLimit what is the strictest one + * @param {Array} rateLimits Each inner array has 5 integers indicating: + * isBloqued, limit, remaining, retry, reset + */ +function getLowerRateLimit(rateLimits) { + /*jshint maxcomplexity:10 */ + if (!rateLimits || !Array.isArray(rateLimits) || !rateLimits.length) { + return; + } + + let minIndex = 0; + let minRemainingValue; + let currentIndex = 0; + for (let rateLimit of rateLimits) { + if (!validateRatelimit(rateLimit)) { + currentIndex++; + continue; + } + + const [isBlocked, , remaining] = rateLimit; + + if (isBlocked === 1) { + minIndex = currentIndex; + break; + } + + if (minRemainingValue === undefined || remaining < minRemainingValue) { + minIndex = currentIndex; + minRemainingValue = remaining; + } + + currentIndex++; + } + + if (validateRatelimit(rateLimits[minIndex])) { + return rateLimits[minIndex]; + } else { + return; + } +} + +function validateRatelimit(rateLimit) { + return rateLimit.length === 5; +} + +module.exports.getStoreKey = getStoreKey; +module.exports.getLowerRateLimit = getLowerRateLimit; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 370c530a..7257164d 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -38,8 +38,7 @@ function LayergroupController( surrogateKeysCache, userLimitsApi, layergroupAffectedTables, - analysisBackend, - metadataBackend + analysisBackend ) { this.pgConnection = pgConnection; this.mapStore = mapStore; @@ -54,7 +53,6 @@ function LayergroupController( this.analysisStatusBackend = new AnalysisStatusBackend(); this.prepareContext = prepareContext; - this.metadataBackend = metadataBackend; } module.exports = LayergroupController; @@ -64,7 +62,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), this.prepareContext, this.tile.bind(this), vectorError() @@ -74,7 +72,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/:z/:x/:y.:format', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), this.prepareContext, this.tile.bind(this), vectorError() @@ -84,7 +82,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), validateLayerRouteMiddleware, this.prepareContext, this.layer.bind(this), @@ -95,7 +93,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/:layer/attributes/:fid', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), this.prepareContext, this.attributes.bind(this) ); @@ -104,7 +102,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), allowQueryParams(['layer']), this.prepareContext, this.center.bind(this) @@ -114,7 +112,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), allowQueryParams(['layer']), this.prepareContext, this.bbox.bind(this) @@ -142,7 +140,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/dataview/:dataviewName', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, this.dataview.bind(this) @@ -152,7 +150,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, this.dataview.bind(this) @@ -162,7 +160,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, this.dataviewSearch.bind(this) @@ -172,7 +170,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), allowQueryParams(allowedDataviewQueryParams), this.prepareContext, this.dataviewSearch.bind(this) @@ -182,7 +180,7 @@ LayergroupController.prototype.register = function(app) { app.base_url_mapconfig + '/:token/analysis/node/:nodeId', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), this.prepareContext, this.analysisNodeStatus.bind(this) ); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f83b1616..e508ce75 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -59,28 +59,28 @@ MapController.prototype.register = function(app) { base_url_mapconfig, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), this.composeCreateMapMiddleware() ); app.post( base_url_mapconfig, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), this.composeCreateMapMiddleware() ); app.get( `${base_url_templated}/:template_id/jsonp`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), this.composeCreateMapMiddleware(useTemplate) ); app.post( `${base_url_templated}/:template_id`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), this.composeCreateMapMiddleware(useTemplate) ); app.options( diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 002a8606..b546b5a4 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -29,14 +29,23 @@ function getRequestParams(locals) { return params; } -function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, - surrogateKeysCache, tablesExtentApi, metadataBackend) { +function NamedMapsController( + prepareContext, + namedMapProviderCache, + tileBackend, + previewBackend, + surrogateKeysCache, + tablesExtentApi, + metadataBackend, + userLimitsApi +) { this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; this.metadataBackend = metadataBackend; + this.userLimitsApi = userLimitsApi; this.prepareContext = prepareContext; } @@ -47,7 +56,7 @@ NamedMapsController.prototype.register = function(app) { app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), this.prepareContext, this.getNamedMapProvider('NAMED_MAP_TILE'), this.getAffectedTables(), @@ -65,7 +74,7 @@ NamedMapsController.prototype.register = function(app) { app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), this.prepareContext, this.getNamedMapProvider('STATIC_VIZ_MAP'), diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index ec0a480d..19e8c53a 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -10,10 +10,10 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {TemplateMaps} templateMaps * @constructor */ -function NamedMapsAdminController(authApi, templateMaps, metadataBackend) { +function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { this.authApi = authApi; this.templateMaps = templateMaps; - this.metadataBackend = metadataBackend; + this.userLimitsApi = userLimitsApi; } module.exports = NamedMapsAdminController; @@ -25,7 +25,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), this.checkContentType('POST', 'POST TEMPLATE'), this.authorizedByAPIKey('create', 'POST TEMPLATE'), this.create() @@ -35,7 +35,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/:template_id`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), this.checkContentType('PUT', 'PUT TEMPLATE'), this.authorizedByAPIKey('update', 'PUT TEMPLATE'), this.update() @@ -45,7 +45,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/:template_id`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), this.authorizedByAPIKey('get', 'GET TEMPLATE'), this.retrieve() ); @@ -54,7 +54,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/:template_id`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'), this.destroy() ); @@ -63,7 +63,7 @@ NamedMapsAdminController.prototype.register = function (app) { `${base_url_templated}/`, cors(), userMiddleware, - rateLimit(this.metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'), this.list() ); diff --git a/lib/cartodb/middleware/rate-limit.js b/lib/cartodb/middleware/rate-limit.js index 22247686..4e69a031 100644 --- a/lib/cartodb/middleware/rate-limit.js +++ b/lib/cartodb/middleware/rate-limit.js @@ -1,7 +1,5 @@ 'use strict'; -const REDIS_DB = 8; - const RATE_LIMIT_ENDPOINTS_GROUPS = { ANONYMOUS: 'anonymous', STATIC: 'static', @@ -20,7 +18,7 @@ const RATE_LIMIT_ENDPOINTS_GROUPS = { NAMED_TILES: 'named_tiles' }; -function rateLimitFn(metadataBackend, endpointGroup = null) { +function rateLimitFn(userLimitsApi, endpointGroup = null) { return function rateLimitMiddleware(req, res, next) { if (!global.environment.enabledFeatures.rateLimitsEnabled) { return next(); @@ -28,153 +26,40 @@ function rateLimitFn(metadataBackend, endpointGroup = null) { const user = res.locals.user; - // TODO: remove comments to move it to global - // if (!endpointGroup) { - // endpointGroup = getEndpointGroup(); - // } - if (!endpointGroup || !isRateLimitEnabledByEndpoint(endpointGroup)) { return next(); } - const redisParams = [ - getLuaScript(), - 2, - getStoreKey(user, endpointGroup), // KEY[1] - getStatusKey(user, endpointGroup) // KEY[2] - ]; - - metadataBackend.redisCmd(REDIS_DB, 'EVAL', redisParams, function (err, rateLimits) { + userLimitsApi.getRateLimit(user, endpointGroup, function(err, rateLimit) { if (err) { return next(err); } - - const rateLimit = getLowerRateLimit(rateLimits); - + if (!rateLimit) { return next(); } - + const [isBlocked, limit, remaining, retry, reset] = rateLimit; - + res.set({ 'X-Rate-Limit-Limit': limit, 'X-Rate-Limit-Remaining': remaining, 'X-Rate-Limit-Retry-After': retry, 'X-Rate-Limit-Reset': reset }); - + if (isBlocked) { const err = new Error('You are over the limits.'); err.http_status = 429; return next(err); } - + return next(); }); }; } -/** - * Returns the endpoint key in Redis - */ -// function getEndpointGroup() { -// // TODO: get endpoint from route path -// return null; -// } - -/** - * Returns Redis key where the limits are saved by user and endpoint - * The value is a Redis hash: - * maxBurst (b): Integer (as string) - * countPerPeriod (c): Integer (as string) - * period (p): Integer (as string) - * @param {string} user - * @param {string} endpointGroup - */ -function getStoreKey(user, endpointGroup) { - return `limits:rate:store:${user}:${endpointGroup}`; -} - -/** - * Returns Redis key where the current state of the limit by user and endpoint - * This key is managed by redis-cell (CL.THROTTLE command) - * @param {string} user - * @param {string} endpointGroup - */ -function getStatusKey(user, endpointGroup) { - return `limits:rate:status:${user}:${endpointGroup}`; -} - -function getLuaScript() { - return ` - local results = {} - local resultsCounter = 0 - - local limits = {} - local limitsArray = redis.call("LRANGE", KEYS[1], 0, -1) - - for i, v in ipairs(limitsArray) do - local rest = i % 3 - if rest ~= 0 then - limits[rest] = v - else - resultsCounter = resultsCounter + 1 - results[resultsCounter] = redis.call("CL.THROTTLE", KEYS[2], limits[1], limits[2], v) - end - end - - return results - `; -} - -/** - * Returns the inner rateLimit what is the strictest one - * @param {Array} rateLimits Each inner array has 5 integers indicating: - * isBloqued, limit, remaining, retry, reset - */ -function getLowerRateLimit(rateLimits) { - /*jshint maxcomplexity:10 */ - if (!rateLimits || !Array.isArray(rateLimits) || !rateLimits.length) { - return; - } - - let minIndex = 0; - let minRemainingValue; - let currentIndex = 0; - for (let rateLimit of rateLimits) { - if (!validRatelimit(rateLimit)) { - currentIndex++; - continue; - } - - const [isBlocked, , remaining] = rateLimit; - - if (isBlocked === 1) { - minIndex = currentIndex; - break; - } - - if (minRemainingValue === undefined || remaining < minRemainingValue) { - minIndex = currentIndex; - minRemainingValue = remaining; - } - - currentIndex++; - } - - if (validRatelimit(rateLimits[minIndex])) { - return rateLimits[minIndex]; - } else { - return; - } -} - -function validRatelimit(rateLimit) { - return rateLimit.length === 5; -} - function isRateLimitEnabledByEndpoint(endpointGroup) { return global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup] === true; } @@ -182,5 +67,3 @@ function isRateLimitEnabledByEndpoint(endpointGroup) { module.exports = rateLimitFn; module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS; -module.exports.getStoreKey = getStoreKey; -module.exports.getLowerRateLimit = getLowerRateLimit; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 9e8669ce..db5d66fb 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -76,7 +76,8 @@ module.exports = function(serverOptions) { var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, - render: serverOptions.renderer.mapnik.limits.render || 0 + render: serverOptions.renderer.mapnik.limits.render || 0, + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled } }); @@ -233,8 +234,7 @@ module.exports = function(serverOptions) { surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - analysisBackend, - metadataBackend + analysisBackend ).register(app); new controller.Map( @@ -257,10 +257,11 @@ module.exports = function(serverOptions) { previewBackend, surrogateKeysCache, tablesExtentApi, - metadataBackend + metadataBackend, + userLimitsApi ).register(app); - new controller.NamedMapsAdmin(authApi, templateMaps, metadataBackend).register(app); + new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app); new controller.Analyses(prepareContext).register(app); diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js index 51f0ebcb..edfc5fcc 100644 --- a/test/acceptance/rate-limit.test.js +++ b/test/acceptance/rate-limit.test.js @@ -5,9 +5,12 @@ const redis = require('redis'); const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const TestClient = require('../support/test-client'); +const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api'); const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS, getStoreKey } = rateLimitMiddleware; +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; +const { getStoreKey } = require('../../lib/cartodb/api/user_limits_api'); +let userLimitsApi; let rateLimit; let redisClient; let testClient; @@ -274,13 +277,18 @@ describe('rate limit', function() { describe('rate limit middleware', function () { - before(function () { + before(function (done) { global.environment.enabledFeatures.rateLimitsEnabled = true; global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; const redisPool = new RedisPool(global.environment.redis); const metadataBackend = cartodbRedis({ pool: redisPool }); - rateLimit = rateLimitMiddleware(metadataBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS); + userLimitsApi = new UserLimitsApi(metadataBackend, { + limits: { + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS); redisClient = redis.createClient(global.environment.redis.port); testClient = new TestClient(createMapConfig(), 1234); @@ -290,6 +298,8 @@ describe('rate limit middleware', function () { const period = 1; const burst = 0; setLimit(count, period, burst); + + setTimeout(done, 10); }); after(function () { @@ -449,4 +459,118 @@ describe('rate limit middleware', function () { 1050 ); }); + + it("1 req/sec: 2 req/seg should be limited, removing script SHA from Redis", function (done) { + userLimitsApi.metadataBackend.redisCmd( + 8, + 'SCRIPT', + ['FLUSH'], + function () { + let { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.ifError(err); + assert.deepEqual(res.headers, { + "X-Rate-Limit-Limit": 1, + "X-Rate-Limit-Remaining": 0, + "X-Rate-Limit-Reset": 1, + "X-Rate-Limit-Retry-After": -1 + }); + }); + + setTimeout( + function () { + let { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.ok(err); + assert.deepEqual(res.headers, { + "X-Rate-Limit-Limit": 1, + "X-Rate-Limit-Remaining": 0, + "X-Rate-Limit-Reset": 0, + "X-Rate-Limit-Retry-After": 0 + }); + assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.http_status, 429); + }); + }, + 250 + ); + + setTimeout( + function () { + let { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.ok(err); + assert.deepEqual(res.headers, { + "X-Rate-Limit-Limit": 1, + "X-Rate-Limit-Remaining": 0, + "X-Rate-Limit-Reset": 0, + "X-Rate-Limit-Retry-After": 0 + }); + assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.http_status, 429); + }); + }, + 500 + ); + + setTimeout( + function () { + let { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.ok(err); + assert.deepEqual(res.headers, { + "X-Rate-Limit-Limit": 1, + "X-Rate-Limit-Remaining": 0, + "X-Rate-Limit-Reset": 0, + "X-Rate-Limit-Retry-After": 0 + }); + assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.http_status, 429); + }); + }, + 750 + ); + + setTimeout( + function () { + let { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.ok(err); + assert.deepEqual(res.headers, { + "X-Rate-Limit-Limit": 1, + "X-Rate-Limit-Remaining": 0, + "X-Rate-Limit-Reset": 0, + "X-Rate-Limit-Retry-After": 0 + }); + assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.http_status, 429); + }); + }, + 950 + ); + + setTimeout( + function () { + let { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.ifError(err); + assert.deepEqual(res.headers, { + "X-Rate-Limit-Limit": 1, + "X-Rate-Limit-Remaining": 0, + "X-Rate-Limit-Reset": 1, + "X-Rate-Limit-Retry-After": -1 + }); + }); + }, + 1050 + ); + + setTimeout( function() { + userLimitsApi.preprareRateLimit(); + setTimeout(done, 1000); + }, 1100); + } + ); + + }); }); diff --git a/test/unit/cartodb/rate-limit.test.js b/test/unit/cartodb/rate-limit.test.js index 323cdd6d..1dc6dfc4 100644 --- a/test/unit/cartodb/rate-limit.test.js +++ b/test/unit/cartodb/rate-limit.test.js @@ -1,8 +1,7 @@ require('../../support/test_helper'); const assert = require('assert'); - -const { getLowerRateLimit } = require('../../../lib/cartodb/middleware/rate-limit'); +const { getLowerRateLimit } = require('../../../lib/cartodb/api/user_limits_api'); describe('Lower rate limit', function () {