diff --git a/.gitignore b/.gitignore index ddeef2ef..1000315b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ redis.pid *.log coverage/ .DS_Store +libredis_cell.so diff --git a/NEWS.md b/NEWS.md index aac60129..b61c816f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,13 +1,32 @@ # Changelog +## 6.0.0 +Released 2018-03-19 +Backward incompatible changes: + - Needs Redis v4 + +New features: + - Upgrades camshaft to 0.61.8 + - Upgrades cartodb-redis to 1.0.0 + - Rate limit feature (disabled by default) + ## 5.4.0 -Released yyyy-mm-dd +Released 2018-03-15 - Upgrades Windshaft to 4.5.7 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602)) - Implemented middleware to authorize users via new Api Key system - Keep the old authorization system as fallback - Aggregation widget: Remove NULL categories in 'count' aggregations too - Update request to 2.85.0 - Update camshaft to 0.61.4 (Fixes for AOI and Merge analyses) + - Update windshaft to 4.6.0, which in turn updates @carto/mapnik to 3.6.2-carto.4 and related dependencies. It brings in a cache for rasterized symbols. See https://github.com/CartoDB/node-mapnik/blob/v3.6.2-carto/CHANGELOG.carto.md#362-carto4 + - PostGIS: Variables in postgis SQL queries must now additionally be wrapped in `!` (refs [#29](https://github.com/CartoDB/mapnik/issues/29), [mapnik/#3618](https://github.com/mapnik/mapnik/pull/3618)): +```sql +-- Before +SELECT ... WHERE trait = @variable + +-- Now +SELECT ... WHERE trait = !@variable! +``` ## 5.3.1 Released 2018-02-13 diff --git a/config/environments/development.js.example b/config/environments/development.js.example index df2273c7..12de0912 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -343,7 +343,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 9f7436df..beb9f15c 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -345,7 +345,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: false + layerStats: false, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 66c5a88c..40d3bb8b 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -345,7 +345,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 0c98c4b6..4d2c2d27 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -339,7 +339,28 @@ var config = { // whether the affected tables for a given SQL must query directly postgresql or use the SQL API cdbQueryTablesFromPostgres: true, // whether in mapconfig is available stats & metadata for each layer - layerStats: true + layerStats: true, + // whether it should rate limit endpoints (global configuration) + rateLimitsEnabled: false, + // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true) + rateLimitsByEndpoint: { + anonymous: false, + static: false, + static_named: false, + dataview: false, + dataview_search: false, + analysis: false, + analysis_catalog: false, + tile: false, + attributes: false, + named_list: false, + named_create: false, + named_get: false, + named: false, + named_update: false, + named_delete: false, + named_tiles: false + } } }; diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js index 0f4f0f4c..caf9f151 100644 --- a/lib/cartodb/api/user_limits_api.js +++ b/lib/cartodb/api/user_limits_api.js @@ -11,6 +11,8 @@ function UserLimitsApi(metadataBackend, options) { this.metadataBackend = metadataBackend; this.options = options || {}; this.options.limits = this.options.limits || {}; + + this.preprareRateLimit(); } module.exports = UserLimitsApi; @@ -77,3 +79,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call callback ); }; + +UserLimitsApi.prototype.preprareRateLimit = function () { + if (this.options.limits.rateLimitsEnabled) { + this.metadataBackend.loadRateLimitsScript(); + } +}; + +UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) { + this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback); +}; diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 57e64a61..791f147d 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -7,10 +7,13 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const authorize = require('../middleware/authorize'); const dbConnSetup = require('../middleware/db-conn-setup'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -function AnalysesController(pgConnection, authApi) { +function AnalysesController(pgConnection, authApi, userLimitsApi) { this.pgConnection = pgConnection; this.authApi = authApi; + this.userLimitsApi = userLimitsApi; } module.exports = AnalysesController; @@ -24,6 +27,7 @@ AnalysesController.prototype.register = function (app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index f3976cb6..eb45c022 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -7,6 +7,8 @@ const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DataviewBackend = require('../backends/dataview'); const AnalysisStatusBackend = require('../backends/analysis-status'); const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); @@ -33,8 +35,18 @@ const SUPPORTED_FORMATS = { * @param {AnalysisBackend} analysisBackend * @constructor */ -function LayergroupController(pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend, authApi) { +function LayergroupController( + pgConnection, + mapStore, + tileBackend, + previewBackend, + attributesBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTables, + analysisBackend, + authApi +) { this.pgConnection = pgConnection; this.mapStore = mapStore; this.tileBackend = tileBackend; @@ -60,6 +72,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), layergroupToken(), credentials(), authorize(this.authApi), @@ -84,6 +97,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), layergroupToken(), credentials(), authorize(this.authApi), @@ -109,6 +123,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), layergroupToken(), credentials(), authorize(this.authApi), @@ -133,6 +148,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), layergroupToken(), credentials(), authorize(this.authApi), @@ -155,6 +171,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(['layer']), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), layergroupToken(), credentials(), authorize(this.authApi), @@ -175,6 +192,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(['layer']), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), layergroupToken(), credentials(), authorize(this.authApi), @@ -213,6 +231,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(allowedDataviewQueryParams), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), layergroupToken(), credentials(), authorize(this.authApi), @@ -233,6 +252,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(allowedDataviewQueryParams), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), layergroupToken(), credentials(), authorize(this.authApi), @@ -253,6 +273,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(allowedDataviewQueryParams), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), layergroupToken(), credentials(), authorize(this.authApi), @@ -273,6 +294,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(allowedDataviewQueryParams), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), layergroupToken(), credentials(), authorize(this.authApi), @@ -293,6 +315,7 @@ LayergroupController.prototype.register = function(app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 52d71f93..71229c51 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -16,6 +16,8 @@ const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../utils/layergroup-metadata'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** * @param {AuthApi} authApi @@ -30,9 +32,18 @@ const LayergroupMetadata = require('../utils/layergroup-metadata'); * @param {StatsBackend} statsBackend * @constructor */ -function MapController(pgConnection, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, - statsBackend, authApi) { +function MapController ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authApi +) { this.pgConnection = pgConnection; this.templateMaps = templateMaps; this.mapBackend = mapBackend; @@ -54,18 +65,32 @@ module.exports = MapController; MapController.prototype.register = function(app) { const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; - app.get(`${mapconfigBasePath}`, this.composeCreateMapMiddleware()); - app.post(`${mapconfigBasePath}`, this.composeCreateMapMiddleware()); + app.get( + `${mapconfigBasePath}`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) + ); + + app.post( + `${mapconfigBasePath}`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) + ); const useTemplate = true; - app.get(`${templateBasePath}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate)); - app.post(`${templateBasePath}/:template_id`, this.composeCreateMapMiddleware(useTemplate)); + app.get( + `${templateBasePath}/:template_id/jsonp`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) + ); + + app.post( + `${templateBasePath}/:template_id`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) + ); app.options(app.base_url_mapconfig, cors('Content-Type')); }; -MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) { +MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { const isTemplateInstantiation = useTemplate; const useTemplateHash = useTemplate; const includeQuery = !useTemplate; @@ -77,6 +102,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal cleanUpQueryParams(['aggregation']), locals(), user(), + rateLimit(this.userLimitsApi, endpointGroup), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 6bff861c..815c68c8 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -8,6 +8,8 @@ const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); const vectorError = require('../middleware/vector-error'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DEFAULT_ZOOM_CENTER = { zoom: 1, @@ -32,8 +34,17 @@ function getRequestParams(locals) { return params; } -function NamedMapsController(namedMapProviderCache, tileBackend, previewBackend, - surrogateKeysCache, tablesExtentApi, metadataBackend, pgConnection, authApi) { +function NamedMapsController ( + namedMapProviderCache, + tileBackend, + previewBackend, + surrogateKeysCache, + tablesExtentApi, + metadataBackend, + pgConnection, + authApi, + userLimitsApi +) { this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.previewBackend = previewBackend; @@ -42,6 +53,7 @@ function NamedMapsController(namedMapProviderCache, tileBackend, previewBackend, this.metadataBackend = metadataBackend; this.pgConnection = pgConnection; this.authApi = authApi; + this.userLimitsApi = userLimitsApi; } module.exports = NamedMapsController; @@ -55,6 +67,7 @@ NamedMapsController.prototype.register = function(app) { cleanUpQueryParams(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), layergroupToken(), credentials(), authorize(this.authApi), @@ -83,6 +96,7 @@ NamedMapsController.prototype.register = function(app) { cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index c2e7725e..a519d35b 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -3,6 +3,9 @@ const cors = require('../middleware/cors'); const user = require('../middleware/user'); const locals = require('../middleware/locals'); const credentials = require('../middleware/credentials'); +const userMiddleware = require('../middleware/user'); +const rateLimit = require('../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** * @param {AuthApi} authApi @@ -10,9 +13,10 @@ const credentials = require('../middleware/credentials'); * @param {TemplateMaps} templateMaps * @constructor */ -function NamedMapsAdminController(authApi, templateMaps) { +function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { this.authApi = authApi; this.templateMaps = templateMaps; + this.userLimitsApi = userLimitsApi; } module.exports = NamedMapsAdminController; @@ -25,6 +29,7 @@ NamedMapsAdminController.prototype.register = function (app) { cors(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), credentials(), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), @@ -37,6 +42,7 @@ NamedMapsAdminController.prototype.register = function (app) { cors(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), credentials(), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), @@ -49,6 +55,7 @@ NamedMapsAdminController.prototype.register = function (app) { cors(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), retrieveTemplate({ templateMaps: this.templateMaps }), @@ -60,6 +67,7 @@ NamedMapsAdminController.prototype.register = function (app) { cors(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), destroyTemplate({ templateMaps: this.templateMaps }), @@ -71,6 +79,7 @@ NamedMapsAdminController.prototype.register = function (app) { cors(), locals(), user(), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), listTemplates({ templateMaps: this.templateMaps }), diff --git a/lib/cartodb/middleware/rate-limit.js b/lib/cartodb/middleware/rate-limit.js new file mode 100644 index 00000000..9a3daa9b --- /dev/null +++ b/lib/cartodb/middleware/rate-limit.js @@ -0,0 +1,67 @@ +'use strict'; + +const RATE_LIMIT_ENDPOINTS_GROUPS = { + ANONYMOUS: 'anonymous', + STATIC: 'static', + STATIC_NAMED: 'static_named', + DATAVIEW: 'dataview', + DATAVIEW_SEARCH: 'dataview_search', + ANALYSIS: 'analysis', + ANALYSIS_CATALOG: 'analysis_catalog', + TILE: 'tile', + ATTRIBUTES: 'attributes', + NAMED_LIST: 'named_list', + NAMED_CREATE: 'named_create', + NAMED_GET: 'named_get', + NAMED: 'named', + NAMED_UPDATE: 'named_update', + NAMED_DELETE: 'named_delete', + NAMED_TILES: 'named_tiles' +}; + +function rateLimit(userLimitsApi, endpointGroup = null) { + if (!isRateLimitEnabled(endpointGroup)) { + return function rateLimitDisabledMiddleware(req, res, next) { next(); }; + } + + return function rateLimitMiddleware(req, res, next) { + userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { + if (err) { + return next(err); + } + + if (!userRateLimit) { + return next(); + } + + const [isBlocked, limit, remaining, retry, reset] = userRateLimit; + + res.set({ + 'Carto-Rate-Limit-Limit': limit, + 'Carto-Rate-Limit-Remaining': remaining, + 'Retry-After': retry, + 'Carto-Rate-Limit-Reset': reset + }); + + if (isBlocked) { + const rateLimitError = new Error('You are over the limits.'); + rateLimitError.http_status = 429; + rateLimitError.type = 'limit'; + rateLimitError.subtype = 'rate-limit'; + return next(rateLimitError); + } + + return next(); + }); + }; +} + + +function isRateLimitEnabled(endpointGroup) { + return global.environment.enabledFeatures.rateLimitsEnabled && + endpointGroup && + global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup]; +} + +module.exports = rateLimit; +module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 604c4342..7f7e2712 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -74,7 +74,8 @@ module.exports = function(serverOptions) { var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, - render: serverOptions.renderer.mapnik.limits.render || 0 + render: serverOptions.renderer.mapnik.limits.render || 0, + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled } }); @@ -251,12 +252,13 @@ module.exports = function(serverOptions) { tablesExtentApi, metadataBackend, pgConnection, - authApi + authApi, + userLimitsApi ).register(app); - new controller.NamedMapsAdmin(authApi, templateMaps).register(app); + new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app); - new controller.Analyses(pgConnection, authApi).register(app); + new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app); new controller.ServerInfo(versions).register(app); diff --git a/package.json b/package.json index ecbfd138..b72f9b8c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "5.3.2", + "version": "6.0.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -26,10 +26,10 @@ "dependencies": { "basic-auth": "^2.0.0", "body-parser": "^1.18.2", - "camshaft": "0.61.4", + "camshaft": "0.61.8", "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", - "cartodb-redis": "0.16.0", + "cartodb-redis": "1.0.0", "debug": "^3.1.0", "dot": "~1.0.2", "express": "~4.16.0", @@ -48,7 +48,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.5.7", + "windshaft": "4.6.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/run_tests.sh b/run_tests.sh index d1c63614..8412a851 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,6 +6,7 @@ OPT_DROP_REDIS=yes # drop the redis test environment OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment OPT_COVERAGE=no # run tests with coverage OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files +OPT_REDIS_CELL=yes # download redis cell export PGAPPNAME=cartodb_tiler_tester @@ -49,6 +50,17 @@ die() { exit 1 } +get_redis_cell() { + if test x"$OPT_REDIS_CELL" = xyes; then + echo "Downloading redis-cell" + curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1 + tar xvzf redis-cell.tar.gz > /dev/null 2>&1 + mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so + rm redis-cell.tar.gz + rm libredis_cell.d + fi +} + trap 'cleanup_and_exit' 1 2 3 5 9 13 while [ -n "$1" ]; do @@ -88,6 +100,10 @@ while [ -n "$1" ]; do OPT_CREATE_PGSQL=no shift continue + elif test "$1" = "--norediscell"; then + OPT_REDIS_CELL=no + shift + continue else break fi @@ -99,14 +115,16 @@ if [ -z "$1" ]; then echo " --nocreate do not create the test environment on start" >&2 echo " --nodrop do not drop the test environment on exit" >&2 echo " --with-coverage use istanbul to determine code coverage" >&2 + echo " --norediscell do not download redis-cell" >&2 exit 1 fi TESTS=$@ if test x"$OPT_CREATE_REDIS" = xyes; then + get_redis_cell echo "Starting redis on port ${REDIS_PORT}" - echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log & + echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log & PID_REDIS=$! echo ${PID_REDIS} > ${BASEDIR}/redis.pid fi diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js index 9c0a9966..5844c479 100644 --- a/test/acceptance/named_layers.js +++ b/test/acceptance/named_layers.js @@ -108,7 +108,7 @@ describe('named_layers', function() { }); beforeEach(function(done) { - global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true}; + global.environment.enabledFeatures.cdbQueryTablesFromPostgres = true; templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) { if (err) { return done(err); @@ -125,7 +125,7 @@ describe('named_layers', function() { }); afterEach(function(done) { - global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false}; + global.environment.enabledFeatures.cdbQueryTablesFromPostgres = false; templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) { if (err) { return done(err); diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js new file mode 100644 index 00000000..339e187e --- /dev/null +++ b/test/acceptance/rate-limit.test.js @@ -0,0 +1,260 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const redis = require('redis'); +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); +const TestClient = require('../support/test-client'); +const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api'); +const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; + +let userLimitsApi; +let rateLimit; +let redisClient; +let testClient; +let keysToDelete = ['user:localhost:mapviews:global']; +const user = 'localhost'; + +const query = ` + SELECT + ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator, + 1 cartodb_id, + 2 val +`; + +const createMapConfig = ({ + version = '1.6.0', + type = 'cartodb', + sql = query, + cartocss = TestClient.CARTOCSS.POINTS, + cartocss_version = '2.3.0', + interactivity = 'cartodb_id', + countBy = 'cartodb_id' +} = {}) => ({ + version, + layers: [{ + type, + options: { + source: { + id: 'a0' + }, + cartocss, + cartocss_version, + interactivity + } + }], + analyses: [ + { + id: 'a0', + type: 'source', + params: { + query: sql + } + } + ], + dataviews: { + count: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: countBy, + operation: 'count' + } + } + } +}); + + +function setLimit(count, period, burst) { + redisClient.SELECT(8, err => { + if (err) { + return; + } + + const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`; + redisClient.rpush(key, burst); + redisClient.rpush(key, count); + redisClient.rpush(key, period); + keysToDelete.push(key); + }); +} + +function getReqAndRes() { + return { + req: {}, + res: { + headers: {}, + set(headers) { + this.headers = headers; + }, + locals: { + user: 'localhost' + } + } + }; +} + +function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) { + const response = { + status, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'Carto-Rate-Limit-Limit': limit, + 'Carto-Rate-Limit-Remaining': remaining, + 'Carto-Rate-Limit-Reset': reset, + 'Retry-After': retry + } + }; + + testClient.getLayergroup({ response }, err => { + assert.ifError(err); + if (done) { + setTimeout(done, 1000); + } + }); +} + +function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) { + const { req, res } = getReqAndRes(); + rateLimit(req, res, function (err) { + assert.deepEqual(res.headers, { + "Carto-Rate-Limit-Limit": limit, + "Carto-Rate-Limit-Remaining": remaining, + "Carto-Rate-Limit-Reset": reset, + "Retry-After": retry + }); + + if(status === 200) { + assert.ifError(err); + } else { + assert.ok(err); + assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.http_status, 429); + assert.equal(err.type, 'limit'); + assert.equal(err.subtype, 'rate-limit'); + } + + if (done) { + setTimeout(done, 1000); + } + }); +} + +describe('rate limit', function() { + before(function() { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; + + redisClient = redis.createClient(global.environment.redis.port); + testClient = new TestClient(createMapConfig(), 1234); + }); + + after(function() { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false; + }); + + afterEach(function(done) { + keysToDelete.forEach( key => { + redisClient.del(key); + }); + + redisClient.SELECT(0, () => { + redisClient.del('user:localhost:mapviews:global'); + + redisClient.SELECT(5, () => { + redisClient.del('user:localhost:mapviews:global'); + done(); + }); + }); + }); + + it('should not be rate limited', function (done) { + const count = 1; + const period = 1; + const burst = 1; + setLimit(count, period, burst); + + assertGetLayergroupRequest(200, '2', '1', '1', '-1', done); + }); + + it("1 req/sec: 2 req/seg should be limited", function(done) { + const count = 1; + const period = 1; + const burst = 1; + setLimit(count, period, burst); + + assertGetLayergroupRequest(200, '2', '1', '1', '-1'); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050); + }); + +}); + + +describe('rate limit middleware', function () { + before(function (done) { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; + + const redisPool = new RedisPool(global.environment.redis); + const metadataBackend = cartodbRedis({ pool: redisPool }); + userLimitsApi = new UserLimitsApi(metadataBackend, { + limits: { + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS); + + redisClient = redis.createClient(global.environment.redis.port); + testClient = new TestClient(createMapConfig(), 1234); + + + const count = 1; + const period = 1; + const burst = 0; + setLimit(count, period, burst); + + setTimeout(done, 1000); + }); + + after(function () { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false; + + keysToDelete.forEach(key => { + redisClient.del(key); + }); + }); + + it("1 req/sec: 2 req/seg should be limited", function (done) { + assertRateLimitRequest(200, 1, 0, 1, -1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); + }); + + it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { + userLimitsApi.metadataBackend.redisCmd( + 8, + 'SCRIPT', + ['FLUSH'], + function () { + assertRateLimitRequest(200, 1, 0, 1, -1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); + } + ); + }); +}); diff --git a/test/fixtures/analysis/basic-source-id-mapnik-layer.png b/test/fixtures/analysis/basic-source-id-mapnik-layer.png index bdb315a9..41ee8fd5 100644 Binary files a/test/fixtures/analysis/basic-source-id-mapnik-layer.png and b/test/fixtures/analysis/basic-source-id-mapnik-layer.png differ diff --git a/yarn.lock b/yarn.lock index ea64f336..405e89b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,20 +2,20 @@ # yarn lockfile v1 -"@carto/mapnik@3.6.2-carto.2", "@carto/mapnik@~3.6.2-carto.0": - version "3.6.2-carto.2" - resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.2.tgz#45a055fd2d39530a873ef9ce5a325baacc81c196" +"@carto/mapnik@3.6.2-carto.4": + version "3.6.2-carto.4" + resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.4.tgz#54042a5dbea293c54e1bd286b32277694c5dc2d2" dependencies: mapnik-vector-tile "1.5.0" nan "~2.7.0" node-pre-gyp "~0.6.30" protozero "1.5.1" -"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb3": - version "2.5.1-cdb3" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/e61c7752c033595a273dcd1d4b267252b174bd28" +"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb4": + version "2.5.1-cdb4" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3eb554e5109199f50f457cec72ee288cffa5d6b3" dependencies: - "@carto/mapnik" "3.6.2-carto.2" + "@carto/mapnik" "3.6.2-carto.4" "@mapbox/sphericalmercator" "~1.0.1" mapnik-pool "~0.1.3" @@ -23,11 +23,11 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -abaculus@cartodb/abaculus#2.0.3-cdb2: - version "2.0.3-cdb2" - resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc" +abaculus@cartodb/abaculus#2.0.3-cdb5: + version "2.0.3-cdb5" + resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/b899cbea04b3e6093aa3ef32331920acd5f839a1" dependencies: - "@carto/mapnik" "~3.6.2-carto.0" + "@carto/mapnik" "3.6.2-carto.4" d3-queue "^2.0.2" sphericalmercator "1.0.x" @@ -238,9 +238,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.61.4: - version "0.61.4" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.4.tgz#eeab7e69a07d6fbc39ebc01ad19acf979de8fb2a" +camshaft@0.61.8: + version "0.61.8" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.8.tgz#75669c6c14791a93433e79a8892298e88cb0fce2" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -301,9 +301,9 @@ cartodb-query-tables@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756" -cartodb-redis@0.16.0: - version "0.16.0" - resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.16.0.tgz#969312fd329b24a76bf6e5a4dd961445f2eda734" +cartodb-redis@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-1.0.0.tgz#83b4888ba7abb5d5895c8958b7e15cf4882602aa" dependencies: dot "~1.0.2" redis-mpool "^0.5.0" @@ -2239,11 +2239,11 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb7: - version "0.6.18-cdb7" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/488d2acd65c89cc5382d996eabe8dc1f5051ce0f" +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb8: + version "0.6.18-cdb8" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/9cb4546c8fdd34ced0a41dbf70e143475b4e2067" dependencies: - "@carto/mapnik" "3.6.2-carto.2" + "@carto/mapnik" "3.6.2-carto.4" generic-pool "~2.4.0" mime "~1.6.0" sphericalmercator "~1.0.4" @@ -2400,13 +2400,13 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.5.7: - version "4.5.7" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.7.tgz#1cb2950e529d0a7f8bb55d56f4f568152f05c27e" +windshaft@4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.6.0.tgz#d9394aff73c0aa761207ad2b0f12d1c23ac41244" dependencies: - "@carto/mapnik" "3.6.2-carto.2" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb3 - abaculus cartodb/abaculus#2.0.3-cdb2 + "@carto/mapnik" "3.6.2-carto.4" + "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb4 + abaculus cartodb/abaculus#2.0.3-cdb5 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 cartodb-psql "^0.10.1" @@ -2420,7 +2420,7 @@ windshaft@4.5.7: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb7 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb8 torque.js "~2.11.0" underscore "~1.6.0"