From f9554ec76163353e9992fc83d2e0d2432b0251fd Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 10 Jul 2015 19:10:55 +0200 Subject: [PATCH] Re-enable render limits --- lib/cartodb/api/user_limits_api.js | 28 +++++++++++ lib/cartodb/controllers/layergroup.js | 17 +++++-- lib/cartodb/controllers/map.js | 18 +++++-- lib/cartodb/controllers/named_maps.js | 5 +- .../mapconfig/create_layergroup_provider.js | 48 +++++++++++++++++++ .../models/mapconfig/map_store_provider.js | 29 +++++++++-- .../models/mapconfig/named_map_provider.js | 15 ++++-- lib/cartodb/server.js | 16 +++++-- test/acceptance/limits.js | 2 +- test/support/prepare_db.sh | 4 +- .../cartodb/ported/windshaft_server.test.js | 2 +- 11 files changed, 160 insertions(+), 24 deletions(-) create mode 100644 lib/cartodb/api/user_limits_api.js create mode 100644 lib/cartodb/models/mapconfig/create_layergroup_provider.js diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/api/user_limits_api.js new file mode 100644 index 00000000..ae9cd46b --- /dev/null +++ b/lib/cartodb/api/user_limits_api.js @@ -0,0 +1,28 @@ +/** + * + * @param metadataBackend + * @param options + * @constructor + * @type {UserLimitsApi} + */ +function UserLimitsApi(metadataBackend, options) { + this.metadataBackend = metadataBackend; + this.options = options || {}; + this.options.limits = this.options.limits || {}; +} + +module.exports = UserLimitsApi; + +UserLimitsApi.prototype.getRenderLimits = function (username, callback) { + var self = this; + this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) { + if (err) { + return callback(err); + } + + return callback(null, { + cacheOnTimeout: self.options.limits.cacheOnTimeout || false, + render: renderLimit || self.options.limits.render || 0 + }); + }); +}; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index c81f7af5..b20b0ebc 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -11,14 +11,16 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider' * @param {TileBackend} tileBackend * @param {PreviewBackend} previewBackend * @param {AttributesBackend} attributesBackend + * @param {{UserLimitsApi}} userLimitsApi * @constructor */ -function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend) { +function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi) { this.app = app; this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.attributesBackend = attributesBackend; + this.userLimitsApi = userLimitsApi; } module.exports = LayergroupController; @@ -85,6 +87,8 @@ LayergroupController.prototype.layer = function(req, res, next) { LayergroupController.prototype.tileOrLayer = function (req, res) { var self = this; + console.log(req.context.user); + step( function mapController$prepareParams() { self.app.req2params(req, this); @@ -94,7 +98,10 @@ LayergroupController.prototype.tileOrLayer = function (req, res) { if ( err ) { throw err; } - self.tileBackend.getTile(new MapStoreMapConfigProvider(self.mapStore, req.params), req.params, this); + self.tileBackend.getTile( + new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params), + req.params, this + ); }, function mapController$finalize(err, tile, headers, stats) { req.profiler.add(stats); @@ -179,10 +186,12 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo req.profiler.done('req2params'); assert.ifError(err); if (center) { - self.previewBackend.getImage(new MapStoreMapConfigProvider(self.mapStore, req.params), + self.previewBackend.getImage( + new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params), format, width, height, zoom, center, this); } else { - self.previewBackend.getImage(new MapStoreMapConfigProvider(self.mapStore, req.params), + self.previewBackend.getImage( + new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params), format, width, height, zoom /* bounds */, this); } }, diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1ad2f3c9..d5b43425 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -12,6 +12,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); +var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); /** * @param app @@ -21,10 +22,11 @@ var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider' * @param metadataBackend * @param {QueryTablesApi} queryTablesApi * @param {SurrogateKeysCache} surrogateKeysCache + * @param {{UserLimitsApi}} userLimitsApi * @constructor */ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi, - surrogateKeysCache) { + surrogateKeysCache, userLimitsApi) { this.app = app; this.pgConnection = pgConnection; this.templateMaps = templateMaps; @@ -32,6 +34,7 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack this.metadataBackend = metadataBackend; this.queryTablesApi = queryTablesApi; this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); } @@ -136,7 +139,11 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { function createLayergroup(err, requestMapConfig, datasource) { assert.ifError(err); mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource()); - self.mapBackend.createLayergroup(mapConfig, req.params, this); + self.mapBackend.createLayergroup( + mapConfig, req.params, + new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params), + this + ); }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); @@ -170,6 +177,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn mapConfigProvider = new NamedMapMapConfigProvider( self.templateMaps, self.pgConnection, + self.userLimitsApi, cdbuser, req.params.template_id, templateParams, @@ -181,7 +189,11 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn function createLayergroup(err, mapConfig_, rendererParams/*, context*/) { assert.ifError(err); mapConfig = mapConfig_; - self.mapBackend.createLayergroup(mapConfig, rendererParams, this); + self.mapBackend.createLayergroup( + mapConfig, rendererParams, + new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams), + this + ); }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 2ef6766c..7e414e19 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -7,7 +7,7 @@ var cors = require('../middleware/cors'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previewBackend, surrogateKeysCache, - tablesExtentApi) { + tablesExtentApi, userLimitsApi) { this.app = app; this.pgConnection = pgConnection; this.templateMaps = templateMaps; @@ -15,6 +15,7 @@ function NamedMapsController(app, pgConnection, templateMaps, tileBackend, previ this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; + this.userLimitsApi = userLimitsApi; } module.exports = NamedMapsController; @@ -40,6 +41,7 @@ NamedMapsController.prototype.tile = function(req, res) { namedMapProvider = new NamedMapMapConfigProvider( self.templateMaps, self.pgConnection, + self.userLimitsApi, cdbUser, req.params.template_id, req.query.config, @@ -86,6 +88,7 @@ NamedMapsController.prototype.staticMap = function(req, res) { namedMapProvider = new NamedMapMapConfigProvider( self.templateMaps, self.pgConnection, + self.userLimitsApi, cdbUser, req.params.template_id, req.query.config, diff --git a/lib/cartodb/models/mapconfig/create_layergroup_provider.js b/lib/cartodb/models/mapconfig/create_layergroup_provider.js new file mode 100644 index 00000000..409d9bda --- /dev/null +++ b/lib/cartodb/models/mapconfig/create_layergroup_provider.js @@ -0,0 +1,48 @@ +var assert = require('assert'); +var step = require('step'); + +var MapStoreMapConfigProvider = require('./map_store_provider'); + +/** + * @param {MapConfig} mapConfig + * @param {String} user + * @param {UserLimitsApi} userLimitsApi + * @param {Object} params + * @constructor + * @type {CreateLayergroupMapConfigProvider} + */ +function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) { + this.mapConfig = mapConfig; + this.user = user; + this.userLimitsApi = userLimitsApi; + this.params = params; + this.cacheBuster = params.cache_buster || 0; +} + +module.exports = CreateLayergroupMapConfigProvider; + +CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { + var self = this; + var context = {}; + step( + function prepareContextLimits() { + self.userLimitsApi.getRenderLimits(self.user, this); + }, + function handleRenderLimits(err, renderLimits) { + assert.ifError(err); + context.limits = renderLimits; + return null; + }, + function finish(err) { + return callback(err, self.mapConfig, self.params, context); + } + ); +}; + +CreateLayergroupMapConfigProvider.prototype.getKey = MapStoreMapConfigProvider.prototype.getKey; + +CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigProvider.prototype.getCacheBuster; + +CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter; + +CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; diff --git a/lib/cartodb/models/mapconfig/map_store_provider.js b/lib/cartodb/models/mapconfig/map_store_provider.js index e970614f..c07f9abc 100644 --- a/lib/cartodb/models/mapconfig/map_store_provider.js +++ b/lib/cartodb/models/mapconfig/map_store_provider.js @@ -1,14 +1,20 @@ var _ = require('underscore'); +var assert = require('assert'); var dot = require('dot'); +var step = require('step'); /** * @param {MapStore} mapStore + * @param {String} user + * @param {UserLimitsApi} userLimitsApi * @param {Object} params * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) { this.mapStore = mapStore; + this.user = user; + this.userLimitsApi = userLimitsApi; this.params = params; this.token = params.token; this.cacheBuster = params.cache_buster || 0; @@ -18,9 +24,24 @@ module.exports = MapStoreMapConfigProvider; MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; - this.mapStore.load(this.token, function(err, mapConfig) { - return callback(err, mapConfig, self.params, {}); - }); + var context = {}; + step( + function prepareContextLimits() { + self.userLimitsApi.getRenderLimits(self.user, this); + }, + function handleRenderLimits(err, renderLimits) { + assert.ifError(err); + context.limits = renderLimits; + return null; + }, + function loadMapConfig(err) { + assert.ifError(err); + self.mapStore.load(self.token, this); + }, + function finish(err, mapConfig) { + return callback(err, mapConfig, self.params, context); + } + ); }; MapStoreMapConfigProvider.prototype.getKey = function() { diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index ece76731..3ee5b18e 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -10,9 +10,11 @@ var templateName = require('../../backends/template_maps').templateName; * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, owner, templateId, config, authToken, params) { +function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, owner, templateId, config, authToken, + params) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; + this.userLimitsApi = userLimitsApi; this.owner = owner; this.templateName = templateName(templateId); this.config = config; @@ -26,7 +28,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, owner, templateId this.err = null; this.mapConfig = null; this.rendererParams = null; - this.context = null; + this.context = {}; } module.exports = NamedMapMapConfigProvider; @@ -40,7 +42,6 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { var mapConfig = null; var rendererParams; - var context = {}; step( function getTemplate() { @@ -74,11 +75,15 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { }); self.setDBParams(self.owner, rendererParams, this); }, - function cacheAndReturnMapConfig(err) { + function prepareContextLimits(err) { + assert.ifError(err); + self.userLimitsApi.getRenderLimits(self.owner, this); + }, + function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; self.mapConfig = (mapConfig === null) ? null : MapConfig.create(mapConfig); self.rendererParams = rendererParams; - self.context = context; + self.context.limits = renderLimits || {}; return callback(self.err, self.mapConfig, self.rendererParams, self.context); } ); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index c6b03d2d..ffd71219 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -18,6 +18,7 @@ var mapnik = windshaft.mapnik; var TemplateMaps = require('./backends/template_maps.js'); var QueryTablesApi = require('./api/query_tables_api'); +var UserLimitsApi = require('./api/user_limits_api'); var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); @@ -61,6 +62,12 @@ module.exports = function(serverOptions) { var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); var queryTablesApi = new QueryTablesApi(pgQueryRunner); + var userLimitsApi = new UserLimitsApi(metadataBackend, { + limits: { + cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, + render: serverOptions.renderer.mapnik.limits.render || 0 + } + }); var templateMaps = new TemplateMaps(redisPool, { max_user_templates: global.environment.maxUserTemplates @@ -187,7 +194,8 @@ module.exports = function(serverOptions) { mapStore, tileBackend, previewBackend, - attributesBackend + attributesBackend, + userLimitsApi ).register(app); new controller.Map( @@ -197,7 +205,8 @@ module.exports = function(serverOptions) { mapBackend, metadataBackend, queryTablesApi, - surrogateKeysCache + surrogateKeysCache, + userLimitsApi ).register(app); new controller.NamedMaps( @@ -207,7 +216,8 @@ module.exports = function(serverOptions) { tileBackend, previewBackend, surrogateKeysCache, - tablesExtentApi + tablesExtentApi, + userLimitsApi ).register(app); new controller.NamedMapsAdmin(app, templateMaps).register(app); diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index 707b5e55..4d5a1233 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -7,7 +7,7 @@ var redis = require('redis'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -describe.skip('render limits', function() { +describe('render limits', function() { var layergroupUrl = '/api/v1/map'; diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 9369be3f..fd920e8d 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -79,8 +79,8 @@ if test x"$PREPARE_PGSQL" = xyes; then psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql +# curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql +# curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 diff --git a/test/unit/cartodb/ported/windshaft_server.test.js b/test/unit/cartodb/ported/windshaft_server.test.js index 0e70e27b..9685e71e 100644 --- a/test/unit/cartodb/ported/windshaft_server.test.js +++ b/test/unit/cartodb/ported/windshaft_server.test.js @@ -30,7 +30,7 @@ describe('windshaft', function() { function(){ var ws = cartodbServer({unbuffered_logging:true}); ws.listen(); - }, /Cannot read property 'mapnik_version' of undefined/ + }, /Cannot read property 'mapnik' of undefined/ ); });