diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b20b0ebc..68a23fe2 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -11,16 +11,21 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider' * @param {TileBackend} tileBackend * @param {PreviewBackend} previewBackend * @param {AttributesBackend} attributesBackend - * @param {{UserLimitsApi}} userLimitsApi + * @param {UserLimitsApi} userLimitsApi + * @param {QueryTablesApi} queryTablesApi * @constructor */ -function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi) { +function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi, + queryTablesApi) { this.app = app; this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.attributesBackend = attributesBackend; this.userLimitsApi = userLimitsApi; + this.queryTablesApi = queryTablesApi; + + this.channelCache = {}; } module.exports = LayergroupController; @@ -62,7 +67,7 @@ LayergroupController.prototype.attributes = function(req, res) { var statusCode = self.app.findStatusCode(err); self.app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err); } else { - self.app.sendResponse(res, [tile, 200]); + self.sendResponse(req, res, [tile, 200]); } } ); @@ -149,7 +154,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.app.sendWithHeaders(res, tile, 200, headers); + this.sendResponse(req, res, [tile, headers, 200]); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } @@ -206,8 +211,120 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo self.app.sendError(res, {errors: ['' + err] }, self.app.findStatusCode(err), 'STATIC_MAP', err); } else { res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format); - self.app.sendResponse(res, [image, 200]); + self.sendResponse(req, res, [image, 200]); } } ); }; + +LayergroupController.prototype.sendResponse = function(req, res, args) { + var self = this; + + res.header('Cache-Control', 'public,max-age=31536000'); + + // Set Last-Modified header + var lastUpdated; + if (req.params.cache_buster) { + // Assuming cache_buster is a timestamp + lastUpdated = new Date(parseInt(req.params.cache_buster)); + } else { + lastUpdated = new Date(); + } + res.header('Last-Modified', lastUpdated.toUTCString()); + + step( + function getCacheChannel() { + self.cacheChannel(req, this); + }, + function sendResponse(err, cacheChannel) { + if (err) { + console.log('ERROR generating cache channel: ' + err); + } + if (!!cacheChannel) { + res.header('X-Cache-Channel', cacheChannel); + } + self.app.sendResponse(res, args); + } + ); + +}; + +LayergroupController.prototype.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames.join(','); +}; + +LayergroupController.prototype.cacheChannel = function(req, callback) { + if (req.profiler) { + req.profiler.start('addCacheChannel'); + } + + var dbName = req.params.dbname; + + // no token means no tables associated + if (!req.params.token) { + return callback(null, this.buildCacheChannel(dbName, [])); + } + + var self = this; + + var cacheKey = [ dbName, req.params.token ].join(':'); + + step( + function checkCached() { + if ( self.channelCache.hasOwnProperty(cacheKey) ) { + return callback(null, self.channelCache[cacheKey]); + } + return null; + }, + function extractSQL(err) { + assert.ifError(err); + + step( + function loadFromStore() { + self.mapStore.load(req.params.token, this); + }, + function getSQL(err, mapConfig) { + if (req.profiler) { + req.profiler.done('mapStore_load'); + } + assert.ifError(err); + + var queries = mapConfig.getLayers() + .map(function(lyr) { + return lyr.options.sql; + }) + .filter(function(sql) { + return !!sql; + }); + + return queries.length ? queries.join(';') : null; + }, + this + ); + }, + function findAffectedTables(err, sql) { + assert.ifError(err); + + if ( ! sql ) { + throw new Error("this request doesn't need an X-Cache-Channel generated"); + } + + self.queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel + }, + function buildCacheChannel(err, tableNames) { + assert.ifError(err); + + if (req.profiler) { + req.profiler.done('affectedTables'); + } + + var cacheChannel = self.buildCacheChannel(dbName, tableNames); + self.channelCache[cacheKey] = cacheChannel; + + return cacheChannel; + }, + function finish(err, cacheChannel) { + callback(err, cacheChannel); + } + ); +}; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 9e6fcf44..bc558b8a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -37,6 +37,8 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + + this.channelCache = {}; } module.exports = MapController; @@ -274,8 +276,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la req.profiler.done('queryTablesAndLastUpdated'); } assert.ifError(err); - var cacheChannel = self.app.buildCacheChannel(dbName, result.affectedTables); - self.app.channelCache[cacheKey] = cacheChannel; + var cacheChannel = self.buildCacheChannel(dbName, result.affectedTables); + self.channelCache[cacheKey] = cacheChannel; // last update for layergroup cache buster layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; @@ -298,3 +300,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la } ); }; + +MapController.prototype.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames.join(','); +}; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 0e0ce057..d0245f8f 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -63,7 +63,7 @@ NamedMapsController.prototype.tile = function(req, res) { self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, namedMapProvider.getTemplateName())); res.setHeader('Content-Type', headers['Content-Type']); res.setHeader('Cache-Control', 'public,max-age=7200,must-revalidate'); - self.app.sendWithHeaders(res, tile, 200, headers); + self.app.sendResponse(res, [tile, headers, 200]); } } ); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7ac8573b..132db032 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -74,12 +74,6 @@ module.exports = function(serverOptions) { max_user_templates: global.environment.maxUserTemplates }); - // This is for Templated maps - // - // "named" is the official, "template" is for backward compatibility up to 1.6.x - // - var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)'; - var surrogateKeysCacheBackends = []; if (serverOptions.varnish_purge_enabled) { @@ -197,7 +191,8 @@ module.exports = function(serverOptions) { tileBackend, previewBackend, attributesBackend, - userLimitsApi + userLimitsApi, + queryTablesApi ).register(app); new controller.Map( @@ -243,98 +238,33 @@ module.exports = function(serverOptions) { } }); - // GET routes for which we don't want to request any caching. - // POST/PUT/DELETE requests are never cached anyway. - var noCacheGETRoutes = [ - '/', - '/version', - // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 - serverOptions.base_url_mapconfig, - serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', - template_baseurl, - template_baseurl + '/:template_id', - template_baseurl + '/:template_id/jsonp' - ]; - app.sendResponse = function(res, args) { - var statusCode; - if ( res._windshaftStatusCode ) { - // Added by our override of sendError - statusCode = res._windshaftStatusCode; - } else { - if ( args.length > 2 ) statusCode = args[2]; - else { - statusCode = args[1] || 200; + var req = res.req; + + if (global.environment && global.environment.api_hostname) { + res.header('X-Served-By-Host', global.environment.api_hostname); + } + + if (req && req.params && req.params.dbhost) { + res.header('X-Served-By-DB-Host', req.params.dbhost); + } + + if ( req && req.profiler ) { + res.header('X-Tiler-Profiler', req.profiler.toJSONString()); + } + +// res.send(body|status[, headers|status[, status]]) + res.send.apply(res, args); + + if ( req && req.profiler ) { + try { + // May throw due to dns, see + // See http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + console.error("error sending profiling stats: " + err); } } - var req = res.req; - step ( - function addCacheChannel() { - if ( ! req ) { - // having no associated request can happen when - // using fake response objects for testing layergroup - // creation - return false; - } - if ( ! req.params ) { - // service requests (/version, /) - // have no need for an X-Cache-Channel - return false; - } - if ( statusCode != 200 ) { - // We do not want to cache - // unsuccessful responses - return false; - } - if ( _.contains(noCacheGETRoutes, req.route.path) ) { -//console.log("Skipping cache channel in route:\n" + req.route.path); - return false; - } -//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + -// mapCreateRoutes.join("\n")); - app.addCacheChannel(req, this); - }, - function sendResponse(err/*, added*/) { - if ( err ) console.log(err + err.stack); - // When using custom results from tryFetch* methods, - // there is no "req" link in the result object. - // In those cases we don't want to send stats now - // as they will be sent at the real end of request - var req = res.req; - - if (global.environment && global.environment.api_hostname) { - res.header('X-Served-By-Host', global.environment.api_hostname); - } - - if (req && req.params && req.params.dbhost) { - res.header('X-Served-By-DB-Host', req.params.dbhost); - } - - if ( req && req.profiler ) { - res.header('X-Tiler-Profiler', req.profiler.toJSONString()); - } - - res.send.apply(res, args); - - if ( req && req.profiler ) { - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - console.error("error sending profiling stats: " + err); - } - } - return null; - }, - function finish(err) { - if ( err ) console.log(err + err.stack); - } - ); - }; - - app.sendWithHeaders = function(res, what, status, headers) { - app.sendResponse(res, [what, headers, status]); }; app.sendError = function(res, err, statusCode, label, tolog) { @@ -490,136 +420,6 @@ module.exports = function(serverOptions) { ); }; - // TODO: review lifetime of elements of this cache - // NOTE: by-token indices should only be dropped when - // the corresponding layegroup is dropped, because - // we have no SQL after layer creation. - app.channelCache = {}; - - app.buildCacheChannel = function (dbName, tableNames){ - return dbName + ':' + tableNames.join(','); - }; - - app.generateCacheChannel = function(app, req, callback){ - // Build channelCache key - var dbName = req.params.dbname; - var cacheKey = [ dbName, req.params.token ].join(':'); - - // no token means no tables associated - if (!req.params.token) { - return callback(null, this.buildCacheChannel(dbName, [])); - } - - step( - function checkCached() { - if ( app.channelCache.hasOwnProperty(cacheKey) ) { - return callback(null, app.channelCache[cacheKey]); - } - return null; - }, - function extractSQL(err) { - assert.ifError(err); - - step( - function loadFromStore() { - mapStore.load(req.params.token, this); - }, - function getSQL(err, mapConfig) { - if (req.profiler) { - req.profiler.done('mapStore_load'); - } - assert.ifError(err); - - var queries = mapConfig.getLayers() - .map(function(lyr) { - return lyr.options.sql; - }) - .filter(function(sql) { - return !!sql; - }); - - return queries.length ? queries.join(';') : null; - }, - this - ); - }, - function findAffectedTables(err, sql) { - assert.ifError(err); - - if ( ! sql ) { - throw new Error("this request doesn't need an X-Cache-Channel generated"); - } - - queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel - }, - function buildCacheChannel(err, tableNames) { - assert.ifError(err); - - if (req.profiler) { - req.profiler.done('affectedTables'); - } - - var cacheChannel = app.buildCacheChannel(dbName,tableNames); - app.channelCache[cacheKey] = cacheChannel; - - return cacheChannel; - }, - function finish(err, cacheChannel) { - callback(err, cacheChannel); - } - ); - }; - - // Set the cache chanel info to invalidate the cache on the frontend server - // - // @param req The request object. - // The function will have no effect unless req.res exists. - // It is expected that req.params contains 'table' and 'dbname' - // - // @param cb function(err, channel) will be called when ready. - // the channel parameter will be null if nothing was added - // - app.addCacheChannel = function(req, cb) { - // skip non-GET requests, or requests for which there's no response - if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } - if (req.profiler) { - req.profiler.start('addCacheChannel'); - } - var res = req.res; - if ( req.params.token ) { - res.header('Cache-Control', 'public,max-age=31536000'); // 1 year - } else { - var ttl = global.environment.varnish.ttl || 86400; - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); - } - - // Set Last-Modified header - var lastUpdated; - if ( req.params.cache_buster ) { - // Assuming cache_buster is a timestamp - // FIXME: store lastModified in the cache channel instead - lastUpdated = new Date(parseInt(req.params.cache_buster)); - } else { - lastUpdated = new Date(); - } - res.header('Last-Modified', lastUpdated.toUTCString()); - - app.generateCacheChannel(app, req, function(err, channel){ - if (req.profiler) { - req.profiler.done('generateCacheChannel'); - req.profiler.end(); - } - if ( ! err ) { - res.header('X-Cache-Channel', channel); - cb(null, channel); - } else { - console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err )); - // TODO: evaluate if we should bubble up the error instead - cb(null, 'ERROR'); - } - }); - }; - return app; };