diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 70046ffb..e28c51e6 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -149,7 +149,9 @@ var config = { } ,varnish: { host: 'localhost', - port: 6082, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 0d3a161c..6c4ea756 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -143,7 +143,9 @@ var config = { } ,varnish: { host: 'localhost', - port: 6082, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 4e2cb673..f8da00db 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -143,7 +143,9 @@ var config = { } ,varnish: { host: 'localhost', - port: 6082, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 73e522bd..5866e6c5 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -145,7 +145,9 @@ var config = { } ,varnish: { host: '', - port: null, + port: null, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/lib/cartodb/cache/backend/varnish_http.js b/lib/cartodb/cache/backend/varnish_http.js new file mode 100644 index 00000000..f9af6f6a --- /dev/null +++ b/lib/cartodb/cache/backend/varnish_http.js @@ -0,0 +1,32 @@ +var request = require('request'); + +function VarnishHttpCacheBackend(host, port) { + this.host = host; + this.port = port; +} + +module.exports = VarnishHttpCacheBackend; + +/** + * @param cacheObject should respond to `key() -> String` method + * @param {Function} callback + */ +VarnishHttpCacheBackend.prototype.invalidate = function(cacheObject, callback) { + request( + { + method: 'PURGE', + url: 'http://' + this.host + ':' + this.port + '/key', + headers: { + 'Invalidation-Match': '\\b' + cacheObject.key() + '\\b' + } + }, + function(err, response) { + if (err || response.statusCode !== 204) { + return callback(new Error('Unable to invalidate Varnish object')); + } + return callback(null); + } + ); +}; + +module.exports = VarnishHttpCacheBackend; \ No newline at end of file diff --git a/lib/cartodb/cache/model/named_maps_entry.js b/lib/cartodb/cache/model/named_maps_entry.js new file mode 100644 index 00000000..aaab7f3a --- /dev/null +++ b/lib/cartodb/cache/model/named_maps_entry.js @@ -0,0 +1,18 @@ +var crypto = require('crypto'); + +function NamedMaps(owner, name) { + this.namespace = 'n'; + this.owner = owner; + this.name = name; +} + +module.exports = NamedMaps; + + +NamedMaps.prototype.key = function() { + return this.namespace + ':' + shortHashKey(this.owner + ':' + this.name); +}; + +function shortHashKey(target) { + return crypto.createHash('sha256').update(target).digest('base64').substring(0,6); +} diff --git a/lib/cartodb/cache/surrogate_keys_cache.js b/lib/cartodb/cache/surrogate_keys_cache.js new file mode 100644 index 00000000..53b9f45a --- /dev/null +++ b/lib/cartodb/cache/surrogate_keys_cache.js @@ -0,0 +1,26 @@ +/** + * @param cacheBackend should respond to `invalidate(cacheObject, callback)` method + * @constructor + */ +function SurrogateKeysCache(cacheBackend) { + this.cacheBackend = cacheBackend; +} + +module.exports = SurrogateKeysCache; + + +/** + * @param response should respond to `header(key, value)` method + * @param cacheObject should respond to `key() -> String` method + */ +SurrogateKeysCache.prototype.tag = function(response, cacheObject) { + response.header('Surrogate-Key', cacheObject.key()); +}; + +/** + * @param cacheObject should respond to `key() -> String` method + * @param {Function} callback + */ +SurrogateKeysCache.prototype.invalidate = function(cacheObject, callback) { + this.cacheBackend.invalidate(cacheObject, callback); +}; diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 46a855f7..55e59917 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -54,6 +54,26 @@ var CartodbWindshaft = function(serverOptions) { var templateMaps = new TemplateMaps(redisPool, templateMapsOpts); serverOptions.templateMaps = templateMaps; + var SurrogateKeysCache = require('./cache/surrogate_keys_cache'), + NamedMapsCacheEntry = require('./cache/model/named_maps_entry'), + VarnishHttpCacheBackend = require('./cache/backend/varnish_http'), + varnishHttpCacheBackend = new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port), + surrogateKeysCache = new SurrogateKeysCache(varnishHttpCacheBackend); + + if (serverOptions.varnish_purge_enabled) { + function invalidateNamedMap(owner, templateName) { + surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { + if (err) { + console.warn('Cache: surrogate key invalidation failed'); + } + }); + } + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, invalidateNamedMap); + }); + } + // boot var ws = new Windshaft.Server(serverOptions); @@ -138,7 +158,7 @@ var CartodbWindshaft = function(serverOptions) { var TemplateMapsController = require('./controllers/template_maps'), templateMapsController = new TemplateMapsController( - ws, serverOptions, templateMaps, cartoData, template_baseurl + ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry ); templateMapsController.register(ws); diff --git a/lib/cartodb/controllers/template_maps.js b/lib/cartodb/controllers/template_maps.js index 3ee87bf9..db525c7c 100644 --- a/lib/cartodb/controllers/template_maps.js +++ b/lib/cartodb/controllers/template_maps.js @@ -1,12 +1,15 @@ var Step = require('step'); var _ = require('underscore'); -function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl) { +function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache, + NamedMapsCacheEntry) { this.app = app; this.serverOptions = serverOptions; this.templateMaps = templateMaps; this.metadataBackend = metadataBackend; this.templateBaseUrl = templateBaseUrl; + this.surrogateKeysCache = surrogateKeysCache; + this.NamedMapsCacheEntry = NamedMapsCacheEntry; } module.exports = TemplateMapsController; @@ -430,6 +433,9 @@ TemplateMapsController.prototype.instantiateTemplate = function(req, res, templa } var tplhash = self.templateMaps.fingerPrint(template).substring(0,8); layergroup.layergroupid = cdbuser + '@' + tplhash + '@' + layergroup.layergroupid; + + self.surrogateKeysCache.tag(res, new self.NamedMapsCacheEntry(cdbuser, template.name)); + return layergroup; }, callback diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index d5d16454..a731535b 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -89,7 +89,9 @@ module.exports = function(redisPool) { enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, varnish_port: global.environment.varnish.port, + varnish_http_port: global.environment.varnish.http_port, varnish_secret: global.environment.varnish.secret, + varnish_purge_enabled: global.environment.varnish.purge_enabled, cache_enabled: global.environment.cache_enabled, log_format: global.environment.log_format, useProfiler: global.environment.useProfiler diff --git a/lib/cartodb/template_maps.js b/lib/cartodb/template_maps.js index 419739ce..233b1a49 100644 --- a/lib/cartodb/template_maps.js +++ b/lib/cartodb/template_maps.js @@ -3,6 +3,11 @@ var crypto = require('crypto'), _ = require('underscore'), dot = require('dot'); + +var EventEmitter = require('events').EventEmitter; +var util = require('util'); + + // Class handling map templates // // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps @@ -16,6 +21,10 @@ var crypto = require('crypto'), // // function TemplateMaps(redis_pool, opts) { + if (!(this instanceof TemplateMaps)) return new TemplateMaps(); + + EventEmitter.call(this); + this.redis_pool = redis_pool; this.opts = opts || {}; @@ -42,6 +51,11 @@ function TemplateMaps(redis_pool, opts) { this.lock_ttl = this.opts['lock_ttl'] || 5000; } +util.inherits(TemplateMaps, EventEmitter); + +module.exports = TemplateMaps; + + var o = TemplateMaps.prototype; //--------------- PRIVATE METHODS -------------------------------- @@ -193,6 +207,10 @@ o.addTemplate = function(owner, template, callback) { return true; }, function finish(err) { + if (!err) { + self.emit('add', owner, templateName, template); + } + callback(err, templateName, template); } ); @@ -223,6 +241,10 @@ o.delTemplate = function(owner, tpl_id, callback) { return true; }, function finish(err) { + if (!err) { + self.emit('delete', owner, tpl_id); + } + callback(err); } ); @@ -286,6 +308,10 @@ o.updTemplate = function(owner, tpl_id, template, callback) { return true; }, function finish(err) { + if (!err) { + self.emit('update', owner, templateName, template); + } + callback(err, template); } ); @@ -438,5 +464,3 @@ o.fingerPrint = function(template) { .digest('hex') ; }; - -module.exports = TemplateMaps; diff --git a/test/acceptance/cache/surrogate_keys_invalidation.js b/test/acceptance/cache/surrogate_keys_invalidation.js new file mode 100644 index 00000000..c4bb3818 --- /dev/null +++ b/test/acceptance/cache/surrogate_keys_invalidation.js @@ -0,0 +1,211 @@ +var assert = require('../../support/assert'); +var redis = require('redis'); +var Step = require('step'); + +var helper = require(__dirname + '/../../support/test_helper'); + +var SqlApiEmulator = require(__dirname + '/../../support/SQLAPIEmu.js'); + +var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry'); +var SurrogateKeysCache = require(__dirname + '/../../../lib/cartodb/cache/surrogate_keys_cache'); + +var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/cartodb_windshaft'); +var ServerOptions = require(__dirname + '/../../../lib/cartodb/server_options'); +var serverOptions = ServerOptions(); + + +suite('templates surrogate keys', function() { + + var redisClient, + sqlApiServer, + server; + + var templateOwner = 'localhost', + templateName = 'acceptance', + expectedTemplateId = templateOwner + '@' + templateName, + template = { + version: '0.0.1', + name: templateName, + auth: { + method: 'open' + }, + layergroup: { + version: '1.2.0', + layers: [ + { + options: { + sql: 'select 1 cartodb_id, null::geometry as the_geom_webmercator', + cartocss: '#layer { marker-fill:blue; }', + cartocss_version: '2.3.0' + } + } + ] + } + }, + expectedBody = { template_id: expectedTemplateId }; + + suiteSetup(function(done) { + // Enable Varnish purge for tests + serverOptions.varnish_purge_enabled = true; + + server = new CartodbWindshaft(serverOptions); + + sqlApiServer = new SqlApiEmulator(global.environment.sqlapi.port, done); + + redisClient = redis.createClient(global.environment.redis.port); + }); + + var surrogateKeysCacheInvalidateFn = SurrogateKeysCache.prototype.invalidate; + + beforeEach(function(done) { + var postTemplateRequest = { + url: '/tiles/template?api_key=1234', + method: 'POST', + headers: { + host: templateOwner, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template) + }; + + Step( + function postTemplate() { + var next = this; + assert.response(server, + postTemplateRequest, + { + status: 200 + }, + function(res) { + next(null, res); + } + ); + }, + function rePostTemplate(err, res) { + if (err) { + throw err; + } + var parsedBody = JSON.parse(res.body); + assert.deepEqual(parsedBody, expectedBody); + return true; + }, + function finish(err) { + done(err); + } + ); + }); + + test("update template calls surrogate keys invalidation", function(done) { + var cacheEntryKey; + var surrogateKeysCacheInvalidateMethodInvoked = false; + SurrogateKeysCache.prototype.invalidate = function(cacheEntry) { + cacheEntryKey = cacheEntry.key(); + surrogateKeysCacheInvalidateMethodInvoked = true; + }; + + Step( + function putValidTemplate() { + var updateTemplateRequest = { + url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234', + method: 'PUT', + headers: { + host: templateOwner, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template) + }; + var next = this; + assert.response(server, + updateTemplateRequest, + { + status: 200 + }, + function(res) { + next(null, res); + } + ); + }, + function checkValidUpdate(err, res) { + if (err) { + throw err; + } + var parsedBody = JSON.parse(res.body); + assert.deepEqual(parsedBody, expectedBody); + + assert.ok(surrogateKeysCacheInvalidateMethodInvoked); + assert.equal(cacheEntryKey, new NamedMapsCacheEntry(templateOwner, templateName).key()); + + return null; + }, + function finish(err) { + if ( err ) { + return done(err); + } + redisClient.keys("map_*|localhost", function(err, keys) { + if ( err ) { + return done(err); + } + redisClient.del(keys, function(err) { + return done(err); + }); + }); + } + ); + }); + + test("delete template calls surrogate keys invalidation", function(done) { + + var cacheEntryKey; + var surrogateKeysCacheInvalidateMethodInvoked = false; + SurrogateKeysCache.prototype.invalidate = function(cacheEntry) { + cacheEntryKey = cacheEntry.key(); + surrogateKeysCacheInvalidateMethodInvoked = true; + }; + + Step( + function putValidTemplate() { + var deleteTemplateRequest = { + url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234', + method: 'DELETE', + headers: { + host: templateOwner, + 'Content-Type': 'application/json' + } + }; + var next = this; + assert.response(server, + deleteTemplateRequest, + { + status: 204 + }, + function(res) { + next(null, res); + } + ); + }, + function checkValidUpdate(err) { + if (err) { + throw err; + } + + assert.ok(surrogateKeysCacheInvalidateMethodInvoked); + assert.equal(cacheEntryKey, new NamedMapsCacheEntry(templateOwner, templateName).key()); + + return null; + }, + function finish(err) { + done(err); + } + ); + }); + + afterEach(function(done) { + SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn; + done(); + }); + + suiteTeardown(function(done) { + sqlApiServer.close(done); + }); + +}); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index ced90a40..3a8c2f42 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -5,6 +5,7 @@ var querystring = require('querystring'); var semver = require('semver'); var Step = require('step'); var strftime = require('strftime'); +var NamedMapsCacheEntry = require(__dirname + '/../../lib/cartodb/cache/model/named_maps_entry'); var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); var redis_stats_db = 5; @@ -1590,6 +1591,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()); return null; }, function finish(err) { @@ -1605,10 +1607,9 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio version: '0.0.1', name: 'acceptance_open_jsonp_params', auth: { method: 'open' }, - /* placeholders: { color: { type: "css_color", default: "red" } - },*/ + }, layergroup: { version: '1.0.0', layers: [ @@ -1661,9 +1662,11 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio function checkInstanciation(err, res) { if ( err ) throw err; + console.log(err, res.body, res.headers); assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 - helper.checkNoCache(res); + helper.checkCache(res); + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()); return null; }, function finish(err) { @@ -1845,6 +1848,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio assert.ok(parsed.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from response body: " + res.body); layergroupid = parsed.layergroupid; + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance2.name).key()); return null; }, function updateTemplate(err, res) @@ -1889,6 +1893,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio assert.ok(parsed.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from response body: " + res.body); assert.ok(layergroupid != parsed.layergroupid); + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance2.name).key()); return null; }, function finish(err) { diff --git a/test/support/test_helper.js b/test/support/test_helper.js index c040b552..9b4f0631 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -50,10 +50,16 @@ function checkCache(res) { assert.ok(res.headers.hasOwnProperty('last-modified')); } +function checkSurrogateKey(res, expectedKey) { + assert.ok(res.headers.hasOwnProperty('surrogate-key')); + assert.equal(res.headers['surrogate-key'], expectedKey); +} + module.exports = { lzma_compress_to_base64: lzma_compress_to_base64, checkNoCache: checkNoCache, + checkSurrogateKey: checkSurrogateKey, checkCache: checkCache }; diff --git a/test/unit/cartodb/cache/model/named_maps_entry.test.js b/test/unit/cartodb/cache/model/named_maps_entry.test.js new file mode 100644 index 00000000..bd11b5ac --- /dev/null +++ b/test/unit/cartodb/cache/model/named_maps_entry.test.js @@ -0,0 +1,29 @@ +var assert = require('assert'); +var _ = require('underscore'); +var NamedMapsCacheEntry = require('../../../../../lib/cartodb/cache/model/named_maps_entry'); + +suite('cache named_maps_entry', function() { + + var namedMapOwner = 'foo', + namedMapName = 'wadus_name', + namedMapsCacheEntry = new NamedMapsCacheEntry(namedMapOwner, namedMapName), + entryKey = namedMapsCacheEntry.key(); + + test('key is a string', function() { + assert.ok(_.isString(entryKey)); + }); + + test('key is 8 chars length', function() { + assert.equal(entryKey.length, 8); + var entryKeyParts = entryKey.split(':'); + assert.equal(entryKeyParts.length, 2); + assert.equal(entryKeyParts[0], 'n'); + }); + + test('key is name spaced for named maps', function() { + var entryKeyParts = entryKey.split(':'); + assert.equal(entryKeyParts.length, 2); + assert.equal(entryKeyParts[0], 'n'); + }); + +});