From f5a3b7773745f4cd3aceec5a6b5ab1f22e203a16 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 16:35:47 +0100 Subject: [PATCH 1/9] Make TemplateMaps to emit messages when adding/updating/deleting templates --- lib/cartodb/template_maps.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/template_maps.js b/lib/cartodb/template_maps.js index 419739ce..233b1a49 100644 --- a/lib/cartodb/template_maps.js +++ b/lib/cartodb/template_maps.js @@ -3,6 +3,11 @@ var crypto = require('crypto'), _ = require('underscore'), dot = require('dot'); + +var EventEmitter = require('events').EventEmitter; +var util = require('util'); + + // Class handling map templates // // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps @@ -16,6 +21,10 @@ var crypto = require('crypto'), // // function TemplateMaps(redis_pool, opts) { + if (!(this instanceof TemplateMaps)) return new TemplateMaps(); + + EventEmitter.call(this); + this.redis_pool = redis_pool; this.opts = opts || {}; @@ -42,6 +51,11 @@ function TemplateMaps(redis_pool, opts) { this.lock_ttl = this.opts['lock_ttl'] || 5000; } +util.inherits(TemplateMaps, EventEmitter); + +module.exports = TemplateMaps; + + var o = TemplateMaps.prototype; //--------------- PRIVATE METHODS -------------------------------- @@ -193,6 +207,10 @@ o.addTemplate = function(owner, template, callback) { return true; }, function finish(err) { + if (!err) { + self.emit('add', owner, templateName, template); + } + callback(err, templateName, template); } ); @@ -223,6 +241,10 @@ o.delTemplate = function(owner, tpl_id, callback) { return true; }, function finish(err) { + if (!err) { + self.emit('delete', owner, tpl_id); + } + callback(err); } ); @@ -286,6 +308,10 @@ o.updTemplate = function(owner, tpl_id, template, callback) { return true; }, function finish(err) { + if (!err) { + self.emit('update', owner, templateName, template); + } + callback(err, template); } ); @@ -438,5 +464,3 @@ o.fingerPrint = function(template) { .digest('hex') ; }; - -module.exports = TemplateMaps; From 885accdadf162de33f13db04d0977252b613cf9b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 16:36:45 +0100 Subject: [PATCH 2/9] Adds varnish http port to the default configurations --- config/environments/development.js.example | 1 + config/environments/production.js.example | 1 + config/environments/staging.js.example | 1 + config/environments/test.js.example | 1 + lib/cartodb/server_options.js | 1 + 5 files changed, 5 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 70046ffb..24daad9b 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -150,6 +150,7 @@ var config = { ,varnish: { host: 'localhost', port: 6082, + http_port: 6081, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 0d3a161c..8f36f66a 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -144,6 +144,7 @@ var config = { ,varnish: { host: 'localhost', port: 6082, + http_port: 6081, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 4e2cb673..f559d2a2 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -144,6 +144,7 @@ var config = { ,varnish: { host: 'localhost', port: 6082, + http_port: 6081, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 73e522bd..2c3a8629 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -146,6 +146,7 @@ var config = { ,varnish: { host: '', port: null, + http_port: 6081, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index d5d16454..45dcde1a 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -89,6 +89,7 @@ module.exports = function(redisPool) { enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, varnish_port: global.environment.varnish.port, + varnish_http_port: global.environment.varnish.http_port, varnish_secret: global.environment.varnish.secret, cache_enabled: global.environment.cache_enabled, log_format: global.environment.log_format, From 272e8cd221feb8ded516bae39ae48ea5544a268a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 16:37:38 +0100 Subject: [PATCH 3/9] Adds Surrogate Keys to named maps --- lib/cartodb/cache/backend/varnish_http.js | 32 +++++++++++++++++ lib/cartodb/cache/model/named_maps_entry.js | 18 ++++++++++ lib/cartodb/cache/surrogate_keys_cache.js | 26 ++++++++++++++ lib/cartodb/cartodb_windshaft.js | 40 +++++++++++++++------ lib/cartodb/controllers/template_maps.js | 8 ++++- 5 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 lib/cartodb/cache/backend/varnish_http.js create mode 100644 lib/cartodb/cache/model/named_maps_entry.js create mode 100644 lib/cartodb/cache/surrogate_keys_cache.js diff --git a/lib/cartodb/cache/backend/varnish_http.js b/lib/cartodb/cache/backend/varnish_http.js new file mode 100644 index 00000000..25ff2745 --- /dev/null +++ b/lib/cartodb/cache/backend/varnish_http.js @@ -0,0 +1,32 @@ +var request = require('request'); + +function VarnishHttpCacheBackend(host, port) { + this.host = host; + this.port = port; +} + +module.exports = VarnishHttpCacheBackend; + +/** + * @param cacheObject should respond to `key() -> String` method + * @param {Function} callback + */ +VarnishHttpCacheBackend.prototype.invalidate = function(cacheObject, callback) { + request( + { + method: 'PURGE', + url: 'http://' + this.host + ':' + this.port + '/key', + headers: { + 'Invalidation-Match': '\b' + cacheObject.key() + '\b' + } + }, + function(err, response) { + if (err || response.statusCode !== 204) { + return callback(new Error('Unable to invalidate Varnish object')); + } + return callback(null); + } + ); +}; + +module.exports = VarnishHttpCacheBackend; \ No newline at end of file diff --git a/lib/cartodb/cache/model/named_maps_entry.js b/lib/cartodb/cache/model/named_maps_entry.js new file mode 100644 index 00000000..aaab7f3a --- /dev/null +++ b/lib/cartodb/cache/model/named_maps_entry.js @@ -0,0 +1,18 @@ +var crypto = require('crypto'); + +function NamedMaps(owner, name) { + this.namespace = 'n'; + this.owner = owner; + this.name = name; +} + +module.exports = NamedMaps; + + +NamedMaps.prototype.key = function() { + return this.namespace + ':' + shortHashKey(this.owner + ':' + this.name); +}; + +function shortHashKey(target) { + return crypto.createHash('sha256').update(target).digest('base64').substring(0,6); +} diff --git a/lib/cartodb/cache/surrogate_keys_cache.js b/lib/cartodb/cache/surrogate_keys_cache.js new file mode 100644 index 00000000..53b9f45a --- /dev/null +++ b/lib/cartodb/cache/surrogate_keys_cache.js @@ -0,0 +1,26 @@ +/** + * @param cacheBackend should respond to `invalidate(cacheObject, callback)` method + * @constructor + */ +function SurrogateKeysCache(cacheBackend) { + this.cacheBackend = cacheBackend; +} + +module.exports = SurrogateKeysCache; + + +/** + * @param response should respond to `header(key, value)` method + * @param cacheObject should respond to `key() -> String` method + */ +SurrogateKeysCache.prototype.tag = function(response, cacheObject) { + response.header('Surrogate-Key', cacheObject.key()); +}; + +/** + * @param cacheObject should respond to `key() -> String` method + * @param {Function} callback + */ +SurrogateKeysCache.prototype.invalidate = function(cacheObject, callback) { + this.cacheBackend.invalidate(cacheObject, callback); +}; diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 46a855f7..0a42ff06 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -25,15 +25,6 @@ var CartodbWindshaft = function(serverOptions) { var cartoData = require('cartodb-redis')({pool: redisPool}); - if(serverOptions.cache_enabled) { - console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); - Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret); - serverOptions.afterStateChange = function(req, data, callback) { - Cache.invalidate_db(req.params.dbname, req.params.table); - callback(null, data); - } - } - serverOptions.beforeStateChange = function(req, callback) { var err = null; if ( ! req.params.hasOwnProperty('_authorizedByApiKey') ) { @@ -54,6 +45,35 @@ var CartodbWindshaft = function(serverOptions) { var templateMaps = new TemplateMaps(redisPool, templateMapsOpts); serverOptions.templateMaps = templateMaps; + var SurrogateKeysCache = require('./cache/surrogate_keys_cache'), + NamedMapsCacheEntry = require('./cache/model/named_maps_entry'), + VarnishHttpCacheBackend = require('./cache/backend/varnish_http'), + varnishHttpCacheBackend = new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port), + surrogateKeysCache = new SurrogateKeysCache(varnishHttpCacheBackend); + + if (serverOptions.cache_enabled) { + + console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); + + Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret); + serverOptions.afterStateChange = function(req, data, callback) { + Cache.invalidate_db(req.params.dbname, req.params.table); + callback(null, data); + }; + + function invalidateNamedMap(owner, templateName) { + surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { + if (err) { + console.warn('Cache: surrogate key invalidation failed'); + } + }); + } + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, invalidateNamedMap); + }); + } + // boot var ws = new Windshaft.Server(serverOptions); @@ -138,7 +158,7 @@ var CartodbWindshaft = function(serverOptions) { var TemplateMapsController = require('./controllers/template_maps'), templateMapsController = new TemplateMapsController( - ws, serverOptions, templateMaps, cartoData, template_baseurl + ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry ); templateMapsController.register(ws); diff --git a/lib/cartodb/controllers/template_maps.js b/lib/cartodb/controllers/template_maps.js index 3ee87bf9..db525c7c 100644 --- a/lib/cartodb/controllers/template_maps.js +++ b/lib/cartodb/controllers/template_maps.js @@ -1,12 +1,15 @@ var Step = require('step'); var _ = require('underscore'); -function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl) { +function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache, + NamedMapsCacheEntry) { this.app = app; this.serverOptions = serverOptions; this.templateMaps = templateMaps; this.metadataBackend = metadataBackend; this.templateBaseUrl = templateBaseUrl; + this.surrogateKeysCache = surrogateKeysCache; + this.NamedMapsCacheEntry = NamedMapsCacheEntry; } module.exports = TemplateMapsController; @@ -430,6 +433,9 @@ TemplateMapsController.prototype.instantiateTemplate = function(req, res, templa } var tplhash = self.templateMaps.fingerPrint(template).substring(0,8); layergroup.layergroupid = cdbuser + '@' + tplhash + '@' + layergroup.layergroupid; + + self.surrogateKeysCache.tag(res, new self.NamedMapsCacheEntry(cdbuser, template.name)); + return layergroup; }, callback From 0abd6a2293657f6ec1609d3abd83da08961ca3f2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 17:02:13 +0100 Subject: [PATCH 4/9] Adds check for surrogate key headers in template instances p.s. it fixes instantiate template with params test --- test/acceptance/templates.js | 11 ++++++++--- test/support/test_helper.js | 6 ++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index ced90a40..3a8c2f42 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -5,6 +5,7 @@ var querystring = require('querystring'); var semver = require('semver'); var Step = require('step'); var strftime = require('strftime'); +var NamedMapsCacheEntry = require(__dirname + '/../../lib/cartodb/cache/model/named_maps_entry'); var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); var redis_stats_db = 5; @@ -1590,6 +1591,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 helper.checkCache(res); + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()); return null; }, function finish(err) { @@ -1605,10 +1607,9 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio version: '0.0.1', name: 'acceptance_open_jsonp_params', auth: { method: 'open' }, - /* placeholders: { color: { type: "css_color", default: "red" } - },*/ + }, layergroup: { version: '1.0.0', layers: [ @@ -1661,9 +1662,11 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio function checkInstanciation(err, res) { if ( err ) throw err; + console.log(err, res.body, res.headers); assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 - helper.checkNoCache(res); + helper.checkCache(res); + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()); return null; }, function finish(err) { @@ -1845,6 +1848,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio assert.ok(parsed.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from response body: " + res.body); layergroupid = parsed.layergroupid; + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance2.name).key()); return null; }, function updateTemplate(err, res) @@ -1889,6 +1893,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio assert.ok(parsed.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from response body: " + res.body); assert.ok(layergroupid != parsed.layergroupid); + helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance2.name).key()); return null; }, function finish(err) { diff --git a/test/support/test_helper.js b/test/support/test_helper.js index c040b552..9b4f0631 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -50,10 +50,16 @@ function checkCache(res) { assert.ok(res.headers.hasOwnProperty('last-modified')); } +function checkSurrogateKey(res, expectedKey) { + assert.ok(res.headers.hasOwnProperty('surrogate-key')); + assert.equal(res.headers['surrogate-key'], expectedKey); +} + module.exports = { lzma_compress_to_base64: lzma_compress_to_base64, checkNoCache: checkNoCache, + checkSurrogateKey: checkSurrogateKey, checkCache: checkCache }; From fabb438cf0b9e92e16168944b70929a2a4c67d08 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 17:22:49 +0100 Subject: [PATCH 5/9] Escape \b for the regex --- lib/cartodb/cache/backend/varnish_http.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/cache/backend/varnish_http.js b/lib/cartodb/cache/backend/varnish_http.js index 25ff2745..f9af6f6a 100644 --- a/lib/cartodb/cache/backend/varnish_http.js +++ b/lib/cartodb/cache/backend/varnish_http.js @@ -17,7 +17,7 @@ VarnishHttpCacheBackend.prototype.invalidate = function(cacheObject, callback) { method: 'PURGE', url: 'http://' + this.host + ':' + this.port + '/key', headers: { - 'Invalidation-Match': '\b' + cacheObject.key() + '\b' + 'Invalidation-Match': '\\b' + cacheObject.key() + '\\b' } }, function(err, response) { From 6e7051814697ae8c815ea55cbf56c2f5b4999fc8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 17:46:16 +0100 Subject: [PATCH 6/9] Split between old cache_enabled and new purge_enabled configuration --- config/environments/development.js.example | 1 + config/environments/production.js.example | 1 + config/environments/staging.js.example | 1 + config/environments/test.js.example | 1 + lib/cartodb/cartodb_windshaft.js | 20 ++++++++++---------- lib/cartodb/server_options.js | 1 + 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 24daad9b..e249c479 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -151,6 +151,7 @@ var config = { host: 'localhost', port: 6082, http_port: 6081, + purge_enabled: false, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 8f36f66a..2b03478a 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -145,6 +145,7 @@ var config = { host: 'localhost', port: 6082, http_port: 6081, + purge_enabled: false, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index f559d2a2..2403f26d 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -145,6 +145,7 @@ var config = { host: 'localhost', port: 6082, http_port: 6081, + purge_enabled: false, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 2c3a8629..1c662c63 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -147,6 +147,7 @@ var config = { host: '', port: null, http_port: 6081, + purge_enabled: false, secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 0a42ff06..a43ae965 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -25,6 +25,15 @@ var CartodbWindshaft = function(serverOptions) { var cartoData = require('cartodb-redis')({pool: redisPool}); + if(serverOptions.cache_enabled) { + console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); + Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret); + serverOptions.afterStateChange = function(req, data, callback) { + Cache.invalidate_db(req.params.dbname, req.params.table); + callback(null, data); + }; + } + serverOptions.beforeStateChange = function(req, callback) { var err = null; if ( ! req.params.hasOwnProperty('_authorizedByApiKey') ) { @@ -51,16 +60,7 @@ var CartodbWindshaft = function(serverOptions) { varnishHttpCacheBackend = new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port), surrogateKeysCache = new SurrogateKeysCache(varnishHttpCacheBackend); - if (serverOptions.cache_enabled) { - - console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); - - Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret); - serverOptions.afterStateChange = function(req, data, callback) { - Cache.invalidate_db(req.params.dbname, req.params.table); - callback(null, data); - }; - + if (serverOptions.varnish_purge_enabled) { function invalidateNamedMap(owner, templateName) { surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { if (err) { diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 45dcde1a..a731535b 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -91,6 +91,7 @@ module.exports = function(redisPool) { varnish_port: global.environment.varnish.port, varnish_http_port: global.environment.varnish.http_port, varnish_secret: global.environment.varnish.secret, + varnish_purge_enabled: global.environment.varnish.purge_enabled, cache_enabled: global.environment.cache_enabled, log_format: global.environment.log_format, useProfiler: global.environment.useProfiler From 571a635feddad19b0c6cb484aec8d200cc3aa57a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 23 Jan 2015 17:46:58 +0100 Subject: [PATCH 7/9] Old style, avoid merge conflicts, missing history --- lib/cartodb/cartodb_windshaft.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index a43ae965..55e59917 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -31,7 +31,7 @@ var CartodbWindshaft = function(serverOptions) { serverOptions.afterStateChange = function(req, data, callback) { Cache.invalidate_db(req.params.dbname, req.params.table); callback(null, data); - }; + } } serverOptions.beforeStateChange = function(req, callback) { From 81f1b0dcf826ab82579599599a8c005ce0303a70 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 26 Jan 2015 15:02:28 +0100 Subject: [PATCH 8/9] Adds tests for named maps surrogate keys and for invalidation --- .../cache/surrogate_keys_invalidation.js | 211 ++++++++++++++++++ .../cache/model/named_maps_entry.test.js | 29 +++ 2 files changed, 240 insertions(+) create mode 100644 test/acceptance/cache/surrogate_keys_invalidation.js create mode 100644 test/unit/cartodb/cache/model/named_maps_entry.test.js diff --git a/test/acceptance/cache/surrogate_keys_invalidation.js b/test/acceptance/cache/surrogate_keys_invalidation.js new file mode 100644 index 00000000..c4bb3818 --- /dev/null +++ b/test/acceptance/cache/surrogate_keys_invalidation.js @@ -0,0 +1,211 @@ +var assert = require('../../support/assert'); +var redis = require('redis'); +var Step = require('step'); + +var helper = require(__dirname + '/../../support/test_helper'); + +var SqlApiEmulator = require(__dirname + '/../../support/SQLAPIEmu.js'); + +var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry'); +var SurrogateKeysCache = require(__dirname + '/../../../lib/cartodb/cache/surrogate_keys_cache'); + +var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/cartodb_windshaft'); +var ServerOptions = require(__dirname + '/../../../lib/cartodb/server_options'); +var serverOptions = ServerOptions(); + + +suite('templates surrogate keys', function() { + + var redisClient, + sqlApiServer, + server; + + var templateOwner = 'localhost', + templateName = 'acceptance', + expectedTemplateId = templateOwner + '@' + templateName, + template = { + version: '0.0.1', + name: templateName, + auth: { + method: 'open' + }, + layergroup: { + version: '1.2.0', + layers: [ + { + options: { + sql: 'select 1 cartodb_id, null::geometry as the_geom_webmercator', + cartocss: '#layer { marker-fill:blue; }', + cartocss_version: '2.3.0' + } + } + ] + } + }, + expectedBody = { template_id: expectedTemplateId }; + + suiteSetup(function(done) { + // Enable Varnish purge for tests + serverOptions.varnish_purge_enabled = true; + + server = new CartodbWindshaft(serverOptions); + + sqlApiServer = new SqlApiEmulator(global.environment.sqlapi.port, done); + + redisClient = redis.createClient(global.environment.redis.port); + }); + + var surrogateKeysCacheInvalidateFn = SurrogateKeysCache.prototype.invalidate; + + beforeEach(function(done) { + var postTemplateRequest = { + url: '/tiles/template?api_key=1234', + method: 'POST', + headers: { + host: templateOwner, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template) + }; + + Step( + function postTemplate() { + var next = this; + assert.response(server, + postTemplateRequest, + { + status: 200 + }, + function(res) { + next(null, res); + } + ); + }, + function rePostTemplate(err, res) { + if (err) { + throw err; + } + var parsedBody = JSON.parse(res.body); + assert.deepEqual(parsedBody, expectedBody); + return true; + }, + function finish(err) { + done(err); + } + ); + }); + + test("update template calls surrogate keys invalidation", function(done) { + var cacheEntryKey; + var surrogateKeysCacheInvalidateMethodInvoked = false; + SurrogateKeysCache.prototype.invalidate = function(cacheEntry) { + cacheEntryKey = cacheEntry.key(); + surrogateKeysCacheInvalidateMethodInvoked = true; + }; + + Step( + function putValidTemplate() { + var updateTemplateRequest = { + url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234', + method: 'PUT', + headers: { + host: templateOwner, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template) + }; + var next = this; + assert.response(server, + updateTemplateRequest, + { + status: 200 + }, + function(res) { + next(null, res); + } + ); + }, + function checkValidUpdate(err, res) { + if (err) { + throw err; + } + var parsedBody = JSON.parse(res.body); + assert.deepEqual(parsedBody, expectedBody); + + assert.ok(surrogateKeysCacheInvalidateMethodInvoked); + assert.equal(cacheEntryKey, new NamedMapsCacheEntry(templateOwner, templateName).key()); + + return null; + }, + function finish(err) { + if ( err ) { + return done(err); + } + redisClient.keys("map_*|localhost", function(err, keys) { + if ( err ) { + return done(err); + } + redisClient.del(keys, function(err) { + return done(err); + }); + }); + } + ); + }); + + test("delete template calls surrogate keys invalidation", function(done) { + + var cacheEntryKey; + var surrogateKeysCacheInvalidateMethodInvoked = false; + SurrogateKeysCache.prototype.invalidate = function(cacheEntry) { + cacheEntryKey = cacheEntry.key(); + surrogateKeysCacheInvalidateMethodInvoked = true; + }; + + Step( + function putValidTemplate() { + var deleteTemplateRequest = { + url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234', + method: 'DELETE', + headers: { + host: templateOwner, + 'Content-Type': 'application/json' + } + }; + var next = this; + assert.response(server, + deleteTemplateRequest, + { + status: 204 + }, + function(res) { + next(null, res); + } + ); + }, + function checkValidUpdate(err) { + if (err) { + throw err; + } + + assert.ok(surrogateKeysCacheInvalidateMethodInvoked); + assert.equal(cacheEntryKey, new NamedMapsCacheEntry(templateOwner, templateName).key()); + + return null; + }, + function finish(err) { + done(err); + } + ); + }); + + afterEach(function(done) { + SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn; + done(); + }); + + suiteTeardown(function(done) { + sqlApiServer.close(done); + }); + +}); diff --git a/test/unit/cartodb/cache/model/named_maps_entry.test.js b/test/unit/cartodb/cache/model/named_maps_entry.test.js new file mode 100644 index 00000000..bd11b5ac --- /dev/null +++ b/test/unit/cartodb/cache/model/named_maps_entry.test.js @@ -0,0 +1,29 @@ +var assert = require('assert'); +var _ = require('underscore'); +var NamedMapsCacheEntry = require('../../../../../lib/cartodb/cache/model/named_maps_entry'); + +suite('cache named_maps_entry', function() { + + var namedMapOwner = 'foo', + namedMapName = 'wadus_name', + namedMapsCacheEntry = new NamedMapsCacheEntry(namedMapOwner, namedMapName), + entryKey = namedMapsCacheEntry.key(); + + test('key is a string', function() { + assert.ok(_.isString(entryKey)); + }); + + test('key is 8 chars length', function() { + assert.equal(entryKey.length, 8); + var entryKeyParts = entryKey.split(':'); + assert.equal(entryKeyParts.length, 2); + assert.equal(entryKeyParts[0], 'n'); + }); + + test('key is name spaced for named maps', function() { + var entryKeyParts = entryKey.split(':'); + assert.equal(entryKeyParts.length, 2); + assert.equal(entryKeyParts[0], 'n'); + }); + +}); From 7e2be7b30f81fab373fea579031c1fed66b1a456 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 26 Jan 2015 16:27:59 +0100 Subject: [PATCH 9/9] Document varnish configuration --- config/environments/development.js.example | 6 +++--- config/environments/production.js.example | 6 +++--- config/environments/staging.js.example | 6 +++--- config/environments/test.js.example | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index e249c479..e28c51e6 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -149,9 +149,9 @@ var config = { } ,varnish: { host: 'localhost', - port: 6082, - http_port: 6081, - purge_enabled: false, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 2b03478a..6c4ea756 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -143,9 +143,9 @@ var config = { } ,varnish: { host: 'localhost', - port: 6082, - http_port: 6081, - purge_enabled: false, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 2403f26d..f8da00db 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -143,9 +143,9 @@ var config = { } ,varnish: { host: 'localhost', - port: 6082, - http_port: 6081, - purge_enabled: false, + port: 6082, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 1c662c63..5866e6c5 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -145,9 +145,9 @@ var config = { } ,varnish: { host: '', - port: null, - http_port: 6081, - purge_enabled: false, + port: null, // the por for the telnet interface where varnish is listening to + http_port: 6081, // the port for the HTTP interface where varnish is listening to + purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not secret: 'xxx', ttl: 86400, layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses