Merge pull request #249 from CartoDB/247-add-named-maps-surrogate-keys

Add named maps surrogate keys and call invalidation on template modification/deletion

config/environments/development.js.example

@@ -149,7 +149,9 @@ var config = {
     }
     ,varnish: {
         host: 'localhost',
-        port: 6082,
+        port: 6082, // the por for the telnet interface where varnish is listening to
+        http_port: 6081, // the port for the HTTP interface where varnish is listening to
+        purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses

config/environments/production.js.example

@@ -143,7 +143,9 @@ var config = {
     }
     ,varnish: {
         host: 'localhost',
-        port: 6082,
+        port: 6082, // the por for the telnet interface where varnish is listening to
+        http_port: 6081, // the port for the HTTP interface where varnish is listening to
+        purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses

config/environments/staging.js.example

@@ -143,7 +143,9 @@ var config = {
     }
     ,varnish: {
         host: 'localhost',
-        port: 6082,
+        port: 6082, // the por for the telnet interface where varnish is listening to
+        http_port: 6081, // the port for the HTTP interface where varnish is listening to
+        purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses

config/environments/test.js.example

@@ -145,7 +145,9 @@ var config = {
     }
     ,varnish: {
         host: '',
-        port: null,
+        port: null, // the por for the telnet interface where varnish is listening to
+        http_port: 6081, // the port for the HTTP interface where varnish is listening to
+        purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses

lib/cartodb/cache/backend/varnish_http.js

@@ -0,0 +1,32 @@
+var request = require('request');
+
+function VarnishHttpCacheBackend(host, port) {
+    this.host = host;
+    this.port = port;
+}
+
+module.exports = VarnishHttpCacheBackend;
+
+/**
+ * @param cacheObject should respond to `key() -> String` method
+ * @param {Function} callback
+ */
+VarnishHttpCacheBackend.prototype.invalidate = function(cacheObject, callback) {
+    request(
+        {
+            method: 'PURGE',
+            url: 'http://' + this.host + ':' + this.port + '/key',
+            headers: {
+                'Invalidation-Match': '\\b' + cacheObject.key() + '\\b'
+            }
+        },
+        function(err, response) {
+            if (err || response.statusCode !== 204) {
+                return callback(new Error('Unable to invalidate Varnish object'));
+            }
+            return callback(null);
+        }
+    );
+};
+
+module.exports = VarnishHttpCacheBackend;

lib/cartodb/cache/model/named_maps_entry.js

@@ -0,0 +1,18 @@
+var crypto = require('crypto');
+
+function NamedMaps(owner, name) {
+    this.namespace = 'n';
+    this.owner = owner;
+    this.name = name;
+}
+
+module.exports = NamedMaps;
+
+
+NamedMaps.prototype.key = function() {
+    return this.namespace + ':' + shortHashKey(this.owner + ':' + this.name);
+};
+
+function shortHashKey(target) {
+    return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
+}

lib/cartodb/cache/surrogate_keys_cache.js

@@ -0,0 +1,26 @@
+/**
+ * @param cacheBackend should respond to `invalidate(cacheObject, callback)` method
+ * @constructor
+ */
+function SurrogateKeysCache(cacheBackend) {
+    this.cacheBackend = cacheBackend;
+}
+
+module.exports = SurrogateKeysCache;
+
+
+/**
+ * @param response should respond to `header(key, value)` method
+ * @param cacheObject should respond to `key() -> String` method
+ */
+SurrogateKeysCache.prototype.tag = function(response, cacheObject) {
+    response.header('Surrogate-Key', cacheObject.key());
+};
+
+/**
+ * @param cacheObject should respond to `key() -> String` method
+ * @param {Function} callback
+ */
+SurrogateKeysCache.prototype.invalidate = function(cacheObject, callback) {
+    this.cacheBackend.invalidate(cacheObject, callback);
+};

lib/cartodb/cartodb_windshaft.js

@@ -54,6 +54,26 @@ var CartodbWindshaft = function(serverOptions) {
     var templateMaps = new TemplateMaps(redisPool, templateMapsOpts);
     serverOptions.templateMaps = templateMaps;
 
+    var SurrogateKeysCache = require('./cache/surrogate_keys_cache'),
+        NamedMapsCacheEntry = require('./cache/model/named_maps_entry'),
+        VarnishHttpCacheBackend = require('./cache/backend/varnish_http'),
+        varnishHttpCacheBackend = new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port),
+        surrogateKeysCache = new SurrogateKeysCache(varnishHttpCacheBackend);
+
+    if (serverOptions.varnish_purge_enabled) {
+        function invalidateNamedMap(owner, templateName) {
+            surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
+                if (err) {
+                    console.warn('Cache: surrogate key invalidation failed');
+                }
+            });
+        }
+
+        ['update', 'delete'].forEach(function(eventType) {
+            templateMaps.on(eventType, invalidateNamedMap);
+        });
+    }
+
     // boot
     var ws = new Windshaft.Server(serverOptions);
 
@@ -138,7 +158,7 @@ var CartodbWindshaft = function(serverOptions) {
 
     var TemplateMapsController = require('./controllers/template_maps'),
         templateMapsController = new TemplateMapsController(
-            ws, serverOptions, templateMaps, cartoData, template_baseurl
+            ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry
         );
     templateMapsController.register(ws);
 

lib/cartodb/controllers/template_maps.js

@@ -1,12 +1,15 @@
 var Step = require('step');
 var _ = require('underscore');
 
-function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl) {
+function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache,
+                                NamedMapsCacheEntry) {
     this.app = app;
     this.serverOptions = serverOptions;
     this.templateMaps = templateMaps;
     this.metadataBackend = metadataBackend;
     this.templateBaseUrl = templateBaseUrl;
+    this.surrogateKeysCache = surrogateKeysCache;
+    this.NamedMapsCacheEntry = NamedMapsCacheEntry;
 }
 
 module.exports = TemplateMapsController;
@@ -430,6 +433,9 @@ TemplateMapsController.prototype.instantiateTemplate = function(req, res, templa
             }
             var tplhash = self.templateMaps.fingerPrint(template).substring(0,8);
             layergroup.layergroupid = cdbuser + '@' + tplhash + '@' + layergroup.layergroupid;
+
+            self.surrogateKeysCache.tag(res, new self.NamedMapsCacheEntry(cdbuser, template.name));
+
             return layergroup;
         },
         callback

lib/cartodb/server_options.js

@@ -89,7 +89,9 @@ module.exports = function(redisPool) {
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
+        varnish_http_port: global.environment.varnish.http_port,
         varnish_secret: global.environment.varnish.secret,
+        varnish_purge_enabled: global.environment.varnish.purge_enabled,
         cache_enabled: global.environment.cache_enabled,
         log_format: global.environment.log_format,
         useProfiler: global.environment.useProfiler

lib/cartodb/template_maps.js

@@ -3,6 +3,11 @@ var crypto  = require('crypto'),
     _       = require('underscore'),
     dot     = require('dot');
 
+
+var EventEmitter = require('events').EventEmitter;
+var util = require('util');
+
+
 // Class handling map templates
 //
 // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps
@@ -16,6 +21,10 @@ var crypto  = require('crypto'),
 //
 // 
 function TemplateMaps(redis_pool, opts) {
+  if (!(this instanceof TemplateMaps)) return new TemplateMaps();
+
+  EventEmitter.call(this);
+
   this.redis_pool = redis_pool;
   this.opts = opts || {};
 
@@ -42,6 +51,11 @@ function TemplateMaps(redis_pool, opts) {
   this.lock_ttl = this.opts['lock_ttl'] || 5000;
 }
 
+util.inherits(TemplateMaps, EventEmitter);
+
+module.exports = TemplateMaps;
+
+
 var o = TemplateMaps.prototype;
 
 //--------------- PRIVATE METHODS --------------------------------
@@ -193,6 +207,10 @@ o.addTemplate = function(owner, template, callback) {
             return true;
         },
         function finish(err) {
+            if (!err) {
+                self.emit('add', owner, templateName, template);
+            }
+
             callback(err, templateName, template);
         }
   );
@@ -223,6 +241,10 @@ o.delTemplate = function(owner, tpl_id, callback) {
             return true;
         },
         function finish(err) {
+            if (!err) {
+                self.emit('delete', owner, tpl_id);
+            }
+
             callback(err);
         }
     );
@@ -286,6 +308,10 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
             return true;
         },
         function finish(err) {
+            if (!err) {
+                self.emit('update', owner, templateName, template);
+            }
+
             callback(err, template);
         }
     );
@@ -438,5 +464,3 @@ o.fingerPrint = function(template) {
     .digest('hex')
   ;
 };
-
-module.exports = TemplateMaps;

test/acceptance/cache/surrogate_keys_invalidation.js

@@ -0,0 +1,211 @@
+var assert      = require('../../support/assert');
+var redis       = require('redis');
+var Step        = require('step');
+
+var helper = require(__dirname + '/../../support/test_helper');
+
+var SqlApiEmulator = require(__dirname + '/../../support/SQLAPIEmu.js');
+
+var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry');
+var SurrogateKeysCache = require(__dirname + '/../../../lib/cartodb/cache/surrogate_keys_cache');
+
+var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/cartodb_windshaft');
+var ServerOptions = require(__dirname + '/../../../lib/cartodb/server_options');
+var serverOptions = ServerOptions();
+
+
+suite('templates surrogate keys', function() {
+
+    var redisClient,
+        sqlApiServer,
+        server;
+
+    var templateOwner = 'localhost',
+        templateName = 'acceptance',
+        expectedTemplateId = templateOwner + '@' + templateName,
+        template = {
+            version: '0.0.1',
+            name: templateName,
+            auth: {
+                method: 'open'
+            },
+            layergroup:  {
+                version: '1.2.0',
+                layers: [
+                    {
+                        options: {
+                            sql: 'select 1 cartodb_id, null::geometry as the_geom_webmercator',
+                            cartocss: '#layer { marker-fill:blue; }',
+                            cartocss_version: '2.3.0'
+                        }
+                    }
+                ]
+            }
+        },
+        expectedBody = { template_id: expectedTemplateId };
+
+    suiteSetup(function(done) {
+        // Enable Varnish purge for tests
+        serverOptions.varnish_purge_enabled = true;
+
+        server = new CartodbWindshaft(serverOptions);
+
+        sqlApiServer = new SqlApiEmulator(global.environment.sqlapi.port, done);
+
+        redisClient = redis.createClient(global.environment.redis.port);
+    });
+
+    var surrogateKeysCacheInvalidateFn = SurrogateKeysCache.prototype.invalidate;
+
+    beforeEach(function(done) {
+        var postTemplateRequest = {
+            url: '/tiles/template?api_key=1234',
+            method: 'POST',
+            headers: {
+                host: templateOwner,
+                'Content-Type': 'application/json'
+            },
+            data: JSON.stringify(template)
+        };
+
+        Step(
+            function postTemplate() {
+                var next = this;
+                assert.response(server,
+                    postTemplateRequest,
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        next(null, res);
+                    }
+                );
+            },
+            function rePostTemplate(err, res) {
+                if (err) {
+                    throw err;
+                }
+                var parsedBody = JSON.parse(res.body);
+                assert.deepEqual(parsedBody, expectedBody);
+                return true;
+            },
+            function finish(err) {
+                done(err);
+            }
+        );
+    });
+
+    test("update template calls surrogate keys invalidation", function(done) {
+        var cacheEntryKey;
+        var surrogateKeysCacheInvalidateMethodInvoked = false;
+        SurrogateKeysCache.prototype.invalidate = function(cacheEntry) {
+            cacheEntryKey = cacheEntry.key();
+            surrogateKeysCacheInvalidateMethodInvoked = true;
+        };
+
+        Step(
+            function putValidTemplate() {
+                var updateTemplateRequest = {
+                    url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234',
+                    method: 'PUT',
+                    headers: {
+                        host: templateOwner,
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(template)
+                };
+                var next = this;
+                assert.response(server,
+                    updateTemplateRequest,
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        next(null, res);
+                    }
+                );
+            },
+            function checkValidUpdate(err, res) {
+                if (err) {
+                    throw err;
+                }
+                var parsedBody = JSON.parse(res.body);
+                assert.deepEqual(parsedBody, expectedBody);
+
+                assert.ok(surrogateKeysCacheInvalidateMethodInvoked);
+                assert.equal(cacheEntryKey, new NamedMapsCacheEntry(templateOwner, templateName).key());
+
+                return null;
+            },
+            function finish(err) {
+                if ( err ) {
+                    return done(err);
+                }
+                redisClient.keys("map_*|localhost", function(err, keys) {
+                    if ( err ) {
+                        return done(err);
+                    }
+                    redisClient.del(keys, function(err) {
+                        return done(err);
+                    });
+                });
+            }
+        );
+    });
+
+    test("delete template calls surrogate keys invalidation", function(done) {
+
+        var cacheEntryKey;
+        var surrogateKeysCacheInvalidateMethodInvoked = false;
+        SurrogateKeysCache.prototype.invalidate = function(cacheEntry) {
+            cacheEntryKey = cacheEntry.key();
+            surrogateKeysCacheInvalidateMethodInvoked = true;
+        };
+
+        Step(
+            function putValidTemplate() {
+                var deleteTemplateRequest = {
+                    url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234',
+                    method: 'DELETE',
+                    headers: {
+                        host: templateOwner,
+                        'Content-Type': 'application/json'
+                    }
+                };
+                var next = this;
+                assert.response(server,
+                    deleteTemplateRequest,
+                    {
+                        status: 204
+                    },
+                    function(res) {
+                        next(null, res);
+                    }
+                );
+            },
+            function checkValidUpdate(err) {
+                if (err) {
+                    throw err;
+                }
+
+                assert.ok(surrogateKeysCacheInvalidateMethodInvoked);
+                assert.equal(cacheEntryKey, new NamedMapsCacheEntry(templateOwner, templateName).key());
+
+                return null;
+            },
+            function finish(err) {
+                done(err);
+            }
+        );
+    });
+
+    afterEach(function(done) {
+        SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn;
+        done();
+    });
+
+    suiteTeardown(function(done) {
+        sqlApiServer.close(done);
+    });
+
+});

test/acceptance/templates.js

@@ -5,6 +5,7 @@ var querystring = require('querystring');
 var semver      = require('semver');
 var Step        = require('step');
 var strftime    = require('strftime');
+var NamedMapsCacheEntry = require(__dirname + '/../../lib/cartodb/cache/model/named_maps_entry');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 var redis_stats_db = 5;
 
@@ -1590,6 +1591,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
           assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
+          helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key());
           return null;
         },
         function finish(err) {
@@ -1605,10 +1607,9 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
           version: '0.0.1',
           name: 'acceptance_open_jsonp_params',
           auth: { method: 'open' },
-          /*
           placeholders: {
             color: { type: "css_color", default: "red" }
-          },*/
+          },
           layergroup:  {
             version: '1.0.0',
             layers: [
@@ -1661,9 +1662,11 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
         function checkInstanciation(err, res)
         {
           if ( err ) throw err;
+            console.log(err, res.body, res.headers);
           assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
-          helper.checkNoCache(res);
+          helper.checkCache(res);
+          helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key());
           return null;
         },
         function finish(err) {
@@ -1845,6 +1848,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
           assert.ok(parsed.hasOwnProperty('layergroupid'),
             "Missing 'layergroupid' from response body: " + res.body);
           layergroupid = parsed.layergroupid;
+          helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance2.name).key());
           return null;
         },
         function updateTemplate(err, res)
@@ -1889,6 +1893,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
           assert.ok(parsed.hasOwnProperty('layergroupid'),
             "Missing 'layergroupid' from response body: " + res.body);
           assert.ok(layergroupid != parsed.layergroupid);
+          helper.checkSurrogateKey(res, new NamedMapsCacheEntry('localhost', template_acceptance2.name).key());
           return null;
         },
         function finish(err) {

test/support/test_helper.js

@@ -50,10 +50,16 @@ function checkCache(res) {
     assert.ok(res.headers.hasOwnProperty('last-modified'));
 }
 
+function checkSurrogateKey(res, expectedKey) {
+    assert.ok(res.headers.hasOwnProperty('surrogate-key'));
+    assert.equal(res.headers['surrogate-key'], expectedKey);
+}
+
 
 module.exports = {
   lzma_compress_to_base64: lzma_compress_to_base64,
   checkNoCache: checkNoCache,
+  checkSurrogateKey: checkSurrogateKey,
   checkCache: checkCache
 };
 

test/unit/cartodb/cache/model/named_maps_entry.test.js

@@ -0,0 +1,29 @@
+var assert = require('assert');
+var _ = require('underscore');
+var NamedMapsCacheEntry = require('../../../../../lib/cartodb/cache/model/named_maps_entry');
+
+suite('cache named_maps_entry', function() {
+
+    var namedMapOwner = 'foo',
+        namedMapName = 'wadus_name',
+        namedMapsCacheEntry = new NamedMapsCacheEntry(namedMapOwner, namedMapName),
+        entryKey = namedMapsCacheEntry.key();
+
+    test('key is a string', function() {
+        assert.ok(_.isString(entryKey));
+    });
+
+    test('key is 8 chars length', function() {
+        assert.equal(entryKey.length, 8);
+        var entryKeyParts = entryKey.split(':');
+        assert.equal(entryKeyParts.length, 2);
+        assert.equal(entryKeyParts[0], 'n');
+    });
+
+    test('key is name spaced for named maps', function() {
+        var entryKeyParts = entryKey.split(':');
+        assert.equal(entryKeyParts.length, 2);
+        assert.equal(entryKeyParts[0], 'n');
+    });
+
+});