Shared cache for affected tables in layergroup and map controllers

lib/cartodb/controllers/layergroup.js

@@ -4,6 +4,7 @@ var step = require('step');
 var cors = require('../middleware/cors');
 
 var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
+var TablesCacheEntry = require('../cache/model/database_tables_entry');
 
 /**
  * @param app
@@ -11,19 +12,23 @@ var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'
  * @param {TileBackend} tileBackend
  * @param {PreviewBackend} previewBackend
  * @param {AttributesBackend} attributesBackend
+ * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
  * @param {QueryTablesApi} queryTablesApi
+ * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
-function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, userLimitsApi,
-                              queryTablesApi) {
+function LayergroupController(app, mapStore, tileBackend, previewBackend, attributesBackend, surrogateKeysCache,
+                              userLimitsApi, queryTablesApi, layergroupAffectedTables) {
     this.app = app;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
     this.attributesBackend = attributesBackend;
+    this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.queryTablesApi = queryTablesApi;
+    this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.channelCache = {};
 }
@@ -232,16 +237,19 @@ LayergroupController.prototype.sendResponse = function(req, res, args) {
     }
     res.header('Last-Modified', lastUpdated.toUTCString());
 
+    var dbName = req.params.dbname;
     step(
-        function getCacheChannel() {
-            self.cacheChannel(req, this);
+        function getAffectedTables() {
+            self.getAffectedTables(req.context.user, dbName, req.params.token, this);
         },
-        function sendResponse(err, cacheChannel) {
+        function sendResponse(err, affectedTables) {
             if (err) {
                 console.log('ERROR generating cache channel: ' + err);
             }
-            if (!!cacheChannel) {
-                res.header('X-Cache-Channel', cacheChannel);
+            if (!!affectedTables) {
+                var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
+                res.header('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
+                self.surrogateKeysCache.tag(res, tablesCacheEntry);
             }
             self.app.sendResponse(res, args);
         }
@@ -249,30 +257,13 @@ LayergroupController.prototype.sendResponse = function(req, res, args) {
 
 };
 
-LayergroupController.prototype.buildCacheChannel = function (dbName, tableNames){
-    return dbName + ':' + tableNames.join(',');
-};
-
-LayergroupController.prototype.cacheChannel = function(req, callback) {
-    if (req.profiler) {
-        req.profiler.start('addCacheChannel');
-    }
-
-    var dbName = req.params.dbname;
-
-    // no token means no tables associated
-    if (!req.params.token) {
-        return callback(null, this.buildCacheChannel(dbName, []));
-    }
-
+LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
     var self = this;
 
-    var cacheKey = [ dbName, req.params.token ].join(':');
-
     step(
         function checkCached() {
-            if ( self.channelCache.hasOwnProperty(cacheKey) ) {
-                return callback(null, self.channelCache[cacheKey]);
+            if (self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) {
+                return callback(null, self.layergroupAffectedTables.get(dbName, layergroupId));
             }
             return null;
         },
@@ -281,12 +272,9 @@ LayergroupController.prototype.cacheChannel = function(req, callback) {
 
             step(
                 function loadFromStore() {
-                    self.mapStore.load(req.params.token, this);
+                    self.mapStore.load(layergroupId, this);
                 },
                 function getSQL(err, mapConfig) {
-                    if (req.profiler) {
-                        req.profiler.done('mapStore_load');
-                    }
                     assert.ifError(err);
 
                     var queries = mapConfig.getLayers()
@@ -309,22 +297,17 @@ LayergroupController.prototype.cacheChannel = function(req, callback) {
                 throw new Error("this request doesn't need an X-Cache-Channel generated");
             }
 
-            self.queryTablesApi.getAffectedTablesInQuery(req.context.user, sql, this); // in addCacheChannel
+            self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
         },
         function buildCacheChannel(err, tableNames) {
             assert.ifError(err);
 
-            if (req.profiler) {
-                req.profiler.done('affectedTables');
-            }
+            self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
 
-            var cacheChannel = self.buildCacheChannel(dbName, tableNames);
-            self.channelCache[cacheKey] = cacheChannel;
-
-            return cacheChannel;
+            return tableNames;
         },
-        function finish(err, cacheChannel) {
-            callback(err, cacheChannel);
+        function finish(err, affectedTables) {
+            callback(err, affectedTables);
         }
     );
 };

lib/cartodb/controllers/map.js

@@ -23,11 +23,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
  * @param metadataBackend
  * @param {QueryTablesApi} queryTablesApi
  * @param {SurrogateKeysCache} surrogateKeysCache
- * @param {{UserLimitsApi}} userLimitsApi
+ * @param {UserLimitsApi} userLimitsApi
+ * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
 function MapController(app, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi,
-                       surrogateKeysCache, userLimitsApi) {
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
     this.app = app;
     this.pgConnection = pgConnection;
     this.templateMaps = templateMaps;
@@ -36,9 +37,9 @@ function MapController(app, pgConnection, templateMaps, mapBackend, metadataBack
     this.queryTablesApi = queryTablesApi;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
-    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.layergroupAffectedTables = layergroupAffectedTables;
 
-    this.channelCache = {};
+    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
 }
 
 module.exports = MapController;
@@ -265,31 +266,46 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
     }).join(';');
 
     var dbName = req.params.dbname;
-    var cacheKey = dbName + ':' + layergroup.layergroupid;
+    var layergroupId = layergroup.layergroupid;
 
     step(
-        function getAffectedTablesAndLastUpdatedTime() {
-            self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
+        function checkCachedAffectedTables() {
+            return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
+        },
+        function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
+            assert.ifError(err);
+            if (hasCache) {
+                var next = this;
+                var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
+                self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
+                    if (err) {
+                        return next(err);
+                    }
+                    return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
+                });
+            } else {
+                self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
+            }
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
             if (req.profiler) {
                 req.profiler.done('queryTablesAndLastUpdated');
             }
             assert.ifError(err);
-            var cacheChannel = self.buildCacheChannel(dbName, result.affectedTables);
-            self.channelCache[cacheKey] = cacheChannel;
+            self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
 
             // last update for layergroup cache buster
             layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
             layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
 
             if (req.method === 'GET') {
+                var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
                 var ttl = global.environment.varnish.layergroupTtl || 86400;
                 res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
                 res.header('Last-Modified', (new Date()).toUTCString());
-                res.header('X-Cache-Channel', cacheChannel);
+                res.header('X-Cache-Channel', tableCacheEntry.getCacheChannel());
                 if (result.affectedTables && result.affectedTables.length > 0) {
-                    self.surrogateKeysCache.tag(res, new TablesCacheEntry(dbName, result.affectedTables));
+                    self.surrogateKeysCache.tag(res, tableCacheEntry);
                 }
             }
 
@@ -300,7 +316,3 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
         }
     );
 };
-
-MapController.prototype.buildCacheChannel = function (dbName, tableNames){
-    return dbName + ':' + tableNames.join(',');
-};