Merge pull request #1107 from CartoDB/max-age-directive

Set directive 'max-age' to a fallback value

NEWS.md

@@ -5,6 +5,7 @@ Released 2019-XX-XX
 
 Announcements:
 
+- Cache control header fine tuning. Set a shorter value for "max-age" directive if there is no way to know when to trigger the invalidation.
 - Update deps:
   - windshaft@5.3.0:
     - Update @carto/mapnik to [`3.6.2-carto.15`](https://github.com/CartoDB/node-mapnik/blob/v3.6.2-carto.15/CHANGELOG.carto.md#362-carto15).

config/environments/development.js.example

@@ -326,6 +326,7 @@ var config = {
         purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
+        fallbackTtl: 300,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
     // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly

config/environments/production.js.example

@@ -326,6 +326,7 @@ var config = {
         purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
+        fallbackTtl: 300,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
     // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly

config/environments/staging.js.example

@@ -326,6 +326,7 @@ var config = {
         purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
+        fallbackTtl: 300,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
     // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly

config/environments/test.js.example

@@ -328,6 +328,7 @@ var config = {
         purge_enabled: false, // whether the purge/invalidation mechanism is enabled in varnish or not
         secret: 'xxx',
         ttl: 86400,
+        fallbackTtl: 300,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
     // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly

lib/cartodb/api/middlewares/cache-control-header.js

@@ -1,21 +1,46 @@
 'use strict';
 
 const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
+const FIVE_MINUTES_IN_SECONDS = 60 * 5;
+const FALLBACK_TTL = global.environment.varnish.fallbackTtl || FIVE_MINUTES_IN_SECONDS;
 
-module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) {
+module.exports = function setCacheControlHeader ({
+    ttl = ONE_YEAR_IN_SECONDS,
+    fallbackTtl = FALLBACK_TTL,
+    revalidate = false
+} = {}) {
     return function setCacheControlHeaderMiddleware (req, res, next) {
         if (req.method !== 'GET') {
             return next();
         }
 
-        const directives = [ 'public', `max-age=${ttl}` ];
+        const { mapConfigProvider = { getAffectedTables: callback => callback() } } = res.locals;
 
-        if (revalidate) {
-            directives.push('must-revalidate');
-        }
+        mapConfigProvider.getAffectedTables((err, affectedTables) => {
+            if (err) {
+                global.logger.warn('ERROR generating Cache Control Header:', err);
+                return next();
+            }
 
-        res.set('Cache-Control', directives.join(','));
+            const directives = [ 'public' ];
 
-        next();
+            if (everyAffectedTablesCanBeInvalidated(affectedTables)) {
+                directives.push(`max-age=${ttl}`);
+            } else {
+                directives.push(`max-age=${fallbackTtl}`);
+            }
+
+            if (revalidate) {
+                directives.push('must-revalidate');
+            }
+
+            res.set('Cache-Control', directives.join(','));
+
+            next();
+        });
     };
 };
+
+function everyAffectedTablesCanBeInvalidated (affectedTables) {
+    return affectedTables && affectedTables.getTables().every(table => table.updated_at !== null);
+}

test/acceptance/cache/cache-control-header.js

@@ -0,0 +1,124 @@
+'use strict';
+
+require('../../support/test_helper');
+
+const assert = require('../../support/assert');
+const TestClient = require('../../support/test-client');
+
+const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
+const FIVE_MINUTES_IN_SECONDS = 60 * 5;
+
+const defaultLayers = [{
+    type: 'cartodb',
+    options: {
+        sql: TestClient.SQL.ONE_POINT,
+        cartocss: TestClient.CARTOCSS.POINTS,
+        cartocss_version: '2.3.0'
+    }
+}];
+
+function createMapConfig (layers = defaultLayers) {
+    return {
+        version: '1.8.0',
+        layers: layers
+    };
+}
+
+describe('cache-control header', function () {
+    describe('max-age directive', function () {
+        it('tile from a table which is included in cdb_tablemetada', function (done) {
+            const ttl = ONE_YEAR_IN_SECONDS;
+            const mapConfig = createMapConfig([{
+                type: 'cartodb',
+                options: {
+                    sql: 'select * from test_table',
+                    cartocss: TestClient.CARTOCSS.POINTS,
+                    cartocss_version: '2.3.0'
+                }
+            }]);
+
+            const testClient = new TestClient(mapConfig);
+
+            testClient.getTile(0, 0, 0, {}, function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(res.headers['cache-control'], `public,max-age=${ttl}`);
+                testClient.drain(done);
+            });
+        });
+
+        it('tile from a table which is NOT included in cdb_tablemetada', function (done) {
+            const ttl = global.environment.varnish.fallbackTtl || FIVE_MINUTES_IN_SECONDS;
+            const mapConfig = createMapConfig([{
+                type: 'cartodb',
+                options: {
+                    sql: 'select * from test_table_2',
+                    cartocss: TestClient.CARTOCSS.POINTS,
+                    cartocss_version: '2.3.0'
+                }
+            }]);
+
+            const testClient = new TestClient(mapConfig);
+
+            testClient.getTile(0, 0, 0, {}, function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(res.headers['cache-control'], `public,max-age=${ttl}`);
+                testClient.drain(done);
+            });
+        });
+
+        it('tile from joined tables which one of them is NOT included in cdb_tablemetada', function (done) {
+            const ttl = global.environment.varnish.fallbackTtl || FIVE_MINUTES_IN_SECONDS;
+            const mapConfig = createMapConfig([{
+                type: 'cartodb',
+                options: {
+                    sql: `
+                        select
+                            t.cartodb_id,
+                            t.the_geom,
+                            t.the_geom_webmercator
+                        from
+                            test_table t,
+                            test_table_2 t2
+                        where
+                            t.cartodb_id = t2.cartodb_id
+                    `,
+                    cartocss: TestClient.CARTOCSS.POINTS,
+                    cartocss_version: '2.3.0'
+                }
+            }]);
+
+            const testClient = new TestClient(mapConfig);
+
+            testClient.getTile(0, 0, 0, {}, function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(res.headers['cache-control'], `public,max-age=${ttl}`);
+                testClient.drain(done);
+            });
+        });
+
+        it('tile from a dynamic query which doesn\'t use a table' , function (done) {
+            const ttl = ONE_YEAR_IN_SECONDS;
+            const mapConfig = createMapConfig();
+
+            const testClient = new TestClient(mapConfig);
+
+            testClient.getTile(0, 0, 0, {}, function (err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(res.headers['cache-control'], `public,max-age=${ttl}`);
+                testClient.drain(done);
+            });
+        });
+    });
+});