Release 2.13.0

Keep x-cache-channel in named map static maps

CONTRIBUTING.md

@@ -0,0 +1,11 @@
+Contributing
+---
+
+The issue tracker is at [github.com/CartoDB/Windshaft-cartodb](https://github.com/CartoDB/Windshaft-cartodb).
+
+We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
+
+
+## Submitting Contributions
+
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014, Vizzuality
+Copyright (c) 2015, CartoDB
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without 

NEWS.md

@@ -1,5 +1,60 @@
 # Changelog
 
+## 2.13.0
+
+Released 2015-09-21
+
+New features:
+ - Keep x-cache-channel in named map static maps
+
+
+## 2.12.0
+
+Released 2015-08-27
+
+Announcements:
+ - Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0)
+
+New features:
+ - Make http and https globalAgent options configurable
+   * If config is not provided it configures them with default values
+
+
+## 2.11.0
+
+Released 2015-08-26
+
+Announcements:
+ - Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0)
+
+
+## 2.10.0
+
+Released 2015-08-18
+
+New features:
+ - Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information.
+
+Announcements:
+ - Upgrades windshaft to [0.49.0](https://github.com/CartoDB/Windshaft/releases/tag/0.49.0)
+
+
+## 2.9.0
+
+Released 2015-08-06
+
+New features:
+ - Send memory usage stats
+
+
+## 2.8.0
+
+Released 2015-07-15
+
+Announcements:
+ - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
+
+
 ## 2.7.2
 
 Released 2015-07-14

README.md

@@ -95,3 +95,9 @@ Examples
 --------
 
 [CartoDB's Map Gallery](http://cartodb.com/gallery/) showcases several examples of visualisations built on top of this.
+
+
+Contributing
+---
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).

app.js

@@ -9,6 +9,8 @@
 
 var path = require('path');
 var fs = require('fs');
+var http = require('http');
+var https = require('https');
 var RedisPool = require('redis-mpool');
 var _ = require('underscore');
 
@@ -72,6 +74,17 @@ var redisOpts = _.defaults(global.environment.redis, {
 });
 var redisPool = new RedisPool(redisOpts);
 
+// set global HTTP and HTTPS agent default configurations
+// ref https://nodejs.org/api/http.html#http_new_agent_options
+var agentOptions = _.defaults(global.environment.httpAgent || {}, {
+    keepAlive: false,
+    keepAliveMsecs: 1000,
+    maxSockets: Infinity,
+    maxFreeSockets: 256
+});
+http.globalAgent = new http.Agent(agentOptions);
+https.globalAgent = new https.Agent(agentOptions);
+
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
 var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
@@ -86,6 +99,13 @@ if (global.statsClient) {
         global.statsClient.gauge(keyPrefix + 'unused', status.unused);
         global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
     });
+
+    setInterval(function() {
+        var memoryUsage = process.memoryUsage();
+        Object.keys(memoryUsage).forEach(function(k) {
+            global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
+        });
+    }, 5000);
 }
 
 // Maximum number of connections for one process
@@ -99,10 +119,10 @@ ws.listen(global.environment.port, global.environment.host);
 var version = require("./package").version;
 
 ws.on('listening', function() {
-  console.log(
-      "Windshaft tileserver %s started on %s:%s (%s)",
-      version, global.environment.host, global.environment.port, ENV
-  );
+    console.log(
+        "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
+        version, global.environment.host, global.environment.port, process.pid, ENV
+    );
 });
 
 process.on('SIGHUP', function() {

config/environments/development.js.example

@@ -101,6 +101,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -151,6 +162,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -185,6 +206,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: 'localhost',
         port: 6082, // the por for the telnet interface where varnish is listening to

config/environments/production.js.example

@@ -95,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -145,6 +156,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -179,6 +200,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: 'localhost',
         port: 6082, // the por for the telnet interface where varnish is listening to

config/environments/staging.js.example

@@ -95,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -145,6 +156,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -179,6 +200,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: 'localhost',
         port: 6082, // the por for the telnet interface where varnish is listening to

config/environments/test.js.example

@@ -95,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -147,6 +158,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -181,6 +202,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: '',
         port: null, // the por for the telnet interface where varnish is listening to

lib/cartodb/cartodb_windshaft.js

@@ -187,19 +187,18 @@ var CartodbWindshaft = function(serverOptions) {
      * END Routing
      ******************************************************************************************************************/
 
-    var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive);
+    var healthCheck = new HealthCheck(global.environment.disabled_file);
     ws.get('/health', function(req, res) {
         var healthConfig = global.environment.health || {};
 
         if (!!healthConfig.enabled) {
             var startTime = Date.now();
-            healthCheck.check(healthConfig, function(err, result) {
+            healthCheck.check(function(err) {
                 var ok = !err;
                 var response = {
                     enabled: true,
                     ok: ok,
-                    elapsed: Date.now() - startTime,
-                    result: result
+                    elapsed: Date.now() - startTime
                 };
                 if (err) {
                     response.err = err.message;

lib/cartodb/controllers/named_static_maps.js

@@ -121,7 +121,6 @@ NamedStaticMapsController.prototype.named = function(req, res) {
 
             // added by createLayergroup
             cacheChannel = res.header('X-Cache-Channel');
-            res.removeHeader('X-Cache-Channel');
             self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, template.name));
 
             layergroupId = layergroup.layergroupid.split(":")[0];

lib/cartodb/monitoring/health_check.js

@@ -1,31 +1,20 @@
 var fs = require('fs');
 var step = require('step');
 
-function HealthCheck(metadataBackend, tilelive) {
-    this.metadataBackend = metadataBackend;
-    this.tilelive = tilelive;
+function HealthCheck(disableFile) {
+    this.disableFile = disableFile;
 }
 
 module.exports = HealthCheck;
 
 
-HealthCheck.prototype.check = function(config, callback) {
+HealthCheck.prototype.check = function(callback) {
 
-    var result = {
-            redis: {
-                ok: false
-            },
-            mapnik: {
-                ok: false
-            },
-            tile: {
-                ok: false
-            }
-        };
+    var self = this;
 
     step(
         function getManualDisable() {
-          fs.readFile(global.environment.disabled_file, this);
+          fs.readFile(self.disableFile, this);
         },
         function handleDisabledFile(err, data) {
           var next = this;
@@ -39,7 +28,7 @@ HealthCheck.prototype.check = function(config, callback) {
           }
         },
         function handleResult(err) {
-            callback(err, result);
+            return callback(err);
         }
     );
 };

lib/cartodb/monitoring/map-config.xml

@@ -1,4 +0,0 @@
-<Map
-    background-color="#c33"
-    srs="+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs">
-</Map>

lib/cartodb/server_options.js

@@ -79,6 +79,7 @@ module.exports = function(redisPool) {
         },
         renderer: {
             mapnik: rendererConfig.mapnik,
+            torque: rendererConfig.torque,
             http: rendererConfig.http
         },
         redis: global.environment.redis,

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.7.2",
+    "version": "2.13.0",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,7 +24,7 @@
     "dependencies": {
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "0.47.0",
+        "windshaft": "0.51.0",
         "step": "~0.0.5",
         "queue-async": "~1.0.7",
         "request": "~2.9.203",

test/acceptance/health_check.js

@@ -59,7 +59,7 @@ describe('health checks', function () {
         callback(null, "Maintenance");
       };
       
-      healthCheck.check(null, function(err/*, result*/) {
+      healthCheck.check(function(err) {
         assert.equal(err.message, "Maintenance");
         assert.equal(err.http_status, 503);
         done();

test/support/sql/CDB_QueryTables.sql

@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x) as p, unnest(s) as sc from inp
+      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
+      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;