Release 2.13.0

Keep x-cache-channel in named map static maps

CONTRIBUTING.md

@@ -0,0 +1,11 @@
+Contributing
+---
+
+The issue tracker is at [github.com/CartoDB/Windshaft-cartodb](https://github.com/CartoDB/Windshaft-cartodb).
+
+We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
+
+
+## Submitting Contributions
+
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014, Vizzuality
+Copyright (c) 2015, CartoDB
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without 

NEWS.md

@@ -1,8 +1,79 @@
 # Changelog
 
+## 2.13.0
+
+Released 2015-09-21
+
+New features:
+ - Keep x-cache-channel in named map static maps
+
+
+## 2.12.0
+
+Released 2015-08-27
+
+Announcements:
+ - Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0)
+
+New features:
+ - Make http and https globalAgent options configurable
+   * If config is not provided it configures them with default values
+
+
+## 2.11.0
+
+Released 2015-08-26
+
+Announcements:
+ - Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0)
+
+
+## 2.10.0
+
+Released 2015-08-18
+
+New features:
+ - Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information.
+
+Announcements:
+ - Upgrades windshaft to [0.49.0](https://github.com/CartoDB/Windshaft/releases/tag/0.49.0)
+
+
+## 2.9.0
+
+Released 2015-08-06
+
+New features:
+ - Send memory usage stats
+
+
+## 2.8.0
+
+Released 2015-07-15
+
+Announcements:
+ - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
+
+
+## 2.7.2
+
+Released 2015-07-14
+
+Enhancements:
+ - Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names
+
+
+## 2.7.1
+
+Released 2015-07-06
+
+Bug fixes:
+ - redis-mpool `noReadyCheck` and `unwatchOnRelease` options from config and defaulted
+
+
 ## 2.7.0
 
-Released 2015-07-03
+Released 2015-07-06
 
 Announcements:
  - Upgrades windshaft to [0.47.0](https://github.com/CartoDB/Windshaft/releases/tag/0.47.0)

README.md

@@ -95,3 +95,9 @@ Examples
 --------
 
 [CartoDB's Map Gallery](http://cartodb.com/gallery/) showcases several examples of visualisations built on top of this.
+
+
+Contributing
+---
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).

app.js

@@ -9,6 +9,8 @@
 
 var path = require('path');
 var fs = require('fs');
+var http = require('http');
+var https = require('https');
 var RedisPool = require('redis-mpool');
 var _ = require('underscore');
 
@@ -65,8 +67,23 @@ if ( global.environment.log_filename ) {
 global.log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = global.log4js.getLogger();
 
-var redisOpts = _.extend(global.environment.redis, { name: 'windshaft', unwatchOnRelease: false, noReadyCheck: true }),
-    redisPool = new RedisPool(redisOpts);
+var redisOpts = _.defaults(global.environment.redis, {
+    name: 'windshaft',
+    unwatchOnRelease: false,
+    noReadyCheck: true
+});
+var redisPool = new RedisPool(redisOpts);
+
+// set global HTTP and HTTPS agent default configurations
+// ref https://nodejs.org/api/http.html#http_new_agent_options
+var agentOptions = _.defaults(global.environment.httpAgent || {}, {
+    keepAlive: false,
+    keepAliveMsecs: 1000,
+    maxSockets: Infinity,
+    maxFreeSockets: 256
+});
+http.globalAgent = new http.Agent(agentOptions);
+https.globalAgent = new https.Agent(agentOptions);
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
@@ -82,6 +99,13 @@ if (global.statsClient) {
         global.statsClient.gauge(keyPrefix + 'unused', status.unused);
         global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
     });
+
+    setInterval(function() {
+        var memoryUsage = process.memoryUsage();
+        Object.keys(memoryUsage).forEach(function(k) {
+            global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
+        });
+    }, 5000);
 }
 
 // Maximum number of connections for one process
@@ -95,10 +119,10 @@ ws.listen(global.environment.port, global.environment.host);
 var version = require("./package").version;
 
 ws.on('listening', function() {
-  console.log(
-      "Windshaft tileserver %s started on %s:%s (%s)",
-      version, global.environment.host, global.environment.port, ENV
-  );
+    console.log(
+        "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
+        version, global.environment.host, global.environment.port, process.pid, ENV
+    );
 });
 
 process.on('SIGHUP', function() {

config/environments/development.js.example

@@ -2,6 +2,9 @@ var config = {
      environment: 'development'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -86,8 +89,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -96,6 +101,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -146,6 +162,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -176,7 +202,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
+    }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: 'localhost',

config/environments/production.js.example

@@ -2,6 +2,9 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -90,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -140,6 +156,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -170,7 +196,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
+    }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: 'localhost',

config/environments/staging.js.example

@@ -2,6 +2,9 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -90,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -140,6 +156,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -170,7 +196,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
+    }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: 'localhost',

config/environments/test.js.example

@@ -2,6 +2,9 @@ var config = {
      environment: 'test'
     ,port: 8888
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -90,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -142,6 +158,16 @@ var config = {
               type: 'fs', // 'fs' and 'url' supported
               src: __dirname + '/../../assets/default-placeholder.png'
           }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
     ,millstone: {
@@ -172,7 +198,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
+    }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: '',

lib/cartodb/api/query_tables_api.js

@@ -14,7 +14,7 @@ module.exports = QueryTablesApi;
 
 QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
 
-    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
+    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
 
     this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback);
 };
@@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
         callback(new Error('could not fetch source tables: ' + msg));
         return;
     }
-    var qtables = rows[0].cdb_querytables;
-    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+
+    // This is an Array, so no need to split into parts
+    var tableNames = rows[0].cdb_querytablestext;
     callback(null, tableNames);
 }
 
@@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
 
     var query = [
         'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
+            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
         ')',
         'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
         'FROM CDB_TableMetadata m',
@@ -54,8 +54,8 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
 
     var result = rows[0];
 
-    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+    // This is an Array, so no need to split into parts
+    var tableNames = result.tablenames;
 
     var lastUpdatedTime = result.max || 0;
 

lib/cartodb/cartodb_windshaft.js

@@ -187,19 +187,18 @@ var CartodbWindshaft = function(serverOptions) {
      * END Routing
      ******************************************************************************************************************/
 
-    var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive);
+    var healthCheck = new HealthCheck(global.environment.disabled_file);
     ws.get('/health', function(req, res) {
         var healthConfig = global.environment.health || {};
 
         if (!!healthConfig.enabled) {
             var startTime = Date.now();
-            healthCheck.check(healthConfig, function(err, result) {
+            healthCheck.check(function(err) {
                 var ok = !err;
                 var response = {
                     enabled: true,
                     ok: ok,
-                    elapsed: Date.now() - startTime,
-                    result: result
+                    elapsed: Date.now() - startTime
                 };
                 if (err) {
                     response.err = err.message;

lib/cartodb/controllers/named_static_maps.js

@@ -121,7 +121,6 @@ NamedStaticMapsController.prototype.named = function(req, res) {
 
             // added by createLayergroup
             cacheChannel = res.header('X-Cache-Channel');
-            res.removeHeader('X-Cache-Channel');
             self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbUser, template.name));
 
             layergroupId = layergroup.layergroupid.split(":")[0];

lib/cartodb/monitoring/health_check.js

@@ -1,31 +1,20 @@
 var fs = require('fs');
 var step = require('step');
 
-function HealthCheck(metadataBackend, tilelive) {
-    this.metadataBackend = metadataBackend;
-    this.tilelive = tilelive;
+function HealthCheck(disableFile) {
+    this.disableFile = disableFile;
 }
 
 module.exports = HealthCheck;
 
 
-HealthCheck.prototype.check = function(config, callback) {
+HealthCheck.prototype.check = function(callback) {
 
-    var result = {
-            redis: {
-                ok: false
-            },
-            mapnik: {
-                ok: false
-            },
-            tile: {
-                ok: false
-            }
-        };
+    var self = this;
 
     step(
         function getManualDisable() {
-          fs.readFile(global.environment.disabled_file, this);
+          fs.readFile(self.disableFile, this);
         },
         function handleDisabledFile(err, data) {
           var next = this;
@@ -39,7 +28,7 @@ HealthCheck.prototype.check = function(config, callback) {
           }
         },
         function handleResult(err) {
-            callback(err, result);
+            return callback(err);
         }
     );
 };

lib/cartodb/monitoring/map-config.xml

@@ -1,4 +0,0 @@
-<Map
-    background-color="#c33"
-    srs="+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs">
-</Map>

lib/cartodb/server_options.js

@@ -79,6 +79,7 @@ module.exports = function(redisPool) {
         },
         renderer: {
             mapnik: rendererConfig.mapnik,
+            torque: rendererConfig.torque,
             http: rendererConfig.http
         },
         redis: global.environment.redis,

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.7.0",
+    "version": "2.13.0",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,7 +24,7 @@
     "dependencies": {
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "0.47.0",
+        "windshaft": "0.51.0",
         "step": "~0.0.5",
         "queue-async": "~1.0.7",
         "request": "~2.9.203",

test/acceptance/health_check.js

@@ -59,7 +59,7 @@ describe('health checks', function () {
         callback(null, "Maintenance");
       };
       
-      healthCheck.check(null, function(err/*, result*/) {
+      healthCheck.check(function(err) {
         assert.equal(err.message, "Maintenance");
         assert.equal(err.http_status, 503);
         done();

test/support/sql/CDB_QueryTables.sql

@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x) as p, unnest(s) as sc from inp
+      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
+      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;