Release 2.12.0

Upgrades windshaft to 0.51.0

NEWS.md

@@ -1,5 +1,60 @@
 # Changelog
 
+## 2.12.0
+
+Released 2015-08-27
+
+Announcements:
+ - Upgrades windshaft to [0.51.0](https://github.com/CartoDB/Windshaft/releases/tag/0.51.0)
+
+New features:
+ - Make http and https globalAgent options configurable
+   * If config is not provided it configures them with default values
+
+
+## 2.11.0
+
+Released 2015-08-26
+
+Announcements:
+ - Upgrades windshaft to [0.50.0](https://github.com/CartoDB/Windshaft/releases/tag/0.50.0)
+
+
+## 2.10.0
+
+Released 2015-08-18
+
+New features:
+ - Exposes metatile cache configuration for tilelive-mapnik, see configuration sample files for more information.
+
+Announcements:
+ - Upgrades windshaft to [0.49.0](https://github.com/CartoDB/Windshaft/releases/tag/0.49.0)
+
+
+## 2.9.0
+
+Released 2015-08-06
+
+New features:
+ - Send memory usage stats
+
+
+## 2.8.0
+
+Released 2015-07-15
+
+Announcements:
+ - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
+
+
+## 2.7.2
+
+Released 2015-07-14
+
+Enhancements:
+ - Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names
+
+
 ## 2.7.1
 
 Released 2015-07-06

app.js

@@ -9,6 +9,8 @@
 
 var path = require('path');
 var fs = require('fs');
+var http = require('http');
+var https = require('https');
 var RedisPool = require('redis-mpool');
 var _ = require('underscore');
 
@@ -72,6 +74,17 @@ var redisOpts = _.defaults(global.environment.redis, {
 });
 var redisPool = new RedisPool(redisOpts);
 
+// set global HTTP and HTTPS agent default configurations
+// ref https://nodejs.org/api/http.html#http_new_agent_options
+var agentOptions = _.defaults(global.environment.httpAgent || {}, {
+    keepAlive: false,
+    keepAliveMsecs: 1000,
+    maxSockets: Infinity,
+    maxFreeSockets: 256
+});
+http.globalAgent = new http.Agent(agentOptions);
+https.globalAgent = new https.Agent(agentOptions);
+
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
 var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
@@ -86,6 +99,13 @@ if (global.statsClient) {
         global.statsClient.gauge(keyPrefix + 'unused', status.unused);
         global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
     });
+
+    setInterval(function() {
+        var memoryUsage = process.memoryUsage();
+        Object.keys(memoryUsage).forEach(function(k) {
+            global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
+        });
+    }, 5000);
 }
 
 // Maximum number of connections for one process
@@ -99,10 +119,10 @@ ws.listen(global.environment.port, global.environment.host);
 var version = require("./package").version;
 
 ws.on('listening', function() {
-  console.log(
-      "Windshaft tileserver %s started on %s:%s (%s)",
-      version, global.environment.host, global.environment.port, ENV
-  );
+    console.log(
+        "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
+        version, global.environment.host, global.environment.port, process.pid, ENV
+    );
 });
 
 process.on('SIGHUP', function() {

config/environments/development.js.example

@@ -2,6 +2,9 @@ var config = {
      environment: 'development'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -86,8 +89,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -96,6 +101,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -180,6 +196,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: 'localhost',
         port: 6082, // the por for the telnet interface where varnish is listening to

config/environments/production.js.example

@@ -2,6 +2,9 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -90,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -174,6 +190,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: 'localhost',
         port: 6082, // the por for the telnet interface where varnish is listening to

config/environments/staging.js.example

@@ -2,6 +2,9 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -90,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -174,6 +190,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: 'localhost',
         port: 6082, // the por for the telnet interface where varnish is listening to

config/environments/test.js.example

@@ -2,6 +2,9 @@ var config = {
      environment: 'test'
     ,port: 8888
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
       mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
           // Metatile is the number of tiles-per-side that are going
@@ -90,6 +95,17 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
           // Override metatile behaviour depending on the format
           formatMetatile: {
               png: 2,
@@ -176,6 +192,13 @@ var config = {
         unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
         noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    }
     ,varnish: {
         host: '',
         port: null, // the por for the telnet interface where varnish is listening to

lib/cartodb/api/query_tables_api.js

@@ -14,7 +14,7 @@ module.exports = QueryTablesApi;
 
 QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
 
-    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
+    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
 
     this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback);
 };
@@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
         callback(new Error('could not fetch source tables: ' + msg));
         return;
     }
-    var qtables = rows[0].cdb_querytables;
-    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+
+    // This is an Array, so no need to split into parts
+    var tableNames = rows[0].cdb_querytablestext;
     callback(null, tableNames);
 }
 
@@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
 
     var query = [
         'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
+            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
         ')',
         'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
         'FROM CDB_TableMetadata m',
@@ -54,8 +54,8 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
 
     var result = rows[0];
 
-    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+    // This is an Array, so no need to split into parts
+    var tableNames = result.tablenames;
 
     var lastUpdatedTime = result.max || 0;
 

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.7.1",
+    "version": "2.12.0",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,7 +24,7 @@
     "dependencies": {
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "0.47.0",
+        "windshaft": "0.51.0",
         "step": "~0.0.5",
         "queue-async": "~1.0.7",
         "request": "~2.9.203",