Release 2.9.0

Merge pull request #324 from CartoDB/memory-stats
Send memory usage stats
2015-08-06 18:02:09 +02:00 · 2015-08-06 16:12:18 +02:00 · 2015-08-06 16:06:42 +02:00 · 2015-07-15 11:44:59 +02:00 · 2015-07-15 11:43:54 +02:00 · 2015-07-15 11:43:09 +02:00
9 changed files with 114 additions and 43 deletions
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,8 +1,40 @@
 # Changelog

+## 2.9.0
+
+Released 2015-08-06
+
+New features:
+ - Send memory usage stats
+
+
+## 2.8.0
+
+Released 2015-07-15
+
+Announcements:
+ - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
+
+
+## 2.7.2
+
+Released 2015-07-14
+
+Enhancements:
+ - Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names
+
+
+## 2.7.1
+
+Released 2015-07-06
+
+Bug fixes:
+ - redis-mpool `noReadyCheck` and `unwatchOnRelease` options from config and defaulted
+
+
 ## 2.7.0

-Released 2015-07-03
+Released 2015-07-06

 Announcements:
 - Upgrades windshaft to [0.47.0](https://github.com/CartoDB/Windshaft/releases/tag/0.47.0)
--- a/app.js
+++ b/app.js
@@ -65,8 +65,12 @@ if ( global.environment.log_filename ) {
 global.log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = global.log4js.getLogger();

-var redisOpts = _.extend(global.environment.redis, { name: 'windshaft', unwatchOnRelease: false, noReadyCheck: true }),
-    redisPool = new RedisPool(redisOpts);
+var redisOpts = _.defaults(global.environment.redis, {
+    name: 'windshaft',
+    unwatchOnRelease: false,
+    noReadyCheck: true
+});
+var redisPool = new RedisPool(redisOpts);

 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
@@ -82,6 +86,13 @@ if (global.statsClient) {
        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
    });
+
+    setInterval(function() {
+        var memoryUsage = process.memoryUsage();
+        Object.keys(memoryUsage).forEach(function(k) {
+            global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
+        });
+    }, 5000);
 }

 // Maximum number of connections for one process
--- a/config/environments/development.js.example
+++ b/config/environments/development.js.example
@@ -2,6 +2,9 @@ var config = {
     environment: 'development'
    ,port: 8181
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -86,8 +89,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -176,7 +181,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: 'localhost',
--- a/config/environments/production.js.example
+++ b/config/environments/production.js.example
@@ -2,6 +2,9 @@ var config = {
    environment: 'production'
    ,port: 8181
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -170,7 +175,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: 'localhost',
--- a/config/environments/staging.js.example
+++ b/config/environments/staging.js.example
@@ -2,6 +2,9 @@ var config = {
    environment: 'production'
    ,port: 8181
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -170,7 +175,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: 'localhost',
--- a/config/environments/test.js.example
+++ b/config/environments/test.js.example
@@ -2,6 +2,9 @@ var config = {
     environment: 'test'
    ,port: 8888
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -172,7 +177,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: '',
--- a/lib/cartodb/api/query_tables_api.js
+++ b/lib/cartodb/api/query_tables_api.js
@@ -14,7 +14,7 @@ module.exports = QueryTablesApi;

 QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {

-    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
+    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';

    this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback);
 };
@@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
        callback(new Error('could not fetch source tables: ' + msg));
        return;
    }
-    var qtables = rows[0].cdb_querytables;
-    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+
+    // This is an Array, so no need to split into parts
+    var tableNames = rows[0].cdb_querytablestext;
    callback(null, tableNames);
 }

@@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam

    var query = [
        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
+            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
        ')',
        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
        'FROM CDB_TableMetadata m',
@@ -54,8 +54,8 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {

    var result = rows[0];

-    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+    // This is an Array, so no need to split into parts
+    var tableNames = result.tablenames;

    var lastUpdatedTime = result.max || 0;

--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -1,6 +1,6 @@
 {
  "name": "windshaft-cartodb",
-  "version": "2.7.0",
+  "version": "2.9.0",
  "dependencies": {
    "cartodb-psql": {
      "version": "0.4.0",
@@ -139,14 +139,14 @@
                  "resolved": "https://registry.npmjs.org/async/-/async-1.3.0.tgz"
                },
                "mime-types": {
-                  "version": "2.1.2",
+                  "version": "2.1.3",
                  "from": "mime-types@^2.1.1",
-                  "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.2.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.3.tgz",
                  "dependencies": {
                    "mime-db": {
-                      "version": "1.14.0",
-                      "from": "mime-db@~1.14.0",
-                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.14.0.tgz"
+                      "version": "1.15.0",
+                      "from": "mime-db@~1.15.0",
+                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.15.0.tgz"
                    }
                  }
                }
@@ -276,9 +276,9 @@
              "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz",
              "dependencies": {
                "bluebird": {
-                  "version": "2.9.32",
+                  "version": "2.9.33",
                  "from": "bluebird@^2.9.30",
-                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.32.tgz"
+                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.33.tgz"
                },
                "chalk": {
                  "version": "1.1.0",
@@ -485,9 +485,9 @@
      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
    },
    "windshaft": {
-      "version": "0.47.0",
-      "from": "windshaft@0.47.0",
-      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.47.0.tgz",
+      "version": "0.48.0",
+      "from": "windshaft@0.48.0",
+      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.48.0.tgz",
      "dependencies": {
        "chronograph": {
          "version": "0.1.0",
@@ -2293,8 +2293,8 @@
        },
        "tilelive-mapnik": {
          "version": "0.6.15",
-          "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/cdb",
-          "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/cdb",
+          "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik",
+          "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik",
          "dependencies": {
            "generic-pool": {
              "version": "2.1.1",
@@ -2303,15 +2303,15 @@
            },
            "mime": {
              "version": "1.2.11",
-              "from": "mime@~1.2.9",
+              "from": "mime@~1.2.11",
              "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
            }
          }
        },
        "mapnik": {
-          "version": "1.4.15-cdb1",
-          "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb1",
-          "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb1",
+          "version": "1.4.15-cdb2",
+          "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb2",
+          "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb2",
          "dependencies": {
            "nan": {
              "version": "1.2.0",
@@ -2919,8 +2919,8 @@
        },
        "abaculus": {
          "version": "1.1.0",
-          "from": "https://github.com/CartoDB/abaculus/tarball/cdb",
-          "resolved": "https://github.com/CartoDB/abaculus/tarball/cdb"
+          "from": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik",
+          "resolved": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik"
        },
        "sphericalmercator": {
          "version": "1.0.2",
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
    "private": true,
    "name": "windshaft-cartodb",
-    "version": "2.7.0",
+    "version": "2.9.0",
    "description": "A map tile server for CartoDB",
    "keywords": [
      "cartodb"
@@ -24,7 +24,7 @@
    "dependencies": {
        "underscore"  : "~1.6.0",
        "dot": "~1.0.2",
-        "windshaft": "0.47.0",
+        "windshaft": "0.48.0",
        "step": "~0.0.5",
        "queue-async": "~1.0.7",
        "request": "~2.9.203",
Author	SHA1	Message	Date
Raul Ochoa	e3d5abc9a2	Release 2.9.0	2015-08-06 18:02:09 +02:00
Raul Ochoa	ea7a5da1c1	Merge pull request #324 from CartoDB/memory-stats Send memory usage stats	2015-08-06 16:12:18 +02:00
Raul Ochoa	2e063cc2d2	Send memory usage stats	2015-08-06 16:06:42 +02:00
Raul Ochoa	decd9077e4	Stubs next version	2015-07-15 11:44:59 +02:00
Raul Ochoa	f6c47bf85e	Release 2.8.0	2015-07-15 11:43:54 +02:00
Raul Ochoa	423191c13b	Merge pull request #319 from CartoDB/upgrade-windshaft Upgrades windshaft to 0.48.0	2015-07-15 11:43:09 +02:00
Raul Ochoa	7a5928d957	Upgrades windshaft to 0.48.0	2015-07-15 11:31:26 +02:00
Raul Ochoa	436c334f5a	Stubs next version	2015-07-14 16:33:12 +02:00
Raul Ochoa	4f84138ade	Release 2.7.2	2015-07-14 16:32:04 +02:00
Raul Ochoa	a0d86ac5dc	Update news	2015-07-14 16:28:51 +02:00
Raul Ochoa	e2be4f1275	Merge pull request #314 from CartoDB/use-cdb-query-tables-text Use CDB_QueryTablesText instead of CDB_QueryTables	2015-07-14 16:20:23 +02:00
Raul Ochoa	9a393fa793	Adds some notes about uv_threadpool_size and mapnik renderer pool size	2015-07-07 12:27:09 +02:00
Raul Ochoa	7614f72df6	Stubs next version	2015-07-06 11:58:08 +02:00
Raul Ochoa	ef2db78567	Release 2.7.1	2015-07-06 11:56:30 +02:00
Raul Ochoa	8708468444	fix 2.7.0 release date	2015-07-06 11:56:18 +02:00
Raul Ochoa	cd28a4fbcc	redis-mpool `noReadyCheck` and `unwatchOnRelease` options from config do not extend them as it disallows to pick from config	2015-07-06 11:52:34 +02:00
Raul Ochoa	19f488095b	Stubs next version	2015-07-06 01:07:30 +02:00
Raul Ochoa	9e30f05e7d	Reverts to use cdb branch as is already published	2015-06-29 16:46:07 +02:00
Raul Ochoa	2715f47a22	Points CDB_QueryTables script to the branch with CDB_QueryTablesText	2015-06-24 19:07:41 +02:00
Rafa de la Torre	90d0b23441	Use CDB_QueryTablesText instead of CDB_QueryTables This avoids trouble with len(schema.table_name) > 63 See https://github.com/CartoDB/cartodb-postgresql/issues/86	2015-06-24 15:43:04 +02:00