Release 2.8.0

Merge pull request #319 from CartoDB/upgrade-windshaft
Upgrades windshaft to 0.48.0
2015-07-15 11:43:54 +02:00 · 2015-07-15 11:43:09 +02:00 · 2015-07-15 11:31:26 +02:00 · 2015-07-14 16:33:12 +02:00 · 2015-07-14 16:32:04 +02:00 · 2015-07-14 16:28:51 +02:00
9 changed files with 130 additions and 87 deletions
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,52 @@
 # Changelog

+## 2.8.0
+
+Released 2015-07-15
+
+Announcements:
+ - Upgrades windshaft to [0.48.0](https://github.com/CartoDB/Windshaft/releases/tag/0.48.0)
+
+
+## 2.7.2
+
+Released 2015-07-14
+
+Enhancements:
+ - Replaces `CDB_QueryTables` with `CDB_QueryTablesText` to avoid issues with long schema+table names
+
+
+## 2.7.1
+
+Released 2015-07-06
+
+Bug fixes:
+ - redis-mpool `noReadyCheck` and `unwatchOnRelease` options from config and defaulted
+
+
+## 2.7.0
+
+Released 2015-07-06
+
+Announcements:
+ - Upgrades windshaft to [0.47.0](https://github.com/CartoDB/Windshaft/releases/tag/0.47.0)
+ - Upgrades redis-mpool to [0.4.0](https://github.com/CartoDB/node-redis-mpool/releases/tag/0.4.0)
+
+New features:
+ - Exposes redis `noReadyCheck` config
+
+Bug fixes:
+ - Fixes `unwatchOnRelease` redis config
+
+
+## 2.6.1
+
+Released 2015-07-02
+
+Announcements:
+ - Upgrades windshaft to [0.46.1](https://github.com/CartoDB/Windshaft/releases/tag/0.46.1)
+
+
 ## 2.6.0

 Released 2015-07-02
--- a/app.js
+++ b/app.js
@@ -65,8 +65,12 @@ if ( global.environment.log_filename ) {
 global.log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = global.log4js.getLogger();

-var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
-    redisPool = new RedisPool(redisOpts);
+var redisOpts = _.defaults(global.environment.redis, {
+    name: 'windshaft',
+    unwatchOnRelease: false,
+    noReadyCheck: true
+});
+var redisPool = new RedisPool(redisOpts);

 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
--- a/config/environments/development.js.example
+++ b/config/environments/development.js.example
@@ -2,6 +2,9 @@ var config = {
     environment: 'development'
    ,port: 8181
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -86,8 +89,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -176,7 +181,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: 'localhost',
--- a/config/environments/production.js.example
+++ b/config/environments/production.js.example
@@ -2,6 +2,9 @@ var config = {
    environment: 'production'
    ,port: 8181
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -170,7 +175,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: 'localhost',
--- a/config/environments/staging.js.example
+++ b/config/environments/staging.js.example
@@ -2,6 +2,9 @@ var config = {
    environment: 'production'
    ,port: 8181
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -170,7 +175,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: 'localhost',
--- a/config/environments/test.js.example
+++ b/config/environments/test.js.example
@@ -2,6 +2,9 @@ var config = {
     environment: 'test'
    ,port: 8888
    ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
    ,uv_threadpool_size: undefined
    // Regular expression pattern to extract username
    // from hostname. Must have a single grabbing block.
@@ -80,8 +83,10 @@ var config = {
      cache_ttl: 60000,
      statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
      mapnik: {
-          // The size of the pool of internal mapnik renderers
-          // Check the configuration of uv_threadpool_size to use suitable value
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
          poolSize: 8,

          // Metatile is the number of tiles-per-side that are going
@@ -172,7 +177,9 @@ var config = {
        },
        emitter: {
            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
    }
    ,varnish: {
        host: '',
--- a/lib/cartodb/api/query_tables_api.js
+++ b/lib/cartodb/api/query_tables_api.js
@@ -14,7 +14,7 @@ module.exports = QueryTablesApi;

 QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {

-    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
+    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';

    this.pgQueryRunner.run(username, query, handleAffectedTablesInQueryRows, callback);
 };
@@ -25,9 +25,9 @@ function handleAffectedTablesInQueryRows(err, rows, callback) {
        callback(new Error('could not fetch source tables: ' + msg));
        return;
    }
-    var qtables = rows[0].cdb_querytables;
-    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+
+    // This is an Array, so no need to split into parts
+    var tableNames = rows[0].cdb_querytablestext;
    callback(null, tableNames);
 }

@@ -35,7 +35,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam

    var query = [
        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
+            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
        ')',
        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
        'FROM CDB_TableMetadata m',
@@ -54,8 +54,8 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {

    var result = rows[0];

-    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
+    // This is an Array, so no need to split into parts
+    var tableNames = result.tablenames;

    var lastUpdatedTime = result.max || 0;

--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -1,6 +1,6 @@
 {
  "name": "windshaft-cartodb",
-  "version": "2.6.0",
+  "version": "2.8.0",
  "dependencies": {
    "cartodb-psql": {
      "version": "0.4.0",
@@ -139,14 +139,14 @@
                  "resolved": "https://registry.npmjs.org/async/-/async-1.3.0.tgz"
                },
                "mime-types": {
-                  "version": "2.1.2",
+                  "version": "2.1.3",
                  "from": "mime-types@^2.1.1",
-                  "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.2.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.3.tgz",
                  "dependencies": {
                    "mime-db": {
-                      "version": "1.14.0",
-                      "from": "mime-db@~1.14.0",
-                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.14.0.tgz"
+                      "version": "1.15.0",
+                      "from": "mime-db@~1.15.0",
+                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.15.0.tgz"
                    }
                  }
                }
@@ -180,9 +180,9 @@
              "resolved": "https://registry.npmjs.org/qs/-/qs-3.1.0.tgz"
            },
            "tunnel-agent": {
-              "version": "0.4.0",
+              "version": "0.4.1",
              "from": "tunnel-agent@~0.4.0",
-              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.0.tgz"
+              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.1.tgz"
            },
            "tough-cookie": {
              "version": "2.0.0",
@@ -276,9 +276,9 @@
              "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz",
              "dependencies": {
                "bluebird": {
-                  "version": "2.9.30",
+                  "version": "2.9.33",
                  "from": "bluebird@^2.9.30",
-                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.30.tgz"
+                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.33.tgz"
                },
                "chalk": {
                  "version": "1.1.0",
@@ -437,9 +437,8 @@
      "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
    },
    "redis-mpool": {
-      "version": "0.3.0",
-      "from": "redis-mpool@~0.3.0",
-      "resolved": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.3.0.tgz",
+      "version": "0.4.0",
+      "from": "redis-mpool@~0.4.0",
      "dependencies": {
        "generic-pool": {
          "version": "2.1.1",
@@ -486,9 +485,9 @@
      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
    },
    "windshaft": {
-      "version": "0.46.0",
-      "from": "windshaft@0.46.0",
-      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.46.0.tgz",
+      "version": "0.48.0",
+      "from": "windshaft@0.48.0",
+      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.48.0.tgz",
      "dependencies": {
        "chronograph": {
          "version": "0.1.0",
@@ -496,44 +495,9 @@
          "resolved": "git://github.com/CartoDB/chronographjs.git#0b8c35eee510cfa14a16be24d70533b38ecc1d2d"
        },
        "grainstore": {
-          "version": "0.23.0",
-          "from": "grainstore@~0.23.0",
-          "resolved": "https://registry.npmjs.org/grainstore/-/grainstore-0.23.0.tgz",
+          "version": "1.0.0",
+          "from": "grainstore@~1.0.0",
          "dependencies": {
-            "redis-mpool": {
-              "version": "0.1.0",
-              "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
-              "resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
-              "dependencies": {
-                "generic-pool": {
-                  "version": "2.1.1",
-                  "from": "generic-pool@~2.1.1",
-                  "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz"
-                },
-                "redis": {
-                  "version": "0.12.1",
-                  "from": "redis@~0.12.1",
-                  "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz"
-                },
-                "hiredis": {
-                  "version": "0.1.17",
-                  "from": "hiredis@~0.1.17",
-                  "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz",
-                  "dependencies": {
-                    "bindings": {
-                      "version": "1.2.1",
-                      "from": "bindings@*",
-                      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz"
-                    },
-                    "nan": {
-                      "version": "1.1.2",
-                      "from": "nan@~1.1.0",
-                      "resolved": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz"
-                    }
-                  }
-                }
-              }
-            },
            "carto": {
              "version": "0.9.5-cdb2",
              "from": "https://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
@@ -2329,8 +2293,8 @@
        },
        "tilelive-mapnik": {
          "version": "0.6.15",
-          "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/cdb",
-          "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/cdb",
+          "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik",
+          "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/upgrade-mapnik",
          "dependencies": {
            "generic-pool": {
              "version": "2.1.1",
@@ -2339,15 +2303,15 @@
            },
            "mime": {
              "version": "1.2.11",
-              "from": "mime@~1.2.9",
+              "from": "mime@~1.2.11",
              "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
            }
          }
        },
        "mapnik": {
-          "version": "1.4.15-cdb1",
-          "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb1",
-          "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb1",
+          "version": "1.4.15-cdb2",
+          "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb2",
+          "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb2",
          "dependencies": {
            "nan": {
              "version": "1.2.0",
@@ -2866,9 +2830,9 @@
              "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz"
            },
            "tunnel-agent": {
-              "version": "0.4.0",
+              "version": "0.4.1",
              "from": "tunnel-agent@~0.4.0",
-              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.0.tgz"
+              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.1.tgz"
            },
            "tough-cookie": {
              "version": "2.0.0",
@@ -2955,8 +2919,8 @@
        },
        "abaculus": {
          "version": "1.1.0",
-          "from": "https://github.com/CartoDB/abaculus/tarball/cdb",
-          "resolved": "https://github.com/CartoDB/abaculus/tarball/cdb"
+          "from": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik",
+          "resolved": "https://github.com/CartoDB/abaculus/tarball/upgrade-mapnik"
        },
        "sphericalmercator": {
          "version": "1.0.2",
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
    "private": true,
    "name": "windshaft-cartodb",
-    "version": "2.6.0",
+    "version": "2.8.0",
    "description": "A map tile server for CartoDB",
    "keywords": [
      "cartodb"
@@ -24,14 +24,14 @@
    "dependencies": {
        "underscore"  : "~1.6.0",
        "dot": "~1.0.2",
-        "windshaft": "0.46.0",
+        "windshaft": "0.48.0",
        "step": "~0.0.5",
        "queue-async": "~1.0.7",
        "request": "~2.9.203",
        "cartodb-redis": "~0.13.0",
        "cartodb-psql": "~0.4.0",
        "fastly-purge": "~1.0.0",
-        "redis-mpool": "~0.3.0",
+        "redis-mpool": "~0.4.0",
        "lzma": "~1.3.7",
        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
    },
Author	SHA1	Message	Date
Raul Ochoa	f6c47bf85e	Release 2.8.0	2015-07-15 11:43:54 +02:00
Raul Ochoa	423191c13b	Merge pull request #319 from CartoDB/upgrade-windshaft Upgrades windshaft to 0.48.0	2015-07-15 11:43:09 +02:00
Raul Ochoa	7a5928d957	Upgrades windshaft to 0.48.0	2015-07-15 11:31:26 +02:00
Raul Ochoa	436c334f5a	Stubs next version	2015-07-14 16:33:12 +02:00
Raul Ochoa	4f84138ade	Release 2.7.2	2015-07-14 16:32:04 +02:00
Raul Ochoa	a0d86ac5dc	Update news	2015-07-14 16:28:51 +02:00
Raul Ochoa	e2be4f1275	Merge pull request #314 from CartoDB/use-cdb-query-tables-text Use CDB_QueryTablesText instead of CDB_QueryTables	2015-07-14 16:20:23 +02:00
Raul Ochoa	9a393fa793	Adds some notes about uv_threadpool_size and mapnik renderer pool size	2015-07-07 12:27:09 +02:00
Raul Ochoa	7614f72df6	Stubs next version	2015-07-06 11:58:08 +02:00
Raul Ochoa	ef2db78567	Release 2.7.1	2015-07-06 11:56:30 +02:00
Raul Ochoa	8708468444	fix 2.7.0 release date	2015-07-06 11:56:18 +02:00
Raul Ochoa	cd28a4fbcc	redis-mpool `noReadyCheck` and `unwatchOnRelease` options from config do not extend them as it disallows to pick from config	2015-07-06 11:52:34 +02:00
Raul Ochoa	19f488095b	Stubs next version	2015-07-06 01:07:30 +02:00
Raul Ochoa	cd65c6dd0e	Release 2.7.0	2015-07-06 01:05:28 +02:00
Raul Ochoa	0c670cfdfd	Merge pull request #318 from CartoDB/upgrade-windshaft Upgrades windshaft to 0.47.0	2015-07-05 21:51:34 +02:00
Raul Ochoa	27ff1ac4f6	Upgrades windshaft to 0.47.0	2015-07-05 21:16:12 +02:00
Raul Ochoa	3f06de93f7	Merge pull request #317 from CartoDB/improve-redis Improve redis	2015-07-05 21:11:34 +02:00
Raul Ochoa	0da6495330	Fixes `unwatchOnRelease` redis config	2015-07-05 21:00:25 +02:00
Raul Ochoa	bf24347328	Exposes redis `noReadyCheck` config	2015-07-05 20:59:52 +02:00
Raul Ochoa	7a5d73f9df	Upgrades redis-mpool	2015-07-05 20:58:39 +02:00
Raul Ochoa	0d9f34fd48	Stubs next version	2015-07-02 15:48:58 +02:00
Raul Ochoa	da55a3bdd2	Release 2.6.1	2015-07-02 15:47:40 +02:00
Raul Ochoa	333334e598	Upgrades windshaft	2015-07-02 15:41:48 +02:00
Raul Ochoa	7168e4410c	Stubs next version	2015-07-02 14:12:41 +02:00
Raul Ochoa	9e30f05e7d	Reverts to use cdb branch as is already published	2015-06-29 16:46:07 +02:00
Raul Ochoa	2715f47a22	Points CDB_QueryTables script to the branch with CDB_QueryTablesText	2015-06-24 19:07:41 +02:00
Rafa de la Torre	90d0b23441	Use CDB_QueryTablesText instead of CDB_QueryTables This avoids trouble with len(schema.table_name) > 63 See https://github.com/CartoDB/cartodb-postgresql/issues/86	2015-06-24 15:43:04 +02:00