Release 2.7.1

do not extend them as it disallows to pick from config

NEWS.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## 2.7.1
+
+Released 2015-07-06
+
+Bug fixes:
+ - redis-mpool `noReadyCheck` and `unwatchOnRelease` options from config and defaulted
+
+
+## 2.7.0
+
+Released 2015-07-06
+
+Announcements:
+ - Upgrades windshaft to [0.47.0](https://github.com/CartoDB/Windshaft/releases/tag/0.47.0)
+ - Upgrades redis-mpool to [0.4.0](https://github.com/CartoDB/node-redis-mpool/releases/tag/0.4.0)
+
+New features:
+ - Exposes redis `noReadyCheck` config
+
+Bug fixes:
+ - Fixes `unwatchOnRelease` redis config
+
+
+## 2.6.1
+
+Released 2015-07-02
+
+Announcements:
+ - Upgrades windshaft to [0.46.1](https://github.com/CartoDB/Windshaft/releases/tag/0.46.1)
+
+
+## 2.6.0
+
+Released 2015-07-02
+
+Announcements:
+ - Upgrades windshaft to [0.46.0](https://github.com/CartoDB/Windshaft/releases/tag/0.46.0)
+ - New config to set metatile by format
+
+
 ## 2.5.0
 
 Released 2015-06-18

README.md

@@ -59,6 +59,14 @@ happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
 ```npm install``` again.
 
+Upgrading
+---------
+
+Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following:
+
+```
+rm -rf node_modules; npm install
+```
 
 Run
 ---

app.js

@@ -65,8 +65,12 @@ if ( global.environment.log_filename ) {
 global.log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = global.log4js.getLogger();
 
-var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
-    redisPool = new RedisPool(redisOpts);
+var redisOpts = _.defaults(global.environment.redis, {
+    name: 'windshaft',
+    unwatchOnRelease: false,
+    noReadyCheck: true
+});
+var redisPool = new RedisPool(redisOpts);
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28

config/environments/development.js.example

@@ -96,6 +96,12 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
           // Buffer size is the tickness in pixel of a buffer
           // around the rendered (meta?)tile.
           //
@@ -170,7 +176,9 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
     ,varnish: {
         host: 'localhost',

config/environments/production.js.example

@@ -90,6 +90,12 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
           // Buffer size is the tickness in pixel of a buffer
           // around the rendered (meta?)tile.
           //
@@ -164,7 +170,9 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
     ,varnish: {
         host: 'localhost',

config/environments/staging.js.example

@@ -90,6 +90,12 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
           // Buffer size is the tickness in pixel of a buffer
           // around the rendered (meta?)tile.
           //
@@ -164,7 +170,9 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
     ,varnish: {
         host: 'localhost',

config/environments/test.js.example

@@ -90,6 +90,12 @@ var config = {
           // wasted time.
           metatile: 2,
 
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
           // Buffer size is the tickness in pixel of a buffer
           // around the rendered (meta?)tile.
           //
@@ -166,7 +172,9 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
     ,varnish: {
         host: '',

docs/Map-API.md

@@ -880,6 +880,8 @@ By manipulating the `"urlTemplate"` custom basemaps can be used in generating st
 
 **CartoDB**
 
+As described in the [Mapconfig documentation](https://github.com/CartoDB/Windshaft/blob/0.44.1/doc/MapConfig-1.3.0.md), a "cartodb" type layer is now just an alias to a "mapnik" type layer as above, intended for backwards compatibility.
+
 ```javascript
     {
       "type": "cartodb",

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "windshaft-cartodb",
-  "version": "2.5.0",
+  "version": "2.7.1",
   "dependencies": {
     "cartodb-psql": {
       "version": "0.4.0",
@@ -134,19 +134,19 @@
               "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc1.tgz",
               "dependencies": {
                 "async": {
-                  "version": "1.2.1",
+                  "version": "1.3.0",
                   "from": "async@^1.2.1",
-                  "resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz"
+                  "resolved": "https://registry.npmjs.org/async/-/async-1.3.0.tgz"
                 },
                 "mime-types": {
-                  "version": "2.1.1",
+                  "version": "2.1.2",
                   "from": "mime-types@^2.1.1",
-                  "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.1.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.2.tgz",
                   "dependencies": {
                     "mime-db": {
-                      "version": "1.13.0",
-                      "from": "mime-db@~1.13.0",
-                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.13.0.tgz"
+                      "version": "1.14.0",
+                      "from": "mime-db@~1.14.0",
+                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.14.0.tgz"
                     }
                   }
                 }
@@ -180,9 +180,9 @@
               "resolved": "https://registry.npmjs.org/qs/-/qs-3.1.0.tgz"
             },
             "tunnel-agent": {
-              "version": "0.4.0",
+              "version": "0.4.1",
               "from": "tunnel-agent@~0.4.0",
-              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.0.tgz"
+              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.1.tgz"
             },
             "tough-cookie": {
               "version": "2.0.0",
@@ -271,24 +271,24 @@
               "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
             },
             "har-validator": {
-              "version": "1.7.1",
+              "version": "1.8.0",
               "from": "har-validator@^1.6.1",
-              "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.7.1.tgz",
+              "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-1.8.0.tgz",
               "dependencies": {
                 "bluebird": {
-                  "version": "2.9.30",
-                  "from": "bluebird@^2.9.26",
-                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.30.tgz"
+                  "version": "2.9.32",
+                  "from": "bluebird@^2.9.30",
+                  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.32.tgz"
                 },
                 "chalk": {
-                  "version": "1.0.0",
+                  "version": "1.1.0",
                   "from": "chalk@^1.0.0",
-                  "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.0.0.tgz",
+                  "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.0.tgz",
                   "dependencies": {
                     "ansi-styles": {
-                      "version": "2.0.1",
-                      "from": "ansi-styles@^2.0.1",
-                      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.0.1.tgz"
+                      "version": "2.1.0",
+                      "from": "ansi-styles@^2.1.0",
+                      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz"
                     },
                     "escape-string-regexp": {
                       "version": "1.0.3",
@@ -296,38 +296,33 @@
                       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.3.tgz"
                     },
                     "has-ansi": {
-                      "version": "1.0.3",
-                      "from": "has-ansi@^1.0.3",
-                      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-1.0.3.tgz",
+                      "version": "2.0.0",
+                      "from": "has-ansi@^2.0.0",
+                      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
                       "dependencies": {
                         "ansi-regex": {
-                          "version": "1.1.1",
-                          "from": "ansi-regex@^1.1.0",
-                          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-1.1.1.tgz"
-                        },
-                        "get-stdin": {
-                          "version": "4.0.1",
-                          "from": "get-stdin@^4.0.1",
-                          "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz"
+                          "version": "2.0.0",
+                          "from": "ansi-regex@^2.0.0",
+                          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
                         }
                       }
                     },
                     "strip-ansi": {
-                      "version": "2.0.1",
-                      "from": "strip-ansi@^2.0.1",
-                      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-2.0.1.tgz",
+                      "version": "3.0.0",
+                      "from": "strip-ansi@^3.0.0",
+                      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz",
                       "dependencies": {
                         "ansi-regex": {
-                          "version": "1.1.1",
-                          "from": "ansi-regex@^1.0.0",
-                          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-1.1.1.tgz"
+                          "version": "2.0.0",
+                          "from": "ansi-regex@^2.0.0",
+                          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
                         }
                       }
                     },
                     "supports-color": {
-                      "version": "1.3.1",
-                      "from": "supports-color@^1.3.0",
-                      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-1.3.1.tgz"
+                      "version": "2.0.0",
+                      "from": "supports-color@^2.0.0",
+                      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz"
                     }
                   }
                 },
@@ -442,9 +437,8 @@
       "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz"
     },
     "redis-mpool": {
-      "version": "0.3.0",
-      "from": "redis-mpool@~0.3.0",
-      "resolved": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.3.0.tgz",
+      "version": "0.4.0",
+      "from": "redis-mpool@~0.4.0",
       "dependencies": {
         "generic-pool": {
           "version": "2.1.1",
@@ -481,9 +475,9 @@
       "resolved": "https://registry.npmjs.org/request/-/request-2.9.203.tgz"
     },
     "step": {
-      "version": "0.0.5",
+      "version": "0.0.6",
       "from": "step@~0.0.5",
-      "resolved": "https://registry.npmjs.org/step/-/step-0.0.5.tgz"
+      "resolved": "https://registry.npmjs.org/step/-/step-0.0.6.tgz"
     },
     "underscore": {
       "version": "1.6.0",
@@ -491,9 +485,9 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
     },
     "windshaft": {
-      "version": "0.45.0",
-      "from": "windshaft@0.45.0",
-      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.45.0.tgz",
+      "version": "0.47.0",
+      "from": "windshaft@0.47.0",
+      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-0.47.0.tgz",
       "dependencies": {
         "chronograph": {
           "version": "0.1.0",
@@ -501,44 +495,9 @@
           "resolved": "git://github.com/CartoDB/chronographjs.git#0b8c35eee510cfa14a16be24d70533b38ecc1d2d"
         },
         "grainstore": {
-          "version": "0.23.0",
-          "from": "grainstore@~0.23.0",
-          "resolved": "https://registry.npmjs.org/grainstore/-/grainstore-0.23.0.tgz",
+          "version": "1.0.0",
+          "from": "grainstore@~1.0.0",
           "dependencies": {
-            "redis-mpool": {
-              "version": "0.1.0",
-              "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
-              "resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
-              "dependencies": {
-                "generic-pool": {
-                  "version": "2.1.1",
-                  "from": "generic-pool@~2.1.1",
-                  "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz"
-                },
-                "redis": {
-                  "version": "0.12.1",
-                  "from": "redis@~0.12.1",
-                  "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz"
-                },
-                "hiredis": {
-                  "version": "0.1.17",
-                  "from": "hiredis@~0.1.17",
-                  "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz",
-                  "dependencies": {
-                    "bindings": {
-                      "version": "1.2.1",
-                      "from": "bindings@*",
-                      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz"
-                    },
-                    "nan": {
-                      "version": "1.1.2",
-                      "from": "nan@~1.1.0",
-                      "resolved": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz"
-                    }
-                  }
-                }
-              }
-            },
             "carto": {
               "version": "0.9.5-cdb2",
               "from": "https://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
@@ -2871,9 +2830,9 @@
               "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz"
             },
             "tunnel-agent": {
-              "version": "0.4.0",
+              "version": "0.4.1",
               "from": "tunnel-agent@~0.4.0",
-              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.0.tgz"
+              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.1.tgz"
             },
             "tough-cookie": {
               "version": "2.0.0",

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.5.0",
+    "version": "2.7.1",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,14 +24,14 @@
     "dependencies": {
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "0.45.0",
+        "windshaft": "0.47.0",
         "step": "~0.0.5",
         "queue-async": "~1.0.7",
         "request": "~2.9.203",
         "cartodb-redis": "~0.13.0",
         "cartodb-psql": "~0.4.0",
         "fastly-purge": "~1.0.0",
-        "redis-mpool": "~0.3.0",
+        "redis-mpool": "~0.4.0",
         "lzma": "~1.3.7",
         "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
     },

test/acceptance/multilayer_server.js

@@ -256,6 +256,41 @@ describe('tests from old api translated to multilayer', function() {
         );
     });
 
+    // https://github.com/CartoDB/cartodb-postgresql/issues/86
+    it.skip("should not fail with long table names because table name length limit",  function(done) {
+        var tableName = 'long_table_name_with_enough_chars_to_break_querytables_function';
+        var expectedCacheChannel = _.template('<%= databaseName %>:public.<%= tableName %>', {
+            databaseName: _.template(global.environment.postgres_auth_user, {user_id:1}) + '_db',
+            tableName: tableName
+        });
+
+        var layergroup =  singleLayergroupConfig('select * from ' + tableName, '#layer { marker-fill: red; }');
+
+        assert.response(server,
+            {
+                url: layergroupUrl + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+                method: 'GET',
+                headers: {
+                    host: 'localhost'
+                }
+            },
+            {
+                status: 200
+            },
+            function(res) {
+                var parsed = JSON.parse(res.body);
+                assert.ok(parsed.layergroupid);
+
+                assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
+                assert.equal(res.headers['x-cache-channel'], expectedCacheChannel);
+
+                assert.equal(res.headers['x-layergroup-id'], parsed.layergroupid);
+
+                done();
+            }
+        );
+    });
+
     it("creates layergroup fails when postgresql queries fail to figure affected tables in query",  function(done) {
 
         var runQueryFn = PgQueryRunner.prototype.run;

test/support/sql/CDB_QueryTables.sql

@@ -2,12 +2,12 @@
 --
 -- Requires PostgreSQL 9.x+
 --
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
+CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
+RETURNS text[]
 AS $$
 DECLARE
   exp XML;
-  tables NAME[];
+  tables text[];
   rec RECORD;
   rec2 RECORD;
 BEGIN
@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x)::name as p, unnest(s)::name as sc from inp
+      SELECT unnest(x) as p, unnest(s) as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name);
+      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;
@@ -65,3 +65,14 @@ BEGIN
   return tables;
 END
 $$ LANGUAGE 'plpgsql' VOLATILE STRICT;
+
+
+-- Keep CDB_QueryTables with same signature for backwards compatibility.
+-- It should probably be removed in the future.
+CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
+RETURNS name[]
+AS $$
+BEGIN
+  RETURN CDB_QueryTablesText(query)::name[];
+END
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

test/support/sql/windshaft.test.sql

@@ -189,3 +189,31 @@ INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('test_table_private_
 
 -- GRANT SELECT ON CDB_TableMetadata TO :PUBLICUSER;
 GRANT SELECT ON CDB_TableMetadata TO :TESTUSER;
+
+-- long name table
+CREATE TABLE
+long_table_name_with_enough_chars_to_break_querytables_function
+(
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry
+);
+
+INSERT INTO long_table_name_with_enough_chars_to_break_querytables_function SELECT * from test_table;
+
+ALTER TABLE ONLY long_table_name_with_enough_chars_to_break_querytables_function
+    ADD CONSTRAINT long_table_name_with_enough_chars_to_break_querytables_func_pkey PRIMARY KEY (cartodb_id);
+
+CREATE INDEX long_table_name_the_geom_idx
+    ON long_table_name_with_enough_chars_to_break_querytables_function USING gist (the_geom);
+CREATE INDEX long_table_name_the_geom_webmercator_idx
+    ON long_table_name_with_enough_chars_to_break_querytables_function USING gist (the_geom_webmercator);
+
+GRANT ALL ON TABLE long_table_name_with_enough_chars_to_break_querytables_function TO :TESTUSER;
+GRANT SELECT ON TABLE long_table_name_with_enough_chars_to_break_querytables_function TO :PUBLICUSER;
+
+INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('long_table_name_with_enough_chars_to_break_querytables_function'::regclass, '2009-02-13T23:31:30.123Z');

tools/README.md

@@ -0,0 +1,8 @@
+Deprecated tools
+================
+
+All tools and scripts found in this directory are deprecated and no longer maintained.
+
+Use at your own peril.
+
+In future releases they might get removed.