From ac153232d0a6a79dddc33b11c7129282d7e05779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 3 Nov 2017 17:21:01 +0100 Subject: [PATCH 1/9] mvt timeout error creation script --- assets/render-timeout-fallback.mvt | 1 + scripts/mvt-timeout-error.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 assets/render-timeout-fallback.mvt create mode 100755 scripts/mvt-timeout-error.py diff --git a/assets/render-timeout-fallback.mvt b/assets/render-timeout-fallback.mvt new file mode 100644 index 00000000..1ebfd46d --- /dev/null +++ b/assets/render-timeout-fallback.mvt @@ -0,0 +1 @@ +'\x1aZ\n\x14errorTileSquareLayer\x12\x1b\x12\x06\x00\x00\x01\x01\x02\x02\x18\x03"\x0f\t\x00\x80@\x1a\x00\xff?\x80@\x00\x00\x80@\x0f\x1a\x03foo\x1a\x03uid\x1a\x03cat"\x05\n\x03bar"\x02 {"\x06\n\x04flew(\x80 x\x01\x1aD\n\x03air\x12\x15\x12\x06\x00\x00\x01\x01\x02\x02\x18\x02"\t\t\xbe\x02\xb6\x03\n\x81\x1b\x00\x1a\x03foo\x1a\x03uid\x1a\x03cat"\x05\n\x03bar"\x03 \xd2\t"\x06\n\x04flew(\x80 x\x01' \ No newline at end of file diff --git a/scripts/mvt-timeout-error.py b/scripts/mvt-timeout-error.py new file mode 100755 index 00000000..d8028997 --- /dev/null +++ b/scripts/mvt-timeout-error.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +import mapbox_vector_tile + +tile = mapbox_vector_tile.encode([ + { + "name": "errorTileSquareLayer", + "features": [{ + "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))", + "properties":{ + "uid":123, + "foo":"bar", + "cat":"flew" + } + }] + }, + { + "name": "air", + "features": [{ + "geometry":"LINESTRING(159 3877, -1570 3877)", + "properties":{ + "uid":1234, + "foo":"bar", + "cat":"flew" + } + }] + } +]) + +with open('./assets/render-timeout-fallback.mvt', 'w+') as f: + f.write(repr(tile)) \ No newline at end of file From 33fbff5011b1e1e9a0b8263fb484c0703793f437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 11:06:53 +0100 Subject: [PATCH 2/9] some improvements mvt timeout error script --- assets/render-timeout-fallback.mvt | Bin 394 -> 180 bytes scripts/mvt-timeout-error.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/render-timeout-fallback.mvt b/assets/render-timeout-fallback.mvt index 1ebfd46de7ed738db8b9d37413b4ac9528df4557..ee74f5ac795a10a915d3dc315be31cdcc1ae359f 100644 GIT binary patch literal 180 zcmb1&;u1+MD#|Yk$;?R&E-XzfO7%&sOf3?U7Gh&yU}R)sl3-Ti=VWMbkYf06-{8Oi zWbsQer{(8MF_&hhNHHfTmMF1uF()M!DKRNjE3t90q~)ZRYcwcSFiM4Si6YD{DatHJ gEru8_3OAfdiIa05(>7+VMrj5d#xg5h;zThQ0Orpu1poj5 literal 394 zcmb79u@1r@6#SK@OC6!KO*b7J-E=Z64+}|?#zxXcj6d%^D4U6Kc-(u(Cwf^AzDAjZqSHZ$tt5~O_gu0EsuG9O3r?){;sBn~;M{&=vO>JZ zc3M$xo^!j7LvC-hux85^g)-!Ms*5*67sk>si=fLa> Date: Tue, 7 Nov 2017 11:07:38 +0100 Subject: [PATCH 3/9] verctor error middleware --- lib/cartodb/controllers/layergroup.js | 10 ++++++--- lib/cartodb/middleware/vector-error.js | 30 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 lib/cartodb/middleware/vector-error.js diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index c5e4c53a..573c53ee 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -7,6 +7,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); +var vectorError = require('../middleware/vector-error'); var DataviewBackend = require('../backends/dataview'); var AnalysisStatusBackend = require('../backends/analysis-status'); @@ -57,7 +58,8 @@ LayergroupController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.tile.bind(this) + this.tile.bind(this), + vectorError() ); app.get( @@ -65,7 +67,8 @@ LayergroupController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.tile.bind(this) + this.tile.bind(this), + vectorError() ); app.get( @@ -74,7 +77,8 @@ LayergroupController.prototype.register = function(app) { userMiddleware, validateLayerRouteMiddleware, this.prepareContext, - this.layer.bind(this) + this.layer.bind(this), + vectorError() ); app.get( diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js new file mode 100644 index 00000000..3da6ff3b --- /dev/null +++ b/lib/cartodb/middleware/vector-error.js @@ -0,0 +1,30 @@ +const fs = require('fs'); + +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt', {encoding: null}); + +module.exports = function vectorError() { + return function vectorErrorMiddleware(err, req, res, next) { + if(req.params.format === 'mvt') { + + if (isTimeoutError(err)) { + res.set('Content-Type', 'application/x-protobuf'); + return res.status(429).send(timeoutErrorVectorTile); + } + } + + next(err); + } +} + + +function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; +} + +function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); +} + +function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); +} \ No newline at end of file From cda9a09b8eda51e9e00455a9e6572be896bed830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 11:08:01 +0100 Subject: [PATCH 4/9] vector timeout error tests --- test/acceptance/user-database-timeout-limit.js | 17 +++++++++++------ test/acceptance/user-render-timeout-limit.js | 17 +++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 42c468fd..82934e65 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -411,7 +411,7 @@ describe('user database timeout limit', function () { this.testClient.setUserDatabaseTimeoutLimit(0, done); }); - it('"mvt" fails due to statement timeout', function (done) { + it('"mvt" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'mvt', @@ -419,18 +419,23 @@ describe('user database timeout limit', function () { response: { status: 429, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'application/x-protobuf' } } }; - this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); - - assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); - + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + done(); }); + }); }); }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 0bb45c2b..5942d045 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -232,7 +232,7 @@ describe('user render timeout limit', function () { response: { status: 429, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'application/x-protobuf' } } }; @@ -240,15 +240,12 @@ describe('user render timeout limit', function () { this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); - assert.deepEqual(tile, { - errors: ['You are over platform\'s limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details' - }] - }); - + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + done(); }); }); From 1ea4fc50c94d2912ac3f78d142c1f0214225c7be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 11:17:32 +0100 Subject: [PATCH 5/9] make jshint happy --- lib/cartodb/middleware/vector-error.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index 3da6ff3b..df9f170d 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -1,6 +1,9 @@ const fs = require('fs'); -const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt', {encoding: null}); +const timeoutErrorVectorTile = fs.readFileSync( + __dirname + '/../../../assets/render-timeout-fallback.mvt', + { encoding: null } +); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { @@ -13,8 +16,8 @@ module.exports = function vectorError() { } next(err); - } -} + }; +}; function isRenderTimeoutError (err) { From 4fc90db495b9191aa339905f2d109aae00548975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 15:57:33 +0100 Subject: [PATCH 6/9] timeout error vector with lines --- assets/render-timeout-fallback.mvt | Bin 180 -> 1113 bytes scripts/mvt-timeout-error.py | 40 +++++++++++++++++------------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/assets/render-timeout-fallback.mvt b/assets/render-timeout-fallback.mvt index ee74f5ac795a10a915d3dc315be31cdcc1ae359f..9ef336aeecc0d02037512d329f7115dd05e63ef9 100644 GIT binary patch literal 1113 zcmYM!!EO>!6vpu>MLP490!}H8BT^g@5fKp)5fOL*clrWd%%Vw?>ZpnC&cpbg$NAcR zad-du<=ouOx#yI8q}ZT(Z+C>4=a$ZYWjw(*bRDw!=_!`A*mTqCo^ zE7;24i$gM7yo4?Now$70ZHpJMg})Wo$!zf)HupE;2AM6M!Djwi+#<8ZQ`ppBiDNQb zJb_L8rMOLIi^s6BzYurGZ1D&-^5^1&%oYz}Lw_bt$!zfeHt?t7E}1RPV3|J=_sDE< zAJ+GCai7c<_h3DLEY8SmaTnJ0N8$mQEly#nKNJtiY;gih{DF8xW{W$pj-QFgWVW~s zYx}8qLS~C&SnT)3Q!-oJg0;LUo{`z&2CU&H;yIZuuEXknEMAb=V!+^$cu8i9Ls;lX z;uV=KuEA=4PrN3x#Z_3<55*fYTU>!v{6M@Vv+L!?hBy0M{sHlESw~S2luH!cQF%hS Q?sxatz8v&jaPv+41C0vn&j0`b literal 180 zcmb1&;u1+MD#|Yk$;?R&E-XzfO7%&sOf3?U7Gh&yU}R)sl3-Ti=VWMbkYf06-{8Oi zWbsQer{(8MF_&hhNHHfTmMF1uF()M!DKRNjE3t90q~)ZRYcwcSFiM4Si6YD{DatHJ gEru8_3OAfdiIa05(>7+VMrj5d#xg5h;zThQ0Orpu1poj5 diff --git a/scripts/mvt-timeout-error.py b/scripts/mvt-timeout-error.py index a6b764b6..1b2e66ee 100755 --- a/scripts/mvt-timeout-error.py +++ b/scripts/mvt-timeout-error.py @@ -2,30 +2,36 @@ import mapbox_vector_tile +lines_list = [] + +# main diagonal line +lines_list.append({ "geometry":"LINESTRING (0 0, 4096 4096)"}) + +# diagonal lines +for i in range(4096/32, 4096, 4096/32): + start = i + end = 4096 - i + + lines_list.append({ "geometry":"LINESTRING (0 " + str(start) + ", " + str(end) + " 4096)" }) + lines_list.append({ "geometry":"LINESTRING (" + str(start) + " 0, 4096 " + str(end) + ")" }) + +# box lines +lines_list.append({ "geometry":"LINESTRING (0 0, 0 4096)"}) +lines_list.append({ "geometry":"LINESTRING (0 4096, 4096 4096)"}) +lines_list.append({ "geometry":"LINESTRING (4096 4096, 4096 0)"}) +lines_list.append({ "geometry":"LINESTRING (4096 0, 0 0)"}) + + tile = mapbox_vector_tile.encode([ { "name": "errorTileSquareLayer", - "features": [{ - "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))", - "properties":{ - "uid":123, - "foo":"bar", - "cat":"flew" - } - }] + "features": [{ "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))" }] }, { "name": "errorTileStripesLayer", - "features": [{ - "geometry":"LINESTRING(159 3877, -1570 3877)", - "properties":{ - "uid":1234, - "foo":"bar", - "cat":"flew" - } - }] + "features": lines_list } ]) -with open('./assets/render-timeout-fallback.mvt', 'w') as f: +with open('./assets/render-timeout-fallback.mvt', 'w+') as f: f.write(tile) \ No newline at end of file From 970310bf7f4a3cffdeb257053886319064859479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 16:01:08 +0100 Subject: [PATCH 7/9] add vectorError middleware to named map endpoint --- lib/cartodb/controllers/named_maps.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 759ae4b6..442db927 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -9,6 +9,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); +var vectorError = require('../middleware/vector-error'); function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { @@ -33,7 +34,8 @@ NamedMapsController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.tile.bind(this) + this.tile.bind(this), + vectorError() ); app.get( @@ -108,6 +110,7 @@ NamedMapsController.prototype.tile = function(req, res, next) { }, function handleImage(err, tile, headers, stats) { req.profiler.add(stats); + if (err) { err.label = 'NAMED_MAP_TILE'; next(err); From 0bf36fa058bf362fba4176e93f46ae19562d80f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 16:02:37 +0100 Subject: [PATCH 8/9] remove unneeded encoding control --- lib/cartodb/middleware/vector-error.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index df9f170d..c4674cfd 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -1,9 +1,6 @@ const fs = require('fs'); -const timeoutErrorVectorTile = fs.readFileSync( - __dirname + '/../../../assets/render-timeout-fallback.mvt', - { encoding: null } -); +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt'); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { From 298882f4100c891cf9236835a78b49aed7245766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 15 Nov 2017 15:07:30 +0100 Subject: [PATCH 9/9] remove with spaces and adding line as EOF --- lib/cartodb/middleware/vector-error.js | 2 +- test/acceptance/user-database-timeout-limit.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index c4674cfd..f42f1c87 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -27,4 +27,4 @@ function isDatasourceTimeoutError (err) { function isTimeoutError (err) { return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); -} \ No newline at end of file +} diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 82934e65..9893f00a 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -411,7 +411,7 @@ describe('user database timeout limit', function () { this.testClient.setUserDatabaseTimeoutLimit(0, done); }); - it('"mvt" fails due to statement timeout', function (done) { + it('"mvt" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'mvt',