diff --git a/NEWS.md b/NEWS.md index 16cb601e..92ef4bff 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,7 @@ Released 2017-mm-dd - Use express error handler middleware to respond in case of something went wrong. - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params. - Move `LZMA` decompression to its own middleware. + - Implement stats middleware removing some duplicated code while sending response. ## 4.0.0 diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 9a502bd4..b9c6ac1e 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -1,14 +1,9 @@ -var debug = require('debug')('windshaft:cartodb'); - function BaseController() { } module.exports = BaseController; -// jshint maxcomplexity:9 BaseController.prototype.send = function(req, res, body, status, headers) { - res.set('X-Tiler-Profiler', req.profiler.toJSONString()); - if (headers) { res.set(headers); } @@ -24,13 +19,4 @@ BaseController.prototype.send = function(req, res, body, status, headers) { } else { res.send(body); } - - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - debug("error sending profiling stats: " + err); - } }; -// jshint maxcomplexity:6 diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/middleware/error-middleware.js index fd19e24f..80457b9c 100644 --- a/lib/cartodb/middleware/error-middleware.js +++ b/lib/cartodb/middleware/error-middleware.js @@ -31,8 +31,6 @@ module.exports = function errorMiddleware (/* options */) { errors_with_context: allErrors.map(errorMessageWithContext) }; - res.set('X-Tiler-Profiler', req.profiler.toJSONString()); - res.status(statusCode); if (req.query && req.query.callback) { @@ -40,14 +38,6 @@ module.exports = function errorMiddleware (/* options */) { } else { res.json(errorResponseBody); } - - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - debug("error sending profiling stats: " + err); - } }; }; diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/middleware/stats.js new file mode 100644 index 00000000..489ee645 --- /dev/null +++ b/lib/cartodb/middleware/stats.js @@ -0,0 +1,27 @@ +const Profiler = require('../stats/profiler_proxy'); +const debug = require('debug')('windshaft:cartodb:stats'); +const onHeaders = require('on-headers'); + +module.exports = function statsMiddleware(options) { + const { enabled = true, statsClient } = options; + + return function stats(req, res, next) { + req.profiler = new Profiler({ + statsd_client: statsClient, + profile: enabled + }); + + onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString())); + + res.on('finish', () => { + try { + // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + debug("error sending profiling stats: " + err); + } + }); + + next(); + }; +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 1ce6439f..08ec1a66 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -12,7 +12,8 @@ var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); var FastlyCacheBackend = require('./cache/backend/fastly'); var StatsClient = require('./stats/client'); -var Profiler = require('./stats/profiler_proxy'); +const stats = require('./middleware/stats'); + var RendererStatsReporter = require('./stats/reporter/renderer'); var windshaft = require('windshaft'); @@ -362,11 +363,6 @@ function bootstrap(opts) { app.use(bodyParser.json()); app.use(function bootstrap$prepareRequestResponse(req, res, next) { - req.profiler = new Profiler({ - statsd_client: global.statsClient, - profile: opts.useProfiler - }); - if (global.environment && global.environment.api_hostname) { res.set('X-Served-By-Host', global.environment.api_hostname); } @@ -374,6 +370,11 @@ function bootstrap(opts) { next(); }); + app.use(stats({ + enabled: opts.useProfiler, + statsClient: global.statsClient + })); + app.use(lzmaMiddleware); // temporary measure until we upgrade to newer version expressjs so we can check err.status diff --git a/package.json b/package.json index 26f3ed73..3a52e3d9 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "lru-cache": "2.6.5", "lzma": "~2.3.2", "node-statsd": "~0.0.7", + "on-headers": "^1.0.1", "queue-async": "~1.0.7", "redis-mpool": "0.4.1", "request": "^2.83.0", diff --git a/yarn.lock b/yarn.lock index 6d36931f..db232491 100644 --- a/yarn.lock +++ b/yarn.lock @@ -114,11 +114,7 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - -aws4@^1.2.1, aws4@^1.6.0: +aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -163,18 +159,6 @@ boom@2.x.x: dependencies: hoek "2.x.x" -boom@4.x.x: - version "4.3.1" - resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" - dependencies: - hoek "4.x.x" - -boom@5.x.x: - version "5.2.0" - resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" - dependencies: - hoek "4.x.x" - brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -360,7 +344,7 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0: +commander@2.9.0, commander@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: @@ -406,12 +390,6 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" -cryptiles@3.x.x: - version "3.1.2" - resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" - dependencies: - boom "5.x.x" - d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -520,11 +498,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" - -dot@~1.0.2: +dot@^1.0.3, dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -644,7 +618,7 @@ express@~4.16.0: utils-merge "1.0.1" vary "~1.1.2" -extend@~3.0.0, extend@~3.0.1: +extend@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -652,10 +626,6 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" -fast-deep-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" - fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -780,7 +750,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1: +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -811,17 +781,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -873,13 +832,6 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" -har-validator@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" - dependencies: - ajv "^5.1.0" - har-schema "^2.0.0" - has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -903,15 +855,6 @@ hawk@3.1.3, hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" -hawk@~6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" - dependencies: - boom "4.x.x" - cryptiles "3.x.x" - hoek "4.x.x" - sntp "2.x.x" - hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -923,10 +866,6 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" -hoek@4.x.x: - version "4.2.0" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" - hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -958,14 +897,6 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -1092,10 +1023,6 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" -json-schema-traverse@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" - json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1319,17 +1246,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -mime@~1.3.4: - version "1.3.6" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" - -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8: +minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1337,10 +1260,6 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -minimist@~0.0.1: - version "0.0.10" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" - minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1476,7 +1395,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1, oauth-sign@~0.8.2: +oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1498,6 +1417,10 @@ on-finished@~2.3.0: dependencies: ee-first "1.1.1" +on-headers@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.1.tgz#928f5d0f470d49342651ea6794b0857c100693f7" + once@1.x, once@^1.3.0, once@^1.3.3: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -1583,10 +1506,6 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1775,7 +1694,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1: +readable-stream@1.1, readable-stream@~1.1.9: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1805,15 +1724,6 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@~1.1.9: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1923,9 +1833,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" @@ -2000,12 +1910,6 @@ sntp@1.x.x: dependencies: hoek "2.x.x" -sntp@2.x.x: - version "2.0.2" - resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" - dependencies: - hoek "4.x.x" - source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -2040,14 +1944,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4: +sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" -sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" - split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2121,7 +2021,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4, stringstream@~0.0.5: +stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2315,7 +2215,7 @@ utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" -uuid@^3.0.0, uuid@^3.1.0: +uuid@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"