From df931d95a32c69ed268e5fe508478665ea3d0b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 24 Oct 2019 17:34:30 +0200 Subject: [PATCH 01/18] using github:cartodb/windshaft#better-redis-logging --- package-lock.json | 123 +++++++++++++++++++++++----------------------- package.json | 2 +- 2 files changed, 62 insertions(+), 63 deletions(-) diff --git a/package-lock.json b/package-lock.json index f7ab113e..0781a3ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -353,9 +353,9 @@ }, "dependencies": { "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.5.tgz", + "integrity": "sha512-J9dlskqUXK1OeTOYBEn5s8aMukWMwWfs+rPTn/jn50Ux4MNXVhubL1wu/j2t+H4NVI+cXEcCaYellqaPVGXNqQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -496,9 +496,9 @@ "dev": true }, "chownr": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", - "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==" + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.3.tgz", + "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==" }, "chroma-js": { "version": "1.1.1", @@ -665,11 +665,11 @@ "integrity": "sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw==" }, "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", "requires": { - "mimic-response": "^1.0.0" + "mimic-response": "^2.0.0" } }, "deep-eql": { @@ -1128,11 +1128,11 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-minipass": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.6.tgz", - "integrity": "sha512-crhvyXcMejjv3Z5d2Fa9sf5xLYVCF5O1c71QxbVnbLsmYMBEvDAftewesN/HhY03YRoA7zOMxjNGrF5svGaaeQ==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", "requires": { - "minipass": "^2.2.1" + "minipass": "^2.6.0" } }, "fs.realpath": { @@ -2033,9 +2033,9 @@ } }, "graceful-fs": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", - "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==" + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" }, "grainstore": { "version": "2.0.1", @@ -2223,9 +2223,9 @@ "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" }, "hosted-git-info": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.4.tgz", - "integrity": "sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ==" + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz", + "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==" }, "htmlparser2": { "version": "3.8.3", @@ -2289,9 +2289,9 @@ } }, "ignore-walk": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.2.tgz", - "integrity": "sha512-EXyErtpHbn75ZTsOADsfx6J/FPo6/5cjev46PXrcTpd8z3BoRkXgYu9/JVqrI7tusjmwCZutGeRJeU0Wo1e4Cw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", "requires": { "minimatch": "^3.0.4" } @@ -2692,9 +2692,9 @@ "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.0.0.tgz", + "integrity": "sha512-8ilDoEapqA4uQ3TwS0jakGONKXVJqpy+RpM+3b7pLdOjghCrEiGp9SRkFbUHAmZW9vdnrENWHjaweIoTIJExSQ==" }, "minimatch": { "version": "3.0.4", @@ -2710,9 +2710,9 @@ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "minipass": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.6.0.tgz", - "integrity": "sha512-OuNZ0OHrrI+jswzmgivYBZ+fAAGHZA4293d5q0z631/I9QSw3yumKB92njxHIHiB1eAdGRsE+3CcOPkoEyV5FQ==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -2724,18 +2724,18 @@ "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" }, "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" } } }, "minizlib": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.2.tgz", - "integrity": "sha512-hR3At21uSrsjjDTWrbu0IMLTpnkpv8IIMFDFaoz43Tmu4LkmAXfH44vNNzpTnf+OAQQCHrb91y/wc2J4x5XgSQ==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", "requires": { - "minipass": "^2.2.1" + "minipass": "^2.9.0" } }, "mkdirp": { @@ -2891,9 +2891,9 @@ }, "dependencies": { "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.5.tgz", + "integrity": "sha512-J9dlskqUXK1OeTOYBEn5s8aMukWMwWfs+rPTn/jn50Ux4MNXVhubL1wu/j2t+H4NVI+cXEcCaYellqaPVGXNqQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2944,9 +2944,9 @@ "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==" }, "npm-packlist": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.4.tgz", - "integrity": "sha512-zTLo8UcVYtDU3gdeaFu2Xu0n0EvelfHDGuqtNIn5RO7yQj4H1TqNdBc/yZjxnWA0PVB8D3Woyp0i5B43JwQ6Vw==", + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.6.tgz", + "integrity": "sha512-u65uQdb+qwtGvEJh/DgQgW1Xg7sqeNbmxYyrvlNznaVTjV3E5P6F/EFjM+BVHXl7JJlsdG8A64M0XI8FI/IOlg==", "requires": { "ignore-walk": "^3.0.1", "npm-bundled": "^1.0.1" @@ -3632,11 +3632,11 @@ "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" }, "simple-get": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.0.3.tgz", - "integrity": "sha512-Wvre/Jq5vgoz31Z9stYWPLn0PqRqmBDpFSdypAnHu5AvRVCYPRYGnvryNLiXu8GOBNDH82J2FRHUGMjjHUpXFw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", "requires": { - "decompress-response": "^3.3.0", + "decompress-response": "^4.2.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } @@ -3813,13 +3813,13 @@ } }, "tar": { - "version": "4.4.10", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz", - "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==", + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", "requires": { "chownr": "^1.1.1", "fs-minipass": "^1.2.5", - "minipass": "^2.3.5", + "minipass": "^2.8.6", "minizlib": "^1.2.1", "mkdirp": "^0.5.0", "safe-buffer": "^5.1.2", @@ -3832,9 +3832,9 @@ "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" }, "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" } } }, @@ -4032,9 +4032,8 @@ "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU=" }, "windshaft": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-5.6.3.tgz", - "integrity": "sha512-ProHyEDICqIhTkNouT9elj5FP9DCtApEFo7SvPlZbkeYygn1OGLN89Rq0/2WuGK5Y6AbX8INa5iYGxdGYaWKqQ==", + "version": "github:cartodb/windshaft#9db16751947f0c455aa6f464a95fd7dd6fea0f67", + "from": "github:cartodb/windshaft#better-redis-logging", "requires": { "@carto/cartonik": "^0.7.0", "@carto/mapnik": "3.6.2-carto.16", @@ -4051,9 +4050,9 @@ }, "dependencies": { "cartodb-query-tables": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.6.1.tgz", - "integrity": "sha512-hQR9F5tQ6W6uGZk8Us/0fwkAsvYfbsHUzyKqBqDYue+jOa7FrGS+KpWokdLYHhR/ye3N3iR9RBTrIkwp6aoUww==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.6.3.tgz", + "integrity": "sha512-ijHl2Roh+0B1pP8SL3guEAu8tE6yNN3J/oxdUWCFOSKjHmXjwTzyJdjO+tONGcERmlWfS594SCFYElGIweSnQg==", "requires": { "decimal.js": "10.2.0" } @@ -4207,9 +4206,9 @@ }, "dependencies": { "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.5.tgz", + "integrity": "sha512-J9dlskqUXK1OeTOYBEn5s8aMukWMwWfs+rPTn/jn50Ux4MNXVhubL1wu/j2t+H4NVI+cXEcCaYellqaPVGXNqQ==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", diff --git a/package.json b/package.json index 8db23c86..3cf0c366 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "step-profiler": "0.3.0", "turbo-carto": "0.21.2", "underscore": "1.6.0", - "windshaft": "^5.6.3", + "windshaft": "github:cartodb/windshaft#better-redis-logging", "yargs": "11.1.0" }, "devDependencies": { From 080f93f6de65a3bf40f476ddb89df80843688bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 24 Oct 2019 17:34:46 +0200 Subject: [PATCH 02/18] passing logger to MapStore --- lib/api/api-router.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/api/api-router.js b/lib/api/api-router.js index 9349a515..fe08c10d 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -85,7 +85,8 @@ module.exports = class ApiRouter { const mapStore = new windshaft.storage.MapStore({ pool: redisPool, - expire_time: serverOptions.grainstore.default_layergroup_ttl + expire_time: serverOptions.grainstore.default_layergroup_ttl, + logger: global.logger }); const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); From dd5825c770658837b8a7d772478978b8b062dba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 28 Oct 2019 11:55:43 +0100 Subject: [PATCH 03/18] using a new logger --- lib/api/api-router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-router.js b/lib/api/api-router.js index fe08c10d..f72081be 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -86,7 +86,7 @@ module.exports = class ApiRouter { const mapStore = new windshaft.storage.MapStore({ pool: redisPool, expire_time: serverOptions.grainstore.default_layergroup_ttl, - logger: global.logger + logger: global.log4js.getLogger('[windshaft]') }); const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); From db03bcdf8faa281005441aac7d6fdd4d4e11ae25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 28 Oct 2019 15:41:46 +0100 Subject: [PATCH 04/18] windshaft logger by config param --- config/environments/development.js.example | 1 + config/environments/production.js.example | 1 + config/environments/staging.js.example | 1 + config/environments/test.js.example | 1 + lib/api/api-router.js | 3 ++- 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 7004c9ec..c7c3b97e 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: undefined + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>' diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 3e75a897..5b6a632b 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: 'logs/node-windshaft.log' + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'cartodb_user_<%= user_id %>' diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 49e08ac7..1aa59d02 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: 'logs/node-windshaft.log' + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>' diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 04797ae9..d8207730 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -87,6 +87,7 @@ var config = { // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal ,log_filename: '/tmp/node-windshaft.log' + ,log_windshaft: true // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>' diff --git a/lib/api/api-router.js b/lib/api/api-router.js index f72081be..1c74420a 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -83,10 +83,11 @@ module.exports = class ApiRouter { const metadataBackend = cartodbRedis({ pool: redisPool }); const pgConnection = new PgConnection(metadataBackend); + const windshaftLogger = environmentOptions.log_windshaft ? global.log4js.getLogger('[windshaft]') : null const mapStore = new windshaft.storage.MapStore({ pool: redisPool, expire_time: serverOptions.grainstore.default_layergroup_ttl, - logger: global.log4js.getLogger('[windshaft]') + logger: windshaftLogger }); const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); From c4a75de0d885c1a186e0e247875647397724c45e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 28 Oct 2019 18:15:15 +0100 Subject: [PATCH 05/18] windshaft 5.6.4 --- lib/api/api-router.js | 2 +- package-lock.json | 15 ++++++++------- package.json | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/api/api-router.js b/lib/api/api-router.js index 1c74420a..68b790e2 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -83,7 +83,7 @@ module.exports = class ApiRouter { const metadataBackend = cartodbRedis({ pool: redisPool }); const pgConnection = new PgConnection(metadataBackend); - const windshaftLogger = environmentOptions.log_windshaft ? global.log4js.getLogger('[windshaft]') : null + const windshaftLogger = environmentOptions.log_windshaft ? global.log4js.getLogger('[windshaft]') : null; const mapStore = new windshaft.storage.MapStore({ pool: redisPool, expire_time: serverOptions.grainstore.default_layergroup_ttl, diff --git a/package-lock.json b/package-lock.json index 0781a3ad..83734830 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,7 +37,7 @@ "resolved": "https://registry.npmjs.org/@carto/mapnik/-/mapnik-3.6.2-carto.16.tgz", "integrity": "sha512-RX8ov5EpEheToESVKiKnV5yMPLA2KxaX2ANAs9W4856oKFPdbGmB2buDz54mLhwBDfler9GVo0Bzr2ayRVLO2A==", "requires": { - "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#v1.6.1-carto.2", + "mapnik-vector-tile": "github:cartodb/mapnik-vector-tile#e7ca5471f9e5de81243e6035e70444321fc0a82f", "nan": "2.14.0", "node-pre-gyp": "0.13.0" } @@ -422,7 +422,7 @@ "integrity": "sha512-myLV2xo3q9oTT8m8M+c+UTD/ziDN7hrYtZ9yY00KvMnu2NsVeRQsTe8Yxq1GVS8vF9iYfcelwjVEGObPUdLtHw==", "requires": { "debug": "^3.1.0", - "pg": "github:cartodb/node-postgres#6.4.2-cdb2", + "pg": "github:cartodb/node-postgres#5417d7b29b7272ca2e71bb396899ab3f177a9ae6", "underscore": "~1.6.0" } }, @@ -2045,7 +2045,7 @@ "carto": "0.16.3", "debug": "~3.1.0", "generic-pool": "~2.2.0", - "millstone": "github:cartodb/millstone#v0.6.17-carto.3", + "millstone": "github:cartodb/millstone#eeeb308fba4586343bb848fbf8ae0d180192627d", "postcss": "~5.2.8", "postcss-scss": "0.4.0", "postcss-strip-inline-comments": "0.1.5", @@ -3848,7 +3848,7 @@ "resolved": "https://registry.npmjs.org/torque.js/-/torque.js-3.1.1.tgz", "integrity": "sha512-kfIrmI7TGqJT/J9DH8Mgvd9VEwcvAtnvyYyqymSN6WZ5L4BaVQEQ+zu5FgLChNAqCaRkqGc7bKp0Hj9A0rempA==", "requires": { - "carto": "github:cartodb/carto#master", + "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", "d3": "3.5.17", "turbo-carto": "^0.21.1", "turf-jenks": "~1.0.1" @@ -4032,13 +4032,14 @@ "integrity": "sha1-tDFbtCFKPXBY6+7okuE/ok2YsHU=" }, "windshaft": { - "version": "github:cartodb/windshaft#9db16751947f0c455aa6f464a95fd7dd6fea0f67", - "from": "github:cartodb/windshaft#better-redis-logging", + "version": "5.6.4", + "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-5.6.4.tgz", + "integrity": "sha512-W+SbKM5CjpuPXMrLUXzZms6yDk0aQeIs3eAd75Ih6SYwiJo12/vxkHqMe+0KZnVj6JE/oe53tVfWHRibE70NJA==", "requires": { "@carto/cartonik": "^0.7.0", "@carto/mapnik": "3.6.2-carto.16", "canvas": "^2.4.1", - "carto": "github:cartodb/carto#0.15.1-cdb5", + "carto": "github:cartodb/carto#85881d99dd7fcf2c4e16478b04db67108d27a50c", "cartodb-psql": "^0.14.0", "cartodb-query-tables": "^0.6.1", "debug": "3.1.0", diff --git a/package.json b/package.json index 3cf0c366..95c7a566 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "step-profiler": "0.3.0", "turbo-carto": "0.21.2", "underscore": "1.6.0", - "windshaft": "github:cartodb/windshaft#better-redis-logging", + "windshaft": "5.6.4", "yargs": "11.1.0" }, "devDependencies": { From 8a02156ac0c8fdc8ef481aea056bb0f0476ceba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 29 Oct 2019 10:31:31 +0100 Subject: [PATCH 06/18] fix tests --- lib/api/api-router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/api-router.js b/lib/api/api-router.js index 68b790e2..f5af6adf 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -83,7 +83,7 @@ module.exports = class ApiRouter { const metadataBackend = cartodbRedis({ pool: redisPool }); const pgConnection = new PgConnection(metadataBackend); - const windshaftLogger = environmentOptions.log_windshaft ? global.log4js.getLogger('[windshaft]') : null; + const windshaftLogger = environmentOptions.log_windshaft && global.log4js ? global.log4js.getLogger('[windshaft]') : null; const mapStore = new windshaft.storage.MapStore({ pool: redisPool, expire_time: serverOptions.grainstore.default_layergroup_ttl, From 3e7c29498972f2a8b82c4be466727506475d978a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 29 Oct 2019 11:32:27 +0100 Subject: [PATCH 07/18] linter --- lib/api/api-router.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/api/api-router.js b/lib/api/api-router.js index f5af6adf..142c3eba 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -83,7 +83,9 @@ module.exports = class ApiRouter { const metadataBackend = cartodbRedis({ pool: redisPool }); const pgConnection = new PgConnection(metadataBackend); - const windshaftLogger = environmentOptions.log_windshaft && global.log4js ? global.log4js.getLogger('[windshaft]') : null; + const windshaftLogger = environmentOptions.log_windshaft && global.log4js ? + global.log4js.getLogger('[windshaft]') : + null; const mapStore = new windshaft.storage.MapStore({ pool: redisPool, expire_time: serverOptions.grainstore.default_layergroup_ttl, From 0c572b5947d66d0ba8f53b4901bd08c0ef05fcfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 29 Oct 2019 11:37:16 +0100 Subject: [PATCH 08/18] NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index f3cde6ab..b4c1dd42 100644 --- a/NEWS.md +++ b/NEWS.md @@ -13,6 +13,7 @@ Announcements: - Upgraded cartodb-query-tables to version [0.7.0](https://github.com/CartoDB/node-cartodb-query-tables/blob/0.7.0/NEWS.md#version-0.7.0). - Be able to set a coherent TTL in Cache-Control header to expire all resources belonging to a map simultaneously. - When `cache buster` in request path is `0` set header `Last-Modified` to now, it avoids stalled content in 3rd party cache providers when they add `If-Modified-Since` header into the request. +- Adding a logger to MapStore (#1134) ## 7.2.0 Released 2019-09-30 From acecb88efb10bb2fc3adf7596f0d49fdff841514 Mon Sep 17 00:00:00 2001 From: Esther Lozano Date: Mon, 11 Nov 2019 18:14:30 +0100 Subject: [PATCH 09/18] Validate aggregation method is either sum or count --- lib/models/dataview/overviews/aggregation.js | 22 +++ test/acceptance/dataviews/overviews-test.js | 188 +++++++++++++++++++ 2 files changed, 210 insertions(+) diff --git a/lib/models/dataview/overviews/aggregation.js b/lib/models/dataview/overviews/aggregation.js index 1d0f7de6..bc0b9116 100644 --- a/lib/models/dataview/overviews/aggregation.js +++ b/lib/models/dataview/overviews/aggregation.js @@ -94,6 +94,8 @@ var CATEGORIES_LIMIT = 6; function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); + this._checkOptions(options); + this.query = query; this.queries = queries; this.column = options.column; @@ -219,6 +221,26 @@ var aggregationFnQueryTpl = { sum: dot.template('sum({{=it._aggregationColumn}}*_feature_count)') }; +const VALID_OPERATIONS = { + count: [], + sum: ['aggregationColumn'] +}; + +Aggregation.prototype._checkOptions = function (options) { + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error(`Aggregation does not support '${options.aggregation}' operation in dataview overview options`); + } + + const requiredOptions = VALID_OPERATIONS[options.aggregation]; + const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + + if (missingOptions.length > 0) { + throw new Error( + `Aggregation '${options.aggregation}' is missing some options for overview: ${missingOptions.join(',')}` + ); + } +}; + Aggregation.prototype.getAggregationSql = function() { return aggregationFnQueryTpl[this.aggregation]({ _aggregationFn: this.aggregation, diff --git a/test/acceptance/dataviews/overviews-test.js b/test/acceptance/dataviews/overviews-test.js index 8df298b3..ea688882 100644 --- a/test/acceptance/dataviews/overviews-test.js +++ b/test/acceptance/dataviews/overviews-test.js @@ -665,6 +665,194 @@ describe('dataviews using tables with overviews', function() { }); }); }); + + describe.only('agreggation validation', function (){ + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + function createMapConfig(options) { + return { + version: '1.5.0', + analyses: [ + { id: 'data-source', + type: 'source', + params: { + query: 'select * from test_table_overviews' + } + }, + { + id: 'data-source-special-float-values', + type: 'source', + params: { + query: 'select * from test_special_float_values_table_overviews' + } + } + ], + dataviews: { + test_invalid_aggregation: { + type: 'aggregation', + source: {id: 'data-source'}, + options: options + } + }, + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { id: 'data-source' } + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_special_float_values_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { + id: 'data-source-special-float-values' + } + } + } + ] + }; + } + + it('should fail if missing column', function (done) { + var options = { + aggregation: "sum", + aggregationColumn: "value" + }; + var missingCOlumnMapConfig = createMapConfig(options); + + var testClient = new TestClient(missingCOlumnMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation expects 'column' in dataview options"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation expects 'column' in dataview options" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if no aggregation operation', function (done) { + var options = { + column: "value", + aggregationColumn: "value" + }; + var missingOperationMapConfig = createMapConfig(options); + + var testClient = new TestClient(missingOperationMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation expects 'aggregation' operation in dataview options"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation expects 'aggregation' operation in dataview options" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if fake operation', function (done) { + var options = { + column: "value", + aggregation: "wadus", + aggregationColumn: "value" + }; + var wrongOperationMapConfig = createMapConfig(options); + + var testClient = new TestClient(wrongOperationMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation does not support 'wadus' operation"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation does not support 'wadus' operation" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if invalid operation for overview', function (done) { + var options = { + column: "value", + aggregation: "avg", + aggregationColumn: "value" + }; + var wrongOperationMapConfig = createMapConfig(options); + + var testClient = new TestClient(wrongOperationMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation does not support 'avg' operation in dataview overview options"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation does not support 'avg' operation in dataview overview options" + }] + }); + + testClient.drain(done); + }); + }); + + it('should fail if no aggregation column when needed', function (done) { + var options = { + column: "value", + aggregation: "sum" + }; + var missingOptionMapConfig = createMapConfig(options); + + var testClient = new TestClient(missingOptionMapConfig); + testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { + if (err) { + return done(err); + } + + assert.deepStrictEqual(dataview, { + errors: ["Aggregation 'sum' is missing some options: aggregationColumn"], + errors_with_context: [{ + type: 'unknown', + message: "Aggregation 'sum' is missing some options: aggregationColumn" + }] + }); + + testClient.drain(done); + }); + }); + }); }); }); From 2b5ed2120734a51969b847364a6b4b378eeffd04 Mon Sep 17 00:00:00 2001 From: Esther Lozano Date: Tue, 12 Nov 2019 12:37:24 +0100 Subject: [PATCH 10/18] Remove only in tests :P --- test/acceptance/dataviews/overviews-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/overviews-test.js b/test/acceptance/dataviews/overviews-test.js index ea688882..0bcf4049 100644 --- a/test/acceptance/dataviews/overviews-test.js +++ b/test/acceptance/dataviews/overviews-test.js @@ -666,7 +666,7 @@ describe('dataviews using tables with overviews', function() { }); }); - describe.only('agreggation validation', function (){ + describe('agreggation validation', function (){ const params = { response: { status: 400, From b05740048c413fb6b9243998a7b1fa12b8f10358 Mon Sep 17 00:00:00 2001 From: Esther Lozano Date: Tue, 12 Nov 2019 12:48:22 +0100 Subject: [PATCH 11/18] Update NEWS.md --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 058c65bc..b8980a70 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,7 +7,7 @@ Breaking changes: - Schema change for "routes" in configuration file, each "router" is now an array instead of an object. See [`dd06de2`](https://github.com/CartoDB/Windshaft-cartodb/pull/1126/commits/dd06de2632661e19d64c9fbc2be0ba1a8059f54c) for more details. Announcements: - +- Added validation to only allow "count" and "sum" aggregations in dataview overview. - Added mechanism to inject custom middlewares through configuration. - Stop requiring unused config properties: "base_url", "base_url_mapconfig", and "base_url_templated". From 3c586caba4eab9f8061678115435c6f21b3287c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 12 Nov 2019 13:10:35 +0100 Subject: [PATCH 12/18] Qualify calls to cartodb extension so having it in the search_path isn't necessary --- NEWS.md | 1 + lib/backends/dataview.js | 2 +- lib/backends/overviews-metadata.js | 6 +++--- lib/server-options.js | 4 ++-- lib/utils/query-utils.js | 2 +- test/support/sql/windshaft.test.sql | 7 +++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index b4c1dd42..ba5096d1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,7 @@ Announcements: - Be able to set a coherent TTL in Cache-Control header to expire all resources belonging to a map simultaneously. - When `cache buster` in request path is `0` set header `Last-Modified` to now, it avoids stalled content in 3rd party cache providers when they add `If-Modified-Since` header into the request. - Adding a logger to MapStore (#1134) +- Qualify calls to cartodb extension so having it in the search_path isn't necessary. ## 7.2.0 Released 2019-09-30 diff --git a/lib/backends/dataview.js b/lib/backends/dataview.js index 86ead292..d37e3d56 100644 --- a/lib/backends/dataview.js +++ b/lib/backends/dataview.js @@ -8,7 +8,7 @@ var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory const dbParamsFromReqParams = require('../utils/database-params'); var OverviewsQueryRewriter = require('../utils/overviews-query-rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' + zoom_level: 'cartodb.CDB_ZoomFromScale(!scale_denominator!)' }); var dot = require('dot'); diff --git a/lib/backends/overviews-metadata.js b/lib/backends/overviews-metadata.js index f066c4ef..8b7f0122 100644 --- a/lib/backends/overviews-metadata.js +++ b/lib/backends/overviews-metadata.js @@ -12,9 +12,9 @@ OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sq // FIXME: Currently using internal function _cdb_schema_name // CDB_Overviews should provide the schema information directly. const query = ` - SELECT *, _cdb_schema_name(base_table) - FROM CDB_Overviews( - CDB_QueryTablesText($windshaft$${queryUtils.substituteDummyTokens(sql)}$windshaft$) + SELECT *, cartodb._cdb_schema_name(base_table) + FROM cartodb.CDB_Overviews( + cartodb.CDB_QueryTablesText($windshaft$${queryUtils.substituteDummyTokens(sql)}$windshaft$) ); `; this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) { diff --git a/lib/server-options.js b/lib/server-options.js index ff011bac..dd138419 100644 --- a/lib/server-options.js +++ b/lib/server-options.js @@ -31,11 +31,11 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, { }); rendererConfig.mapnik.queryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' + zoom_level: 'cartodb.CDB_ZoomFromScale(!scale_denominator!)' }); rendererConfig.mvt.queryRewriter = new OverviewsQueryRewriter({ - zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' + zoom_level: 'cartodb.CDB_ZoomFromScale(!scale_denominator!)' }); // Perform keyword substitution in statsd diff --git a/lib/utils/query-utils.js b/lib/utils/query-utils.js index bc7c0307..bd5a4a76 100644 --- a/lib/utils/query-utils.js +++ b/lib/utils/query-utils.js @@ -8,7 +8,7 @@ module.exports.getQueryActualRowCount = function (query) { }; function getQueryRowEstimation(query) { - return 'select CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows'; + return 'select cartodb.CDB_EstimateRowCount($windshaft$' + substituteDummyTokens(query) + '$windshaft$) as rows'; } module.exports.getQueryRowEstimation = getQueryRowEstimation; diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index d1bf16eb..e336ffce 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -18,14 +18,13 @@ SET default_with_oids = false; -- public user role DROP USER IF EXISTS :PUBLICUSER; CREATE USER :PUBLICUSER WITH PASSWORD ':PUBLICPASS'; -GRANT USAGE ON SCHEMA cartodb TO :PUBLICUSER; -GRANT ALL ON CDB_TableMetadata TO :PUBLICUSER; +SELECT current_setting('search_path') AS my_path \gset +ALTER ROLE :PUBLICUSER SET search_path = :my_path, cartodb; -- db owner role DROP USER IF EXISTS :TESTUSER; CREATE USER :TESTUSER WITH PASSWORD ':TESTPASS'; -GRANT USAGE ON SCHEMA cartodb TO :TESTUSER; -GRANT ALL ON CDB_TableMetadata TO :TESTUSER; +ALTER ROLE :TESTUSER SET search_path = :my_path, cartodb; -- regular user role 1 DROP USER IF EXISTS test_windshaft_regular1; From aa09c079f608e3cff7af2c0e31eb19c3e6bded53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 12 Nov 2019 13:12:01 +0100 Subject: [PATCH 13/18] Fix multiple DB login issues --- NEWS.md | 1 + lib/backends/pg-connection.js | 10 ++-------- lib/backends/pg-query-runner.js | 9 ++------- .../adapter/mapconfig-named-layers-adapter.js | 7 ++++++- lib/models/mapconfig/provider/named-map-provider.js | 2 +- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/NEWS.md b/NEWS.md index ba5096d1..5a999be9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -15,6 +15,7 @@ Announcements: - When `cache buster` in request path is `0` set header `Last-Modified` to now, it avoids stalled content in 3rd party cache providers when they add `If-Modified-Since` header into the request. - Adding a logger to MapStore (#1134) - Qualify calls to cartodb extension so having it in the search_path isn't necessary. +- Fix multiple DB login issues. ## 7.2.0 Released 2019-09-30 diff --git a/lib/backends/pg-connection.js b/lib/backends/pg-connection.js index 9c7ed4c3..fa6fb542 100644 --- a/lib/backends/pg-connection.js +++ b/lib/backends/pg-connection.js @@ -3,6 +3,7 @@ var PSQL = require('cartodb-psql'); var _ = require('underscore'); const debug = require('debug')('cachechan'); +const dbParamsFromReqParams = require('../utils/database-params'); function PgConnection(metadataBackend) { this.metadataBackend = metadataBackend; @@ -124,14 +125,7 @@ PgConnection.prototype.getConnection = function(username, callback) { if (err) { return callback(err); } - return callback(err, new PSQL({ - user: databaseParams.dbuser, - pass: databaseParams.dbpass, - host: databaseParams.dbhost, - port: databaseParams.dbport, - dbname: databaseParams.dbname - })); - + return callback(err, new PSQL(dbParamsFromReqParams(databaseParams))); }); }; diff --git a/lib/backends/pg-query-runner.js b/lib/backends/pg-query-runner.js index c96041c4..c6a7d2b4 100644 --- a/lib/backends/pg-query-runner.js +++ b/lib/backends/pg-query-runner.js @@ -1,6 +1,7 @@ 'use strict'; var PSQL = require('cartodb-psql'); +const dbParamsFromReqParams = require('../utils/database-params'); function PgQueryRunner(pgConnection) { this.pgConnection = pgConnection; @@ -22,13 +23,7 @@ PgQueryRunner.prototype.run = function(username, query, callback) { return callback(err); } - const psql = new PSQL({ - user: databaseParams.dbuser, - pass: databaseParams.dbpass, - host: databaseParams.dbhost, - port: databaseParams.dbport, - dbname: databaseParams.dbname - }); + const psql = new PSQL(dbParamsFromReqParams(databaseParams)); psql.query(query, function (err, resultSet) { resultSet = resultSet || {}; diff --git a/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js b/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index 68d2a950..39721b41 100644 --- a/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js +++ b/lib/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -92,7 +92,12 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC layers.push(layer); if (layersResult.datasource) { datasourceBuilder.withLayerDatasource(currentLayerIndex, { - user: dbAuth.dbuser + user: dbAuth.dbuser, + // Used internally (PSQL) + pass: dbAuth.dbpassword, + dbpassword: dbAuth.dbpassword, + // Used by Mapnik + password: dbAuth.dbpassword }); } currentLayerIndex++; diff --git a/lib/models/mapconfig/provider/named-map-provider.js b/lib/models/mapconfig/provider/named-map-provider.js index 4b235986..e49893a9 100644 --- a/lib/models/mapconfig/provider/named-map-provider.js +++ b/lib/models/mapconfig/provider/named-map-provider.js @@ -242,7 +242,7 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider { } dbParams.dbuser = databaseParams.dbuser; - dbParams.dbpass = databaseParams.dbpass; + dbParams.dbpassword = databaseParams.dbpassword; dbParams.dbhost = databaseParams.dbhost; dbParams.dbport = databaseParams.dbport; dbParams.dbname = databaseParams.dbname; From bb745b0318caf2e54a35e8f4b4c574c22a553b52 Mon Sep 17 00:00:00 2001 From: Esther Lozano Date: Tue, 12 Nov 2019 14:48:37 +0100 Subject: [PATCH 14/18] Update test/acceptance/dataviews/overviews-test.js Co-Authored-By: Daniel G. Aubert --- test/acceptance/dataviews/overviews-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/overviews-test.js b/test/acceptance/dataviews/overviews-test.js index 0bcf4049..64a071f9 100644 --- a/test/acceptance/dataviews/overviews-test.js +++ b/test/acceptance/dataviews/overviews-test.js @@ -731,7 +731,7 @@ describe('dataviews using tables with overviews', function() { aggregation: "sum", aggregationColumn: "value" }; - var missingCOlumnMapConfig = createMapConfig(options); + var missingColumnMapConfig = createMapConfig(options); var testClient = new TestClient(missingCOlumnMapConfig); testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { From 75583f67c54e8cbd96d0366f5da3cae32f8c28a3 Mon Sep 17 00:00:00 2001 From: Esther Lozano Date: Tue, 12 Nov 2019 14:54:25 +0100 Subject: [PATCH 15/18] Use last version for map config in tests --- test/acceptance/dataviews/overviews-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/dataviews/overviews-test.js b/test/acceptance/dataviews/overviews-test.js index 64a071f9..c75433e1 100644 --- a/test/acceptance/dataviews/overviews-test.js +++ b/test/acceptance/dataviews/overviews-test.js @@ -678,7 +678,7 @@ describe('dataviews using tables with overviews', function() { function createMapConfig(options) { return { - version: '1.5.0', + version: '1.8.0', analyses: [ { id: 'data-source', type: 'source', @@ -733,7 +733,7 @@ describe('dataviews using tables with overviews', function() { }; var missingColumnMapConfig = createMapConfig(options); - var testClient = new TestClient(missingCOlumnMapConfig); + var testClient = new TestClient(missingColumnMapConfig); testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) { if (err) { return done(err); From f17411916ffd0905bc7fbf812558e5e4beece7fa Mon Sep 17 00:00:00 2001 From: Esther Lozano Date: Tue, 12 Nov 2019 17:43:03 +0100 Subject: [PATCH 16/18] Remove unnecessary config in tests --- test/acceptance/dataviews/overviews-test.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/acceptance/dataviews/overviews-test.js b/test/acceptance/dataviews/overviews-test.js index c75433e1..aa97a28f 100644 --- a/test/acceptance/dataviews/overviews-test.js +++ b/test/acceptance/dataviews/overviews-test.js @@ -685,13 +685,6 @@ describe('dataviews using tables with overviews', function() { params: { query: 'select * from test_table_overviews' } - }, - { - id: 'data-source-special-float-values', - type: 'source', - params: { - query: 'select * from test_special_float_values_table_overviews' - } } ], dataviews: { @@ -710,17 +703,6 @@ describe('dataviews using tables with overviews', function() { cartocss_version: '2.3.0', source: { id: 'data-source' } } - }, - { - type: 'mapnik', - options: { - sql: 'select * from test_special_float_values_table_overviews', - cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', - cartocss_version: '2.3.0', - source: { - id: 'data-source-special-float-values' - } - } } ] }; From df10cfe641f2cb00305c7e3b5f0d23b13d9fb125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Nov 2019 13:07:00 +0100 Subject: [PATCH 17/18] Release 8.0.0 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e2fe80ca..fe3d85f5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 8.0.0 -Released 2019-mm-dd +Released 2019-11-13 Breaking changes: - Schema change for "routes" in configuration file, each "router" is now an array instead of an object. See [`dd06de2`](https://github.com/CartoDB/Windshaft-cartodb/pull/1126/commits/dd06de2632661e19d64c9fbc2be0ba1a8059f54c) for more details. From ccd01e6da5864efee169b2264b97914e1f8f50e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 13 Nov 2019 13:17:47 +0100 Subject: [PATCH 18/18] Stubs next version --- NEWS.md | 3 +++ package-lock.json | 2 +- package.json | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index fe3d85f5..830b7de4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 8.0.1 +Released 2019-mm-dd + ## 8.0.0 Released 2019-11-13 diff --git a/package-lock.json b/package-lock.json index 83734830..80265391 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "8.0.0", + "version": "8.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 95c7a566..d10dc853 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "8.0.0", + "version": "8.0.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb"