From 065f56e1619510f8f982912fc91c4b127f6ebbfb Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 7 Feb 2018 18:10:13 +0100 Subject: [PATCH 1/7] Improve the speed of the aggregation dataview Improve the performance of the aggregation dataview. Instead of using a CTE (WITH) for filtered_source, which is only used in one place to calculate ranks, inject it as a subquery. This way the planner has a chance to ignore uneeded columns as well as to parallelize the exectution of the window function (WindowAgg in the query plan). That is the part that takes most of the time of the query. The improvement is about 20-40% in speed on PG10 with 4 cores. --- lib/cartodb/models/dataview/aggregation.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index fed5e14e..0b0cc4c7 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -2,7 +2,7 @@ const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:aggregation'); const filteredQueryTpl = ctx => ` - filtered_source AS ( + ( SELECT * FROM (${ctx.query}) _cdb_filtered_source ${ctx.aggregationColumn && ctx.isFloatColumn ? ` @@ -14,7 +14,7 @@ const filteredQueryTpl = ctx => ` ${ctx.aggregationColumn} != 'NaN'::float` : '' } - ) + ) filtered_source `; const summaryQueryTpl = ctx => ` @@ -43,7 +43,7 @@ const rankedCategoriesQueryTpl = ctx => ` ${ctx.column} AS category, ${ctx.aggregationFn} AS value, row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank - FROM filtered_source + FROM ${filteredQueryTpl(ctx)} ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} GROUP BY ${ctx.column} ORDER BY 2 DESC @@ -134,7 +134,6 @@ const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn const aggregationDataviewQueryTpl = ctx => ` WITH - ${filteredQueryTpl(ctx)}, ${summaryQueryTpl(ctx)}, ${rankedCategoriesQueryTpl(ctx)}, ${categoriesSummaryMinMaxQueryTpl(ctx)}, From 3539b658fb79fcccec5594ef2abe548672baaef2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 12 Feb 2018 12:13:29 +0100 Subject: [PATCH 2/7] Update Windshaft to 4.5.2 --- NEWS.md | 2 +- package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9c9c0c9e..e03a601f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,7 +3,7 @@ ## 5.3.0 Released 2018-02-12 - Upgrades redis-mpool to 0.5.0 - - Upgrades windshaft to 4.5.1 + - Upgrades windshaft to 4.5.2 - Upgrades cartodb-redis to 0.15.0 - Adds metrics option to the Mapnik renderer - Upgrades camshadft to 0.61.2 diff --git a/package.json b/package.json index 371ccfa9..8e3d6917 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.5.1", + "windshaft": "4.5.2", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 29c0f15f..ca67ed6c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2367,9 +2367,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.5.1: - version "4.5.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.1.tgz#1be73b2d81cd8df648423ac219dae27406261aab" +windshaft@4.5.2: + version "4.5.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.2.tgz#d371af414fc4fbf98bd0f1aaabed91aa43224567" dependencies: "@carto/mapnik" "3.6.2-carto.2" "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1 From 251fe9650927a3bb1fd1553b3729996712244707 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 12 Feb 2018 19:24:53 +0100 Subject: [PATCH 3/7] Cosmetic fix, as suggested in PR --- lib/cartodb/models/dataview/aggregation.js | 26 ++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 0b0cc4c7..568d3cc2 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -2,19 +2,17 @@ const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:aggregation'); const filteredQueryTpl = ctx => ` - ( - SELECT * - FROM (${ctx.query}) _cdb_filtered_source - ${ctx.aggregationColumn && ctx.isFloatColumn ? ` - WHERE - ${ctx.aggregationColumn} != 'infinity'::float - AND - ${ctx.aggregationColumn} != '-infinity'::float - AND - ${ctx.aggregationColumn} != 'NaN'::float` : - '' - } - ) filtered_source + SELECT * + FROM (${ctx.query}) _cdb_filtered_source + ${ctx.aggregationColumn && ctx.isFloatColumn ? ` + WHERE + ${ctx.aggregationColumn} != 'infinity'::float + AND + ${ctx.aggregationColumn} != '-infinity'::float + AND + ${ctx.aggregationColumn} != 'NaN'::float` : + '' + } `; const summaryQueryTpl = ctx => ` @@ -43,7 +41,7 @@ const rankedCategoriesQueryTpl = ctx => ` ${ctx.column} AS category, ${ctx.aggregationFn} AS value, row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank - FROM ${filteredQueryTpl(ctx)} + FROM (${filteredQueryTpl(ctx)}) filtered_source ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} GROUP BY ${ctx.column} ORDER BY 2 DESC From b4e06ec1acfa68bd10fdcad78dc01b497a7fc8b3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 12 Feb 2018 19:35:34 +0100 Subject: [PATCH 4/7] Update NEWS.md and stub version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e03a601f..d4a88f33 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 5.3.1 +Released yyyy-mm-dd + - Improve the speed of the aggregation dataview #865 + ## 5.3.0 Released 2018-02-12 - Upgrades redis-mpool to 0.5.0 diff --git a/package.json b/package.json index 8e3d6917..b9540a97 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "5.3.0", + "version": "5.3.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 97972ac73fdf1495fff654f6b9b17f5130c600b8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 13 Feb 2018 09:05:06 +0100 Subject: [PATCH 5/7] Update NEWS.md with v5.3.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index d4a88f33..5260f222 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 5.3.1 -Released yyyy-mm-dd +Released 2018-02-13 - Improve the speed of the aggregation dataview #865 ## 5.3.0 From 0b475ab5e245d964cefb72224276db26689a20f0 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 13 Feb 2018 09:07:48 +0100 Subject: [PATCH 6/7] Stub next version --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5260f222..e330fa22 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 5.3.2 +Released yyyy-mm-dd + ## 5.3.1 Released 2018-02-13 - Improve the speed of the aggregation dataview #865 diff --git a/package.json b/package.json index b9540a97..f606a8bb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "5.3.1", + "version": "5.3.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 6b0ab45e6372bcc366406bd569b6c202473de579 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 13 Feb 2018 11:47:17 +0100 Subject: [PATCH 7/7] Update Windshaft to 4.5.3 --- NEWS.md | 1 + package.json | 2 +- yarn.lock | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index e330fa22..64305a89 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## 5.3.2 Released yyyy-mm-dd + - Upgrades Windshaft to 4.5.3 ## 5.3.1 Released 2018-02-13 diff --git a/package.json b/package.json index f606a8bb..09bfb4a8 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.5.2", + "windshaft": "4.5.3", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index ca67ed6c..26f8cc5b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2367,9 +2367,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.5.2: - version "4.5.2" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.2.tgz#d371af414fc4fbf98bd0f1aaabed91aa43224567" +windshaft@4.5.3: + version "4.5.3" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.3.tgz#45e792af06b224f78f44b6eb3b0ecb9d90dcc943" dependencies: "@carto/mapnik" "3.6.2-carto.2" "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1