From 1d199f8713c8d34ed574d272ed65213c0222277f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Jul 2018 15:19:53 +0200 Subject: [PATCH 01/29] Remove step in method --- lib/cartodb/backends/dataview.js | 79 ++++++++++++++++---------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 4eebb2a2..4a79612f 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -21,51 +21,52 @@ function DataviewBackend(analysisBackend) { module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { + const dataviewName = params.dataviewName; - var dataviewName = params.dataviewName; - step( - function getMapConfig() { - mapConfigProvider.getMapConfig(this); - }, - function runDataviewQuery(err, mapConfig) { - assert.ifError(err); + mapConfigProvider.getMapConfig(function (err, mapConfig) { + if (err) { + return callback(err); + } - var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); - if (!dataviewDefinition) { - throw new Error("Dataview '" + dataviewName + "' does not exists"); - } + var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!dataviewDefinition) { + throw new Error("Dataview '" + dataviewName + "' does not exists"); + } - var ownFilter = +params.own_filter; - var noFilters = +params.no_filters; - if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) { - err = new Error(); - err.message = 'Both own_filter and no_filters cannot be sent in the same request'; - err.type = 'dataview'; - err.http_status = 400; + var ownFilter = +params.own_filter; + var noFilters = +params.no_filters; + if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) { + err = new Error(); + err.message = 'Both own_filter and no_filters cannot be sent in the same request'; + err.type = 'dataview'; + err.http_status = 400; + return callback(err); + } + + var pg = new PSQL(dbParamsFromReqParams(params)); + + var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); + + var dataviewFactory = DataviewFactoryWithOverviews.getFactory( + overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } + ); + + var dataview = dataviewFactory.getDataview(query, dataviewDefinition); + + dataview.getResult(pg, getOverrideParams(params, !!ownFilter), function (err, dataview) { + if (err) { return callback(err); } - var pg = new PSQL(dbParamsFromReqParams(params)); - - var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); - if (params.bbox) { - var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); - query = bboxFilter.sql(query); - } - - var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); - - var dataviewFactory = DataviewFactoryWithOverviews.getFactory( - overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } - ); - - var dataview = dataviewFactory.getDataview(query, dataviewDefinition); - dataview.getResult(pg, getOverrideParams(params, !!ownFilter), this); - }, - function returnCallback(err, result) { - return callback(err, result); - } - ); + return callback(null, dataview); + }); + }); }; function getDataviewQuery(dataviewDefinition, ownFilter, noFilters) { From 60cd91f1446e96203131687050ffe71fb29ddb1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 15:50:41 +0200 Subject: [PATCH 02/29] indentation --- lib/cartodb/backends/filter-stats.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index ba427405..08861b21 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -24,8 +24,8 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) { } FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { - var stats = {}; - var self = this; + var stats = {}; + var self = this; step( function getUnfilteredRows() { getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); From 94b7353fbf20553633a19b1ab2597d3c77a8f14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Jul 2018 15:52:04 +0200 Subject: [PATCH 03/29] Fix uncaught exception --- lib/cartodb/backends/dataview.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 4a79612f..c62d6bb7 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -59,7 +59,15 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param var dataview = dataviewFactory.getDataview(query, dataviewDefinition); - dataview.getResult(pg, getOverrideParams(params, !!ownFilter), function (err, dataview) { + let overrideParams; + + try { + overrideParams = getOverrideParams(params, !!ownFilter); + } catch (error) { + return callback(error); + } + + dataview.getResult(pg, overrideParams, function (err, dataview) { if (err) { return callback(err); } From eeafad7cd97d4f1bc6b79ba87656c4c6a79fa461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 15:52:41 +0200 Subject: [PATCH 04/29] remove step --- lib/cartodb/backends/filter-stats.js | 60 ++++++++++++++-------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 08861b21..7aa3adc7 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -1,5 +1,4 @@ var _ = require('underscore'); -var step = require('step'); var AnalysisFilter = require('../models/filter/analysis'); function FilterStatsBackends(pgQueryRunner) { @@ -26,34 +25,33 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) { FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { var stats = {}; var self = this; - step( - function getUnfilteredRows() { - getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); - }, - function receiveUnfilteredRows(err, rows) { - if (err){ - callback(err); - return; - } - stats.unfiltered_rows = rows; - this(null, rows); - }, - function getFilteredRows() { - if ( filters && !_.isEmpty(filters)) { - var analysisFilter = new AnalysisFilter(filters); - var query = analysisFilter.sql(unfiltered_query); - getEstimatedRows(self.pgQueryRunner, username, query, this); - } else { - this(null, null); - } - }, - function receiveFilteredRows(err, rows) { - if (err){ - callback(err); - return; - } - stats.filtered_rows = rows; - callback(null, stats); - } - ); + + function getUnfilteredRows() { + getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); + }, + function receiveUnfilteredRows(err, rows) { + if (err){ + callback(err); + return; + } + stats.unfiltered_rows = rows; + this(null, rows); + }, + function getFilteredRows() { + if ( filters && !_.isEmpty(filters)) { + var analysisFilter = new AnalysisFilter(filters); + var query = analysisFilter.sql(unfiltered_query); + getEstimatedRows(self.pgQueryRunner, username, query, this); + } else { + this(null, null); + } + }, + function receiveFilteredRows(err, rows) { + if (err){ + callback(err); + return; + } + stats.filtered_rows = rows; + callback(null, stats); + } }; From bafeceeb6ee4fe7ebfdadd993f067901dd42f0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 15:54:26 +0200 Subject: [PATCH 05/29] remove step: first funcion --- lib/cartodb/backends/filter-stats.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 7aa3adc7..de41da3f 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -26,17 +26,14 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu var stats = {}; var self = this; - function getUnfilteredRows() { - getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); - }, - function receiveUnfilteredRows(err, rows) { + getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, function (err, rows) { if (err){ callback(err); return; } stats.unfiltered_rows = rows; this(null, rows); - }, + }) function getFilteredRows() { if ( filters && !_.isEmpty(filters)) { var analysisFilter = new AnalysisFilter(filters); From 2196a89ec3b27bcf7b36885635f014e4bdb7623a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 15:55:24 +0200 Subject: [PATCH 06/29] remove step: 2 function refactor --- lib/cartodb/backends/filter-stats.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index de41da3f..2afd5ffc 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -28,11 +28,10 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, function (err, rows) { if (err){ - callback(err); - return; + return callback(err); } + stats.unfiltered_rows = rows; - this(null, rows); }) function getFilteredRows() { if ( filters && !_.isEmpty(filters)) { From fb683e438d291f5a728c7d74d4f8a8e3d13d8c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 15:56:18 +0200 Subject: [PATCH 07/29] remove step: second function --- lib/cartodb/backends/filter-stats.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 2afd5ffc..d2ffe6ba 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -32,8 +32,7 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu } stats.unfiltered_rows = rows; - }) - function getFilteredRows() { + if ( filters && !_.isEmpty(filters)) { var analysisFilter = new AnalysisFilter(filters); var query = analysisFilter.sql(unfiltered_query); @@ -41,7 +40,7 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu } else { this(null, null); } - }, + }) function receiveFilteredRows(err, rows) { if (err){ callback(err); From 6be1a77a29e01f4e17496166d6098e6b2ce175e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Jul 2018 15:57:47 +0200 Subject: [PATCH 08/29] Use callback to return the error --- lib/cartodb/backends/dataview.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index c62d6bb7..b4eee7fa 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -30,7 +30,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); if (!dataviewDefinition) { - throw new Error("Dataview '" + dataviewName + "' does not exists"); + return callback(new Error("Dataview '" + dataviewName + "' does not exists")); } var ownFilter = +params.own_filter; From 150658d58d999bb21eecd85c70423d71b5216ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 15:59:37 +0200 Subject: [PATCH 09/29] remove step: 3 function refactor --- lib/cartodb/backends/filter-stats.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index d2ffe6ba..5458c637 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -36,9 +36,9 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu if ( filters && !_.isEmpty(filters)) { var analysisFilter = new AnalysisFilter(filters); var query = analysisFilter.sql(unfiltered_query); - getEstimatedRows(self.pgQueryRunner, username, query, this); - } else { - this(null, null); + getEstimatedRows(self.pgQueryRunner, username, query, function () { + + }); } }) function receiveFilteredRows(err, rows) { From cca0848e6d0ef457a58aa25f0beef9699fb82fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Jul 2018 15:59:43 +0200 Subject: [PATCH 10/29] Improve error --- lib/cartodb/backends/dataview.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index b4eee7fa..1be165e8 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -30,7 +30,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); if (!dataviewDefinition) { - return callback(new Error("Dataview '" + dataviewName + "' does not exists")); + const error = new Error(`Dataview '${dataviewName}' does not exists`); + error.type = 'dataview'; + error.http_status = 400; + return callback(error); } var ownFilter = +params.own_filter; From 13075460acbb362bce53e8f8fba86144d7e9f545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Jul 2018 16:00:30 +0200 Subject: [PATCH 11/29] Do not override incoming arguments --- lib/cartodb/backends/dataview.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 1be165e8..25684711 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -39,10 +39,9 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param var ownFilter = +params.own_filter; var noFilters = +params.no_filters; if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) { - err = new Error(); - err.message = 'Both own_filter and no_filters cannot be sent in the same request'; - err.type = 'dataview'; - err.http_status = 400; + const error = new Error('Both own_filter and no_filters cannot be sent in the same request'); + error.type = 'dataview'; + error.http_status = 400; return callback(err); } From c2577d1d2538da3c783e7145b9a69950a094567a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 16:02:04 +0200 Subject: [PATCH 12/29] remove step: last function --- lib/cartodb/backends/filter-stats.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 5458c637..338e9f10 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -36,17 +36,16 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu if ( filters && !_.isEmpty(filters)) { var analysisFilter = new AnalysisFilter(filters); var query = analysisFilter.sql(unfiltered_query); - getEstimatedRows(self.pgQueryRunner, username, query, function () { - + getEstimatedRows(self.pgQueryRunner, username, query, function (err, rows) { + if (err){ + callback(err); + return; + } + stats.filtered_rows = rows; + callback(null, stats); }); } - }) - function receiveFilteredRows(err, rows) { - if (err){ - callback(err); - return; - } - stats.filtered_rows = rows; + callback(null, stats); - } + }) }; From cd08ad693f7df2515d436d3bb71aa07079954732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 16:38:04 +0200 Subject: [PATCH 13/29] remove step: last function refactor --- lib/cartodb/backends/filter-stats.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 338e9f10..48692c41 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -38,14 +38,14 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu var query = analysisFilter.sql(unfiltered_query); getEstimatedRows(self.pgQueryRunner, username, query, function (err, rows) { if (err){ - callback(err); - return; + return callback(err); } - stats.filtered_rows = rows; - callback(null, stats); - }); - } - callback(null, stats); + stats.filtered_rows = rows; + return callback(null, stats); + }); + } else { + return callback(null, stats); + } }) }; From 898bac5b042d9d7ed2b790a25db7cba837f826be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 16:42:17 +0200 Subject: [PATCH 14/29] jshint and style --- lib/cartodb/backends/filter-stats.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 48692c41..9c410362 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -36,6 +36,7 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu if ( filters && !_.isEmpty(filters)) { var analysisFilter = new AnalysisFilter(filters); var query = analysisFilter.sql(unfiltered_query); + getEstimatedRows(self.pgQueryRunner, username, query, function (err, rows) { if (err){ return callback(err); @@ -44,8 +45,9 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu stats.filtered_rows = rows; return callback(null, stats); }); + } else { return callback(null, stats); } - }) + }); }; From 384f4f74e04eddbb38b52ae139b2d7e71b869629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 16:44:30 +0200 Subject: [PATCH 15/29] remove 'self' with arrows functions --- lib/cartodb/backends/filter-stats.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 9c410362..76179ff3 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -24,9 +24,8 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) { FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { var stats = {}; - var self = this; - getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, function (err, rows) { + getEstimatedRows(this.pgQueryRunner, username, unfiltered_query, (err, rows) => { if (err){ return callback(err); } @@ -37,7 +36,7 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu var analysisFilter = new AnalysisFilter(filters); var query = analysisFilter.sql(unfiltered_query); - getEstimatedRows(self.pgQueryRunner, username, query, function (err, rows) { + getEstimatedRows(this.pgQueryRunner, username, query, (err, rows) => { if (err){ return callback(err); } From 6c3b546648a52bba2991b2f4993afd00a5b15d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 17:09:34 +0200 Subject: [PATCH 16/29] remove step --- lib/cartodb/backends/user-limits.js | 54 +++++++++++++---------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js index 519f1cd1..3a137269 100644 --- a/lib/cartodb/backends/user-limits.js +++ b/lib/cartodb/backends/user-limits.js @@ -1,5 +1,3 @@ -var step = require('step'); - /** * * @param metadataBackend @@ -43,41 +41,39 @@ UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callba UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { var self = this; - step( - function isAuthorized() { - var next = this; + function isAuthorized() { + var next = this; - if (!apiKey) { - return next(null, false); - } - - self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { - if (err) { - return next(err); - } - - return next(null, userApiKey === apiKey); - }); - }, - function getUserTimeoutRenderLimits(err, authorized) { - var next = this; + if (!apiKey) { + return next(null, false); + } + self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { if (err) { return next(err); } - self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { - if (err) { - return next(err); - } + return next(null, userApiKey === apiKey); + }); + }, + function getUserTimeoutRenderLimits(err, authorized) { + var next = this; - next(null, { - render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic - }); + if (err) { + return next(err); + } + + self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { + if (err) { + return next(err); + } + + next(null, { + render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic }); - }, - callback - ); + }); + }, + callback }; UserLimitsBackend.prototype.preprareRateLimit = function () { From eaff11ef6e143a5d9a738d8b1e171bedfe412985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 17:15:04 +0200 Subject: [PATCH 17/29] remove step: function 1 --- lib/cartodb/backends/user-limits.js | 32 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js index 3a137269..7ce0f2a3 100644 --- a/lib/cartodb/backends/user-limits.js +++ b/lib/cartodb/backends/user-limits.js @@ -41,21 +41,11 @@ UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callba UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { var self = this; - function isAuthorized() { - var next = this; - - if (!apiKey) { - return next(null, false); + isAuthorized(self.metadataBackend, username, apiKey, (err, authorized) => { + if (err) { + return callback(err); } - - self.metadataBackend.getUserMapKey(username, function (err, userApiKey) { - if (err) { - return next(err); - } - - return next(null, userApiKey === apiKey); - }); - }, + }) function getUserTimeoutRenderLimits(err, authorized) { var next = this; @@ -76,6 +66,20 @@ UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback }; +function isAuthorized(metadataBackend, username, apiKey, callback) { + if (!apiKey) { + return callback(null, false); + } + + metadataBackend.getUserMapKey(username, function (err, userApiKey) { + if (err) { + return callback(err); + } + + return callback(null, userApiKey === apiKey); + }); +} + UserLimitsBackend.prototype.preprareRateLimit = function () { if (this.options.limits.rateLimitsEnabled) { this.metadataBackend.loadRateLimitsScript(); From 70000f9df144ad30bc9ba2bbd5972df879dc0b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 17:16:27 +0200 Subject: [PATCH 18/29] remove step: function 2 --- lib/cartodb/backends/user-limits.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js index 7ce0f2a3..3586432a 100644 --- a/lib/cartodb/backends/user-limits.js +++ b/lib/cartodb/backends/user-limits.js @@ -45,13 +45,6 @@ UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, if (err) { return callback(err); } - }) - function getUserTimeoutRenderLimits(err, authorized) { - var next = this; - - if (err) { - return next(err); - } self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { if (err) { @@ -62,8 +55,7 @@ UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic }); }); - }, - callback + }); }; function isAuthorized(metadataBackend, username, apiKey, callback) { From 12ad4420aaf09be55140a3a8690ff8315f74f144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 17:18:35 +0200 Subject: [PATCH 19/29] remove step: function 2 refactor --- lib/cartodb/backends/user-limits.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js index 3586432a..f306cadf 100644 --- a/lib/cartodb/backends/user-limits.js +++ b/lib/cartodb/backends/user-limits.js @@ -48,12 +48,13 @@ UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { if (err) { - return next(err); + return callback(err); } - next(null, { - render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic - }); + callback( + null, + { render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic } + ); }); }); }; From 3cf378e045d9b3404882ab38f32dd10f374d2c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Mon, 30 Jul 2018 17:21:19 +0200 Subject: [PATCH 20/29] remove 'self' --- lib/cartodb/backends/user-limits.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js index f306cadf..87587f37 100644 --- a/lib/cartodb/backends/user-limits.js +++ b/lib/cartodb/backends/user-limits.js @@ -39,14 +39,12 @@ UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callba }; UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { - var self = this; - - isAuthorized(self.metadataBackend, username, apiKey, (err, authorized) => { + isAuthorized(this.metadataBackend, username, apiKey, (err, authorized) => { if (err) { return callback(err); } - self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) { + this.metadataBackend.getUserTimeoutRenderLimits(username, (err, timeoutRenderLimit) => { if (err) { return callback(err); } From 230b1bb3db020642d41c2b403bc6ab494504c345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 11:59:39 +0200 Subject: [PATCH 21/29] Remove step .searchDataview() --- lib/cartodb/backends/dataview.js | 79 ++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 25684711..1abf488c 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -1,7 +1,5 @@ -var assert = require('assert'); var _ = require('underscore'); var PSQL = require('cartodb-psql'); -var step = require('step'); var BBoxFilter = require('../models/filter/bbox'); var DataviewFactory = require('../models/dataview/factory'); var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); @@ -140,39 +138,52 @@ function getOverrideParams(params, ownFilter) { } DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) { - step( - function getMapConfig() { - mapConfigProvider.getMapConfig(this); - }, - function runDataviewSearchQuery(err, mapConfig) { - assert.ifError(err); - - var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); - if (!dataviewDefinition) { - throw new Error("Dataview '" + dataviewName + "' does not exists"); - } - - var pg = new PSQL(dbParamsFromReqParams(params)); - - var ownFilter = +params.own_filter; - ownFilter = !!ownFilter; - - var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; - - if (params.bbox) { - var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); - query = bboxFilter.sql(query); - } - - var userQuery = params.q; - - var dataview = DataviewFactory.getDataview(query, dataviewDefinition); - dataview.search(pg, userQuery, this); - }, - function returnCallback(err, result) { - return callback(err, result); + mapConfigProvider.getMapConfig(function (err, mapConfig) { + if (err) { + return callback(err); } - ); + + var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!dataviewDefinition) { + const error = new Error(`Dataview '${dataviewName}' does not exists`); + error.type = 'dataview'; + error.http_status = 400; + return callback(error); + } + + var pg; + + try { + pg = new PSQL(dbParamsFromReqParams(params)); + } catch (error) { + return callback(error); + } + + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; + + if (params.bbox) { + try { + var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox }); + query = bboxFilter.sql(query); + } catch (error) { + return callback(error); + } + } + + var userQuery = params.q; + + var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + dataview.search(pg, userQuery, function (err, result) { + if (err) { + return callback(err); + } + + return callback(null, result); + }); + }); }; function getDataviewDefinition(mapConfig, dataviewName) { From 70ac0587dba800b219920a26a9bd83f4d1950ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 12:15:20 +0200 Subject: [PATCH 22/29] Missing error --- lib/cartodb/backends/dataview.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 1abf488c..fe549092 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -40,7 +40,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param const error = new Error('Both own_filter and no_filters cannot be sent in the same request'); error.type = 'dataview'; error.http_status = 400; - return callback(err); + return callback(error); } var pg = new PSQL(dbParamsFromReqParams(params)); From 18603ad24f5c0bd636fa5f0eb350f526fde8f018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 12:43:54 +0200 Subject: [PATCH 23/29] Reduce cyclomatic complexity --- lib/cartodb/backends/dataview.js | 34 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index fe549092..1b13a930 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -152,30 +152,18 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewNa } var pg; + var query; + var dataview; try { pg = new PSQL(dbParamsFromReqParams(params)); + query = getQueryWithFilters(dataviewDefinition, params); + dataview = DataviewFactory.getDataview(query, dataviewDefinition); } catch (error) { return callback(error); } - var ownFilter = +params.own_filter; - ownFilter = !!ownFilter; - - var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; - - if (params.bbox) { - try { - var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox }); - query = bboxFilter.sql(query); - } catch (error) { - return callback(error); - } - } - var userQuery = params.q; - - var dataview = DataviewFactory.getDataview(query, dataviewDefinition); dataview.search(pg, userQuery, function (err, result) { if (err) { return callback(err); @@ -186,6 +174,20 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewNa }); }; +function getQueryWithFilters (dataviewDefinition, params) { + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; + + if (params.bbox) { + var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox }); + query = bboxFilter.sql(query); + } + + return query; +} + function getDataviewDefinition(mapConfig, dataviewName) { var dataviews = mapConfig.dataviews || {}; return dataviews[dataviewName]; From 9124a26a45d7553c5fcde54197b5ee8db8205c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 12:47:03 +0200 Subject: [PATCH 24/29] Move veriable declaration --- lib/cartodb/backends/dataview.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 1b13a930..87605ce6 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -154,6 +154,7 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewNa var pg; var query; var dataview; + var userQuery = params.q; try { pg = new PSQL(dbParamsFromReqParams(params)); @@ -163,7 +164,6 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewNa return callback(error); } - var userQuery = params.q; dataview.search(pg, userQuery, function (err, result) { if (err) { return callback(err); From d4de54f2921f5cc1b474fd58a64878ad8cb943e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 13:26:38 +0200 Subject: [PATCH 25/29] Extract get query with filters --- lib/cartodb/backends/dataview.js | 38 +++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 87605ce6..1c7a02f5 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -34,9 +34,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param return callback(error); } - var ownFilter = +params.own_filter; - var noFilters = +params.no_filters; - if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) { + if (!validFilterParams(params)) { const error = new Error('Both own_filter and no_filters cannot be sent in the same request'); error.type = 'dataview'; error.http_status = 400; @@ -44,15 +42,8 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param } var pg = new PSQL(dbParamsFromReqParams(params)); - - var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); - if (params.bbox) { - var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); - query = bboxFilter.sql(query); - } - + var query = getQueryWithFilters(dataviewDefinition, params); var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); - var dataviewFactory = DataviewFactoryWithOverviews.getFactory( overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } ); @@ -62,6 +53,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param let overrideParams; try { + var ownFilter = +params.own_filter; overrideParams = getOverrideParams(params, !!ownFilter); } catch (error) { return callback(error); @@ -77,6 +69,26 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param }); }; +function validFilterParams (params) { + var ownFilter = +params.own_filter; + var noFilters = +params.no_filters; + + return !(Number.isFinite(ownFilter) && Number.isFinite(noFilters)); +} + +function getQueryWithFilters (dataviewDefinition, params) { + var ownFilter = +params.own_filter; + var noFilters = +params.no_filters; + var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); + + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + return query; +} + function getDataviewQuery(dataviewDefinition, ownFilter, noFilters) { if (noFilters) { return dataviewDefinition.sql.no_filters; @@ -158,7 +170,7 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewNa try { pg = new PSQL(dbParamsFromReqParams(params)); - query = getQueryWithFilters(dataviewDefinition, params); + query = getQueryWithOwnFilters(dataviewDefinition, params); dataview = DataviewFactory.getDataview(query, dataviewDefinition); } catch (error) { return callback(error); @@ -174,7 +186,7 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewNa }); }; -function getQueryWithFilters (dataviewDefinition, params) { +function getQueryWithOwnFilters (dataviewDefinition, params) { var ownFilter = +params.own_filter; ownFilter = !!ownFilter; From ec0e90e8ce1b3d4347cfcd66080a8e3e35bc25a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 13:33:33 +0200 Subject: [PATCH 26/29] Avoid uncaught exceptions --- lib/cartodb/backends/dataview.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 1c7a02f5..bf0b31bf 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -41,18 +41,18 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param return callback(error); } - var pg = new PSQL(dbParamsFromReqParams(params)); - var query = getQueryWithFilters(dataviewDefinition, params); - var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); - var dataviewFactory = DataviewFactoryWithOverviews.getFactory( - overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } - ); - - var dataview = dataviewFactory.getDataview(query, dataviewDefinition); - - let overrideParams; + var pg; + var overrideParams; + var dataview; try { + pg = new PSQL(dbParamsFromReqParams(params)); + var query = getQueryWithFilters(dataviewDefinition, params); + var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params); + var dataviewFactory = DataviewFactoryWithOverviews.getFactory(overviewsQueryRewriter, queryRewriteData, { + bbox: params.bbox + }); + dataview = dataviewFactory.getDataview(query, dataviewDefinition); var ownFilter = +params.own_filter; overrideParams = getOverrideParams(params, !!ownFilter); } catch (error) { From 515e4828868fbc762aebd701f2519441111c0f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 31 Jul 2018 15:07:34 +0200 Subject: [PATCH 27/29] early return --- lib/cartodb/backends/filter-stats.js | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/backends/filter-stats.js b/lib/cartodb/backends/filter-stats.js index 76179ff3..b37d8704 100644 --- a/lib/cartodb/backends/filter-stats.js +++ b/lib/cartodb/backends/filter-stats.js @@ -32,21 +32,20 @@ FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_qu stats.unfiltered_rows = rows; - if ( filters && !_.isEmpty(filters)) { - var analysisFilter = new AnalysisFilter(filters); - var query = analysisFilter.sql(unfiltered_query); - - getEstimatedRows(this.pgQueryRunner, username, query, (err, rows) => { - if (err){ - return callback(err); - } - - stats.filtered_rows = rows; - return callback(null, stats); - }); - - } else { + if (!filters || _.isEmpty(filters)) { return callback(null, stats); } + + var analysisFilter = new AnalysisFilter(filters); + var query = analysisFilter.sql(unfiltered_query); + + getEstimatedRows(this.pgQueryRunner, username, query, (err, rows) => { + if (err){ + return callback(err); + } + + stats.filtered_rows = rows; + return callback(null, stats); + }); }); }; From b8d320c434420eeae76c521f966ffdcb2b89ba3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 31 Jul 2018 15:19:41 +0200 Subject: [PATCH 28/29] return callback --- lib/cartodb/backends/user-limits.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/user-limits.js b/lib/cartodb/backends/user-limits.js index 87587f37..2dc70470 100644 --- a/lib/cartodb/backends/user-limits.js +++ b/lib/cartodb/backends/user-limits.js @@ -49,7 +49,7 @@ UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, return callback(err); } - callback( + return callback( null, { render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic } ); From 3a3baf3c853f4229dd5527e0aca93a52b3ff2a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Jul 2018 15:41:29 +0200 Subject: [PATCH 29/29] Rename variable --- lib/cartodb/backends/dataview.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index bf0b31bf..542bbb4e 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -59,12 +59,12 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param return callback(error); } - dataview.getResult(pg, overrideParams, function (err, dataview) { + dataview.getResult(pg, overrideParams, function (err, dataviewResult) { if (err) { return callback(err); } - return callback(null, dataview); + return callback(null, dataviewResult); }); }); };