From 4bb35f5fabccfcbf4935533008824627c09531e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 26 Mar 2018 15:37:44 +0200 Subject: [PATCH 001/185] Extract cors and user middlewares and set them up at application level --- lib/cartodb/controllers/analyses.js | 4 ---- lib/cartodb/controllers/layergroup/analysis.js | 4 ---- lib/cartodb/controllers/layergroup/attributes.js | 4 ---- lib/cartodb/controllers/layergroup/dataview.js | 10 ---------- lib/cartodb/controllers/layergroup/static.js | 6 ------ lib/cartodb/controllers/layergroup/tile.js | 8 -------- lib/cartodb/controllers/map.js | 3 --- lib/cartodb/controllers/named_maps.js | 6 ------ lib/cartodb/controllers/named_maps_admin.js | 11 ----------- lib/cartodb/server.js | 5 +++++ 10 files changed, 5 insertions(+), 56 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 2d89a091..d08d2779 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -1,6 +1,4 @@ const PSQL = require('cartodb-psql'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); const cleanUpQueryParams = require('../middleware/clean-up-query-params'); const credentials = require('../middleware/credentials'); const authorize = require('../middleware/authorize'); @@ -24,8 +22,6 @@ AnalysesController.prototype.register = function (app) { app.get( `${mapconfigBasePath}/analyses/catalog`, - cors(), - user(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js index 3022bce3..b5faedb1 100644 --- a/lib/cartodb/controllers/layergroup/analysis.js +++ b/lib/cartodb/controllers/layergroup/analysis.js @@ -1,5 +1,3 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); const layergroupToken = require('../../middleware/layergroup-token'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -34,8 +32,6 @@ module.exports = class AnalysisController { app.get( `${mapConfigBasePath}/:token/analysis/node/:nodeId`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js index 07ae5da1..24c016ad 100644 --- a/lib/cartodb/controllers/layergroup/attributes.js +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -1,5 +1,3 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); const layergroupToken = require('../../middleware/layergroup-token'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -38,8 +36,6 @@ module.exports = class AttribitesController { app.get( `${mapConfigBasePath}/:token/:layer/attributes/:fid`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js index bca27f1e..3864ce37 100644 --- a/lib/cartodb/controllers/layergroup/dataview.js +++ b/lib/cartodb/controllers/layergroup/dataview.js @@ -1,5 +1,3 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); const layergroupToken = require('../../middleware/layergroup-token'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -56,8 +54,6 @@ module.exports = class DataviewController { app.get( `${mapConfigBasePath}/:token/dataview/:dataviewName`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), @@ -80,8 +76,6 @@ module.exports = class DataviewController { app.get( `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), @@ -104,8 +98,6 @@ module.exports = class DataviewController { app.get( `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), @@ -128,8 +120,6 @@ module.exports = class DataviewController { app.get( `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/static.js b/lib/cartodb/controllers/layergroup/static.js index 00510780..56f9169d 100644 --- a/lib/cartodb/controllers/layergroup/static.js +++ b/lib/cartodb/controllers/layergroup/static.js @@ -1,5 +1,3 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); const layergroupToken = require('../../middleware/layergroup-token'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -40,8 +38,6 @@ module.exports = class StaticController { app.get( `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), @@ -65,8 +61,6 @@ module.exports = class StaticController { app.get( `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js index 07727236..d0df6fef 100644 --- a/lib/cartodb/controllers/layergroup/tile.js +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -1,5 +1,3 @@ -const cors = require('../../middleware/cors'); -const user = require('../../middleware/user'); const layergroupToken = require('../../middleware/layergroup-token'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -48,8 +46,6 @@ module.exports = class TileController { app.get( `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), @@ -76,8 +72,6 @@ module.exports = class TileController { app.get( `${mapConfigBasePath}/:token/:z/:x/:y.:format`, - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), @@ -105,8 +99,6 @@ module.exports = class TileController { app.get( `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, distinguishLayergroupFromStaticRoute(), - cors(), - user(), layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 11074234..bbd9483b 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -4,7 +4,6 @@ const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; const ResourceLocator = require('../models/resource-locator'); const cors = require('../middleware/cors'); -const user = require('../middleware/user'); const cleanUpQueryParams = require('../middleware/clean-up-query-params'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); @@ -99,8 +98,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us const addContext = !useTemplate; return [ - cors(), - user(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 2cb4f609..044f98fa 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -1,5 +1,3 @@ -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); const cleanUpQueryParams = require('../middleware/clean-up-query-params'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); @@ -54,8 +52,6 @@ NamedMapsController.prototype.register = function(app) { app.get( `${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`, - cors(), - user(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), @@ -80,8 +76,6 @@ NamedMapsController.prototype.register = function(app) { app.get( `${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`, - cors(), - user(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index afcaa1e3..5e3c40dc 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -1,6 +1,5 @@ const { templateName } = require('../backends/template_maps'); const cors = require('../middleware/cors'); -const user = require('../middleware/user'); const credentials = require('../middleware/credentials'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -25,8 +24,6 @@ NamedMapsAdminController.prototype.register = function (app) { app.post( `${templateBasePath}/`, - cors(), - user(), credentials(), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), @@ -37,8 +34,6 @@ NamedMapsAdminController.prototype.register = function (app) { app.put( `${templateBasePath}/:template_id`, - cors(), - user(), credentials(), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), @@ -49,8 +44,6 @@ NamedMapsAdminController.prototype.register = function (app) { app.get( `${templateBasePath}/:template_id`, - cors(), - user(), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), @@ -60,8 +53,6 @@ NamedMapsAdminController.prototype.register = function (app) { app.delete( `${templateBasePath}/:template_id`, - cors(), - user(), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), @@ -71,8 +62,6 @@ NamedMapsAdminController.prototype.register = function (app) { app.get( `${templateBasePath}/`, - cors(), - user(), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 8535783d..13fe1e06 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -1,4 +1,6 @@ var express = require('express'); +const cors = require('./middleware/cors'); +const user = require('./middleware/user'); var bodyParser = require('body-parser'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); @@ -392,6 +394,9 @@ function bootstrap(opts) { setupLogger(app, opts); + app.use(cors()); + app.use(user()); + return app; } From 59db640d0de71599633ab196bdf66a7e97a612d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 26 Mar 2018 16:05:53 +0200 Subject: [PATCH 002/185] Typo --- lib/cartodb/controllers/layergroup/attributes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js index 24c016ad..babe2621 100644 --- a/lib/cartodb/controllers/layergroup/attributes.js +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -12,7 +12,7 @@ const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const sendResponse = require('../../middleware/send-response'); -module.exports = class AttribitesController { +module.exports = class AttributesController { constructor ( attributesBackend, pgConnection, From cb488cbde8302c510d063f159cbcd04521aff46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 26 Mar 2018 19:53:33 +0200 Subject: [PATCH 003/185] Extract middleware served by host header --- app.js | 2 -- lib/cartodb/middleware/served-by-host-header.js | 11 +++++++++++ lib/cartodb/server.js | 9 ++------- 3 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 lib/cartodb/middleware/served-by-host-header.js diff --git a/app.js b/app.js index 1dd465f0..f35d580e 100755 --- a/app.js +++ b/app.js @@ -100,8 +100,6 @@ if ( global.environment.log_filename ) { global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); -global.environment.api_hostname = require('os').hostname().split('.')[0]; - // Include cartodb_windshaft only _after_ the "global" variable is set // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 var cartodbWindshaft = require('./lib/cartodb/server'); diff --git a/lib/cartodb/middleware/served-by-host-header.js b/lib/cartodb/middleware/served-by-host-header.js new file mode 100644 index 00000000..809100a2 --- /dev/null +++ b/lib/cartodb/middleware/served-by-host-header.js @@ -0,0 +1,11 @@ +const os = require('os'); + +module.exports = function servedByHostHeader () { + const hostname = os.hostname().split('.')[0]; + + return function servedByHostHeaderMiddleware (req, res, next) { + res.set('X-Served-By-Host', hostname); + + next(); + }; +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 13fe1e06..9231fb69 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -50,6 +50,7 @@ var StatsBackend = require('./backends/stats'); const lzmaMiddleware = require('./middleware/lzma'); const errorMiddleware = require('./middleware/error-middleware'); +const servedByHostHeader = require('./middleware/served-by-host-header'); module.exports = function(serverOptions) { // Make stats client globally accessible @@ -364,13 +365,7 @@ function bootstrap(opts) { app.use(bodyParser.json()); - app.use(function bootstrap$prepareRequestResponse(req, res, next) { - if (global.environment && global.environment.api_hostname) { - res.set('X-Served-By-Host', global.environment.api_hostname); - } - - next(); - }); + app.use(servedByHostHeader()); app.use(stats({ enabled: opts.useProfiler, From 5b9f6086673847d2e0ea7520ef57982bdbadf6a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Mar 2018 18:46:54 +0200 Subject: [PATCH 004/185] Use express routers --- lib/cartodb/controllers/analyses.js | 8 ++--- .../controllers/layergroup/analysis.js | 8 ++--- .../controllers/layergroup/attributes.js | 8 ++--- .../controllers/layergroup/dataview.js | 20 ++++++------- lib/cartodb/controllers/layergroup/index.js | 2 -- lib/cartodb/controllers/layergroup/static.js | 12 ++++---- lib/cartodb/controllers/layergroup/tile.js | 16 +++++----- lib/cartodb/controllers/map.js | 26 +++++------------ lib/cartodb/controllers/named_maps.js | 11 ++++--- lib/cartodb/controllers/named_maps_admin.js | 29 ++++++++----------- lib/cartodb/controllers/server_info.js | 8 ++--- lib/cartodb/server.js | 24 ++++++++++----- 12 files changed, 75 insertions(+), 97 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index d08d2779..9c0e915c 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -17,11 +17,9 @@ function AnalysesController(pgConnection, authApi, userLimitsApi) { module.exports = AnalysesController; -AnalysesController.prototype.register = function (app) { - const { base_url_mapconfig: mapconfigBasePath } = app; - - app.get( - `${mapconfigBasePath}/analyses/catalog`, +AnalysesController.prototype.register = function (mapRouter) { + mapRouter.get( + `/analyses/catalog`, credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js index b5faedb1..9d58a53f 100644 --- a/lib/cartodb/controllers/layergroup/analysis.js +++ b/lib/cartodb/controllers/layergroup/analysis.js @@ -27,11 +27,9 @@ module.exports = class AnalysisController { this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - - app.get( - `${mapConfigBasePath}/:token/analysis/node/:nodeId`, + register (mapRouter) { + mapRouter.get( + `/:token/analysis/node/:nodeId`, layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js index babe2621..4965fe44 100644 --- a/lib/cartodb/controllers/layergroup/attributes.js +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -31,11 +31,9 @@ module.exports = class AttributesController { this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - - app.get( - `${mapConfigBasePath}/:token/:layer/attributes/:fid`, + register (mapRouter) { + mapRouter.get( + `/:token/:layer/attributes/:fid`, layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js index 3864ce37..b768ee89 100644 --- a/lib/cartodb/controllers/layergroup/dataview.js +++ b/lib/cartodb/controllers/layergroup/dataview.js @@ -46,14 +46,12 @@ module.exports = class DataviewController { this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - + register (mapRouter) { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - app.get( - `${mapConfigBasePath}/:token/dataview/:dataviewName`, + mapRouter.get( + `/:token/dataview/:dataviewName`, layergroupToken(), credentials(), authorize(this.authApi), @@ -74,8 +72,8 @@ module.exports = class DataviewController { sendResponse() ); - app.get( - `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, + mapRouter.get( + `/:token/:layer/widget/:dataviewName`, layergroupToken(), credentials(), authorize(this.authApi), @@ -96,8 +94,8 @@ module.exports = class DataviewController { sendResponse() ); - app.get( - `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, + mapRouter.get( + `/:token/dataview/:dataviewName/search`, layergroupToken(), credentials(), authorize(this.authApi), @@ -118,8 +116,8 @@ module.exports = class DataviewController { sendResponse() ); - app.get( - `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, + mapRouter.get( + `/:token/:layer/widget/:dataviewName/search`, layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/index.js b/lib/cartodb/controllers/layergroup/index.js index e4c1770c..fbbe1fd8 100644 --- a/lib/cartodb/controllers/layergroup/index.js +++ b/lib/cartodb/controllers/layergroup/index.js @@ -109,6 +109,4 @@ LayergroupController.prototype.register = function(app) { ); analysisController.register(app); - - }; diff --git a/lib/cartodb/controllers/layergroup/static.js b/lib/cartodb/controllers/layergroup/static.js index 56f9169d..70957b53 100644 --- a/lib/cartodb/controllers/layergroup/static.js +++ b/lib/cartodb/controllers/layergroup/static.js @@ -31,13 +31,11 @@ module.exports = class StaticController { this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - + register (mapRouter) { const forcedFormat = 'png'; - app.get( - `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, + mapRouter.get( + `/static/center/:token/:z/:lat/:lng/:width/:height.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -59,8 +57,8 @@ module.exports = class StaticController { sendResponse() ); - app.get( - `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, + mapRouter.get( + `/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, layergroupToken(), credentials(), authorize(this.authApi), diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js index d0df6fef..091a8f35 100644 --- a/lib/cartodb/controllers/layergroup/tile.js +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -41,11 +41,9 @@ module.exports = class TileController { this.surrogateKeysCache = surrogateKeysCache; } - register (app) { - const { base_url_mapconfig: mapConfigBasePath } = app; - - app.get( - `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, + register (mapRouter) { + mapRouter.get( + `/:token/:z/:x/:y@:scale_factor?x.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -70,8 +68,8 @@ module.exports = class TileController { sendResponse() ); - app.get( - `${mapConfigBasePath}/:token/:z/:x/:y.:format`, + mapRouter.get( + `/:token/:z/:x/:y.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -96,8 +94,8 @@ module.exports = class TileController { sendResponse() ); - app.get( - `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, + mapRouter.get( + `/:token/:layer/:z/:x/:y.(:format)`, distinguishLayergroupFromStaticRoute(), layergroupToken(), credentials(), diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index bbd9483b..f42d1b5d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -62,32 +62,22 @@ function MapController ( module.exports = MapController; -MapController.prototype.register = function(app) { - const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; - - app.get( - `${mapConfigBasePath}`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - - app.post( - `${mapConfigBasePath}`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); +MapController.prototype.register = function(mapRouter, templateRouter) { + mapRouter.get(`/`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)); + mapRouter.post(`/`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)); + mapRouter.options(`/`, cors('Content-Type')); const useTemplate = true; - app.get( - `${templateBasePath}/:template_id/jsonp`, + templateRouter.get( + `/:template_id/jsonp`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); - app.post( - `${templateBasePath}/:template_id`, + templateRouter.post( + `/:template_id`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); - - app.options(`${mapConfigBasePath}`, cors('Content-Type')); }; MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 044f98fa..c5347190 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -47,11 +47,10 @@ function NamedMapsController ( module.exports = NamedMapsController; -NamedMapsController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; +NamedMapsController.prototype.register = function(mapRouter, templateRouter) { - app.get( - `${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`, + templateRouter.get( + `/:template_id/:layer/:z/:x/:y.(:format)`, credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), @@ -74,8 +73,8 @@ NamedMapsController.prototype.register = function(app) { vectorError() ); - app.get( - `${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`, + mapRouter.get( + `/static/named/:template_id/:width/:height.:format`, credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index 5e3c40dc..95ac0d51 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -19,11 +19,9 @@ function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { module.exports = NamedMapsAdminController; -NamedMapsAdminController.prototype.register = function (app) { - const { base_url_templated: templateBasePath } = app; - - app.post( - `${templateBasePath}/`, +NamedMapsAdminController.prototype.register = function (templateRouter) { + templateRouter.post( + `/`, credentials(), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), @@ -32,8 +30,8 @@ NamedMapsAdminController.prototype.register = function (app) { sendResponse() ); - app.put( - `${templateBasePath}/:template_id`, + templateRouter.put( + `/:template_id`, credentials(), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), @@ -42,8 +40,8 @@ NamedMapsAdminController.prototype.register = function (app) { sendResponse() ); - app.get( - `${templateBasePath}/:template_id`, + templateRouter.get( + `/:template_id`, credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), @@ -51,8 +49,8 @@ NamedMapsAdminController.prototype.register = function (app) { sendResponse() ); - app.delete( - `${templateBasePath}/:template_id`, + templateRouter.delete( + `/:template_id`, credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), @@ -60,8 +58,8 @@ NamedMapsAdminController.prototype.register = function (app) { sendResponse() ); - app.get( - `${templateBasePath}/`, + templateRouter.get( + `/`, credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), @@ -69,10 +67,7 @@ NamedMapsAdminController.prototype.register = function (app) { sendResponse() ); - app.options( - `${templateBasePath}/:template_id`, - cors('Content-Type') - ); + templateRouter.options(`/:template_id`, cors('Content-Type')); }; function checkContentType ({ action, label }) { diff --git a/lib/cartodb/controllers/server_info.js b/lib/cartodb/controllers/server_info.js index c70de9fa..12f6432b 100644 --- a/lib/cartodb/controllers/server_info.js +++ b/lib/cartodb/controllers/server_info.js @@ -12,10 +12,10 @@ function ServerInfoController(versions) { module.exports = ServerInfoController; -ServerInfoController.prototype.register = function(app) { - app.get('/health', this.health.bind(this)); - app.get('/', this.welcome.bind(this)); - app.get('/version', this.version.bind(this)); +ServerInfoController.prototype.register = function(monitorRouter) { + monitorRouter.get('/health', this.health.bind(this)); + monitorRouter.get('/', this.welcome.bind(this)); + monitorRouter.get('/version', this.version.bind(this)); }; ServerInfoController.prototype.welcome = function(req, res) { diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 9231fb69..f9161b21 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -116,8 +116,6 @@ module.exports = function(serverOptions) { // initialize express server var app = bootstrap(serverOptions); - // Extend windshaft with all the elements of the options object - _.extend(app, serverOptions); var mapStore = new windshaft.storage.MapStore({ pool: redisPool, @@ -222,6 +220,12 @@ module.exports = function(serverOptions) { * Routing ******************************************************************************************************************/ + const mapRouter = express.Router(); + const templateRouter = express.Router(); + const monitorRouter = express.Router(); + + const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = serverOptions; + new controller.Layergroup( pgConnection, mapStore, @@ -233,7 +237,7 @@ module.exports = function(serverOptions) { layergroupAffectedTablesCache, analysisBackend, authApi - ).register(app); + ).register(mapRouter); new controller.Map( pgConnection, @@ -246,7 +250,7 @@ module.exports = function(serverOptions) { mapConfigAdapter, statsBackend, authApi - ).register(app); + ).register(mapRouter, templateRouter); new controller.NamedMaps( namedMapProviderCache, @@ -258,13 +262,17 @@ module.exports = function(serverOptions) { pgConnection, authApi, userLimitsApi - ).register(app); + ).register(mapRouter, templateRouter); - new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app); + new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(templateRouter); - new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app); + new controller.Analyses(pgConnection, authApi, userLimitsApi).register(mapRouter); - new controller.ServerInfo(versions).register(app); + new controller.ServerInfo(versions).register(monitorRouter); + + app.use(mapConfigBasePath, mapRouter); + app.use(templateBasePath, templateRouter); + app.use('/', monitorRouter); /******************************************************************************************************************* * END Routing From d8d681e8bccdb9bc6cbbb08747efb33433360b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 12:39:39 +0200 Subject: [PATCH 005/185] Create separated controllers (named & anonymous) for map controller --- .../controllers/{map.js => map/anonymous.js} | 207 ++------ lib/cartodb/controllers/map/index.js | 60 +++ lib/cartodb/controllers/map/named.js | 446 ++++++++++++++++++ .../cartodb/ported/windshaft_server.test.js | 8 - 4 files changed, 540 insertions(+), 181 deletions(-) rename lib/cartodb/controllers/{map.js => map/anonymous.js} (69%) create mode 100644 lib/cartodb/controllers/map/index.js create mode 100644 lib/cartodb/controllers/map/named.js diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map/anonymous.js similarity index 69% rename from lib/cartodb/controllers/map.js rename to lib/cartodb/controllers/map/anonymous.js index f42d1b5d..9e269a60 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -2,21 +2,20 @@ const _ = require('underscore'); const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const ResourceLocator = require('../models/resource-locator'); -const cors = require('../middleware/cors'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const credentials = require('../middleware/credentials'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const authorize = require('../middleware/authorize'); -const cacheControlHeader = require('../middleware/cache-control-header'); -const cacheChannelHeader = require('../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../middleware/last-modified-header'); -const sendResponse = require('../middleware/send-response'); -const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); -const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); -const LayergroupMetadata = require('../utils/layergroup-metadata'); -const rateLimit = require('../middleware/rate-limit'); +const ResourceLocator = require('../../models/resource-locator'); +const cors = require('../../middleware/cors'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); +const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); +const LayergroupMetadata = require('../../utils/layergroup-metadata'); +const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** @@ -32,7 +31,7 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {StatsBackend} statsBackend * @constructor */ -function MapController ( +function AnonymousMapController ( pgConnection, templateMaps, mapBackend, @@ -60,42 +59,37 @@ function MapController ( this.authApi = authApi; } -module.exports = MapController; +module.exports = AnonymousMapController; -MapController.prototype.register = function(mapRouter, templateRouter) { - mapRouter.get(`/`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)); - mapRouter.post(`/`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)); +AnonymousMapController.prototype.register = function (mapRouter) { + mapRouter.get(`/`, this.composeCreateMapMiddleware()); + mapRouter.post(`/`, this.composeCreateMapMiddleware()); mapRouter.options(`/`, cors('Content-Type')); - - const useTemplate = true; - - templateRouter.get( - `/:template_id/jsonp`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) - ); - - templateRouter.post( - `/:template_id`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) - ); }; -MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { - const isTemplateInstantiation = useTemplate; - const useTemplateHash = useTemplate; - const includeQuery = !useTemplate; - const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; - const addContext = !useTemplate; +AnonymousMapController.prototype.composeCreateMapMiddleware = function () { + const isTemplateInstantiation = false; + const useTemplateHash = false; + const includeQuery = true; + const label = 'ANONYMOUS LAYERGROUP'; + const addContext = true; return [ credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, endpointGroup), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), cleanUpQueryParams(['aggregation']), initProfiler(isTemplateInstantiation), checkJsonContentType(), - this.getCreateMapMiddlewares(useTemplate), + checkCreateLayergroup(), + prepareAdapterMapConfig(this.mapConfigAdapter), + createLayergroup ( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ), incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), @@ -115,39 +109,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us ]; }; -MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { - if (useTemplate) { - return [ - checkInstantiteLayergroup(), - getTemplate( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsApi, - this.mapConfigAdapter, - this.layergroupAffectedTables - ), - instantiateLayergroup( - this.mapBackend, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTables - ) - ]; - } - - return [ - checkCreateLayergroup(), - prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup ( - this.mapBackend, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTables - ) - ]; -}; - function initProfiler (isTemplateInstantiation) { const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; @@ -170,30 +131,6 @@ function checkJsonContentType () { }; } -function checkInstantiteLayergroup () { - return function checkInstantiteLayergroupMiddleware(req, res, next) { - if (req.method === 'GET') { - const { callback, config } = req.query; - - if (callback === undefined || callback.length === 0) { - return next(new Error('callback parameter should be present and be a function name')); - } - - if (config) { - try { - req.body = JSON.parse(config); - } catch(e) { - return next(new Error('Invalid config parameter, should be a valid JSON')); - } - } - } - - req.profiler.done('checkInstantiteLayergroup'); - - return next(); - }; -} - function checkCreateLayergroup () { return function checkCreateLayergroupMiddleware (req, res, next) { if (req.method === 'GET') { @@ -215,51 +152,6 @@ function checkCreateLayergroup () { }; } -function getTemplate ( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - affectedTablesCache -) { - return function getTemplateMiddleware (req, res, next) { - const templateParams = req.body; - const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const { template_id } = req.params; - const { auth_token } = req.query; - - const params = { dbuser, dbname, dbpassword, dbhost, dbport }; - - const mapConfigProvider = new NamedMapMapConfigProvider( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - affectedTablesCache, - user, - template_id, - templateParams, - auth_token, - params - ); - - mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { - req.profiler.done('named.getMapConfig'); - if (err) { - return next(err); - } - - res.locals.mapConfig = mapConfig; - res.locals.rendererParams = rendererParams; - res.locals.mapConfigProvider = mapConfigProvider; - - next(); - }); - }; -} - function prepareAdapterMapConfig (mapConfigAdapter) { return function prepareAdapterMapConfigMiddleware(req, res, next) { const requestMapConfig = req.body; @@ -343,37 +235,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { - return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapConfig, rendererParams } = res.locals; - const mapConfigProvider = new CreateLayergroupMapConfigProvider( - mapConfig, - user, - userLimitsApi, - pgConnection, - affectedTablesCache, - rendererParams - ); - - mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.body = layergroup; - - const { mapConfigProvider } = res.locals; - - res.locals.analysesResults = mapConfigProvider.analysesResults; - res.locals.template = mapConfigProvider.template; - res.locals.context = mapConfigProvider.context; - - next(); - }); - }; -} - function incrementMapViewCount (metadataBackend) { return function incrementMapViewCountMiddleware(req, res, next) { const { mapConfig, user } = res.locals; diff --git a/lib/cartodb/controllers/map/index.js b/lib/cartodb/controllers/map/index.js new file mode 100644 index 00000000..70d45fe9 --- /dev/null +++ b/lib/cartodb/controllers/map/index.js @@ -0,0 +1,60 @@ +const AnonymousMapController = require('./anonymous'); +const NamedMapController = require('./named'); + +module.exports = class MapController { + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authApi = authApi; + } + + register (mapRouter, templateRouter) { + const anonymousMapController = new AnonymousMapController( + this.pgConnection, + this.templateMaps, + this.mapBackend, + this.metadataBackend, + this.surrogateKeysCache, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.mapConfigAdapter, + this.statsBackend, + this.authApi + ); + + anonymousMapController.register(mapRouter); + + const namedMapController = new NamedMapController( + this.pgConnection, + this.templateMaps, + this.mapBackend, + this.metadataBackend, + this.surrogateKeysCache, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.mapConfigAdapter, + this.statsBackend, + this.authApi + ); + + namedMapController.register(templateRouter); + } +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js new file mode 100644 index 00000000..9f4c6433 --- /dev/null +++ b/lib/cartodb/controllers/map/named.js @@ -0,0 +1,446 @@ +const _ = require('underscore'); +const ResourceLocator = require('../../models/resource-locator'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); +const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); +const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); +const LayergroupMetadata = require('../../utils/layergroup-metadata'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +/** + * @param {AuthApi} authApi + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsApi} userLimitsApi + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @constructor + */ +function NamedMapController ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authApi +) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTables = layergroupAffectedTables; + + this.mapConfigAdapter = mapConfigAdapter; + const resourceLocator = new ResourceLocator(global.environment); + this.layergroupMetadata = new LayergroupMetadata(resourceLocator); + + this.statsBackend = statsBackend; + this.authApi = authApi; +} + +module.exports = NamedMapController; + +NamedMapController.prototype.register = function (templateRouter) { + templateRouter.get( + `/:template_id/jsonp`, + this.composeInstantiateTemplateMiddleware() + ); + + templateRouter.post( + `/:template_id`, + this.composeInstantiateTemplateMiddleware() + ); +}; + +NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () { + const isTemplateInstantiation = true; + const useTemplateHash = true; + const includeQuery = false; + const label = 'NAMED MAP LAYERGROUP'; + const addContext = false; + + return [ + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkInstantiteLayergroup(), + getTemplate( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsApi, + this.mapConfigAdapter, + this.layergroupAffectedTables + ), + instantiateLayergroup( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), + setLastUpdatedTimeToLayergroup(), + setLayerStats(this.pgConnection, this.statsBackend), + setLayergroupIdHeader(this.templateMaps ,useTemplateHash), + setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), + setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), + setTurboCartoMetadataToLayergroup(this.layergroupMetadata), + setAggregationMetadataToLayergroup(this.layergroupMetadata), + setTilejsonMetadataToLayergroup(this.layergroupMetadata), + sendResponse(), + augmentError({ label, addContext }) + ]; +}; + +function initProfiler (isTemplateInstantiation) { + const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; + + return function initProfilerMiddleware (req, res, next) { + req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + req.profiler.done(`${operation}.initProfilerMiddleware`); + next(); + }; +} + +function checkJsonContentType () { + return function checkJsonContentTypeMiddleware(req, res, next) { + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('POST data must be of type application/json')); + } + + req.profiler.done('checkJsonContentTypeMiddleware'); + + next(); + }; +} + +function checkInstantiteLayergroup () { + return function checkInstantiteLayergroupMiddleware(req, res, next) { + if (req.method === 'GET') { + const { callback, config } = req.query; + + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); + } + + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } + } + } + + req.profiler.done('checkInstantiteLayergroup'); + + return next(); + }; +} + +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { + return function getTemplateMiddleware (req, res, next) { + const templateParams = req.body; + const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id } = req.params; + const { auth_token } = req.query; + + const params = { dbuser, dbname, dbpassword, dbhost, dbport }; + + const mapConfigProvider = new NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache, + user, + template_id, + templateParams, + auth_token, + params + ); + + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { + req.profiler.done('named.getMapConfig'); + if (err) { + return next(err); + } + + res.locals.mapConfig = mapConfig; + res.locals.rendererParams = rendererParams; + res.locals.mapConfigProvider = mapConfigProvider; + + next(); + }); + }; +} + +function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { + return function instantiateLayergroupMiddleware (req, res, next) { + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + rendererParams + ); + + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); + if (err) { + return next(err); + } + + res.body = layergroup; + + const { mapConfigProvider } = res.locals; + + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; + + next(); + }); + }; +} + +function incrementMapViewCount (metadataBackend) { + return function incrementMapViewCountMiddleware(req, res, next) { + const { mapConfig, user } = res.locals; + + // Error won't blow up, just be logged. + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { + req.profiler.done('incMapviewCount'); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + } + + next(); + }); + }; +} + +function augmentLayergroupData () { + return function augmentLayergroupDataMiddleware (req, res, next) { + const layergroup = res.body; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + next(); + }; +} + +function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + return next(err); + } + + if (!affectedTables) { + return next(); + } + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }); + }; +} + +function getLastUpdatedTime(analysesResults, lastUpdateTime) { + if (!Array.isArray(analysesResults)) { + return lastUpdateTime; + } + return analysesResults.reduce(function(lastUpdateTime, analysis) { + return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { + var nodeUpdatedAtDate = node.getUpdatedAt(); + var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; + return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; + }, lastUpdateTime); + }, lastUpdateTime); +} + +function setLayerStats (pgConnection, statsBackend) { + return function setLayerStatsMiddleware(req, res, next) { + const { user, mapConfig } = res.locals; + const layergroup = res.body; + + pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { + if (err) { + return next(err); + } + + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + next(); + }); + }); + }; +} + +function setLayergroupIdHeader (templateMaps, useTemplateHash) { + return function setLayergroupIdHeaderMiddleware (req, res, next) { + const { user, template } = res.locals; + const layergroup = res.body; + + if (useTemplateHash) { + var templateHash = templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + } + + res.set('X-Layergroup-Id', layergroup.layergroupid); + + next(); + }; +} + +function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { + return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { + const { user, mapConfig } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); + + next(); + }; +} + +function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { + return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { + const { user, analysesResults = [] } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + + next(); + }; +} + +function setTurboCartoMetadataToLayergroup (layergroupMetadata) { + return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { + const { mapConfig, context } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); + + next(); + }; +} + +function setAggregationMetadataToLayergroup (layergroupMetadata) { + return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { + const { mapConfig, context } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); + + next(); + }; +} + +function setTilejsonMetadataToLayergroup (layergroupMetadata) { + return function augmentLayergroupTilejsonMiddleware (req, res, next) { + const { user, mapConfig } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); + + next(); + }; +} + +function augmentError (options) { + const { addContext = false, label = 'MAPS CONTROLLER' } = options; + + return function augmentErrorMiddleware (err, req, res, next) { + req.profiler.done('error'); + const { mapConfig } = res.locals; + + if (addContext) { + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; + } + + err.label = label; + + next(err); + }; +} + +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} diff --git a/test/unit/cartodb/ported/windshaft_server.test.js b/test/unit/cartodb/ported/windshaft_server.test.js index e6f07ea7..3f71ff76 100644 --- a/test/unit/cartodb/ported/windshaft_server.test.js +++ b/test/unit/cartodb/ported/windshaft_server.test.js @@ -33,12 +33,4 @@ describe('windshaft', function() { }, /Cannot read property 'mapnik' of undefined/ ); }); - - it('options are set on main windshaft object', function(){ - var ws = cartodbServer(serverOptions); - assert.ok(_.isObject(ws.bind)); - assert.ok(_.isObject(ws.grainstore)); - assert.equal(ws.base_url, '/tiles/:table'); - }); - }); From 64d540f23b094c3608949434256e162f3f0cbdc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 12:45:03 +0200 Subject: [PATCH 006/185] Extract initProfiler middlewar --- lib/cartodb/controllers/map/anonymous.js | 11 +---------- .../controllers/map/middlewares/init-profiler.js | 9 +++++++++ lib/cartodb/controllers/map/named.js | 11 +---------- 3 files changed, 11 insertions(+), 20 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/init-profiler.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 9e269a60..a8b7f353 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -8,6 +8,7 @@ const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); +const initProfiler = require('./middlewares/init-profiler'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -109,16 +110,6 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { ]; }; -function initProfiler (isTemplateInstantiation) { - const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; - - return function initProfilerMiddleware (req, res, next) { - req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); - req.profiler.done(`${operation}.initProfilerMiddleware`); - next(); - }; -} - function checkJsonContentType () { return function checkJsonContentTypeMiddleware(req, res, next) { if (req.method === 'POST' && !req.is('application/json')) { diff --git a/lib/cartodb/controllers/map/middlewares/init-profiler.js b/lib/cartodb/controllers/map/middlewares/init-profiler.js new file mode 100644 index 00000000..6d0be211 --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/init-profiler.js @@ -0,0 +1,9 @@ +module.exports = function initProfiler (isTemplateInstantiation) { + const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; + + return function initProfilerMiddleware (req, res, next) { + req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + req.profiler.done(`${operation}.initProfilerMiddleware`); + next(); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index 9f4c6433..ecaf3917 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -4,6 +4,7 @@ const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); +const initProfiler = require('./middlewares/init-profiler'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -119,16 +120,6 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () ]; }; -function initProfiler (isTemplateInstantiation) { - const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; - - return function initProfilerMiddleware (req, res, next) { - req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); - req.profiler.done(`${operation}.initProfilerMiddleware`); - next(); - }; -} - function checkJsonContentType () { return function checkJsonContentTypeMiddleware(req, res, next) { if (req.method === 'POST' && !req.is('application/json')) { From b68d2d9115f0e69d7c5fbf025de863c036d81b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 12:49:11 +0200 Subject: [PATCH 007/185] Extract checkJsonContentType middleware --- lib/cartodb/controllers/map/anonymous.js | 13 +------------ .../map/middlewares/check-json-content-type.js | 11 +++++++++++ lib/cartodb/controllers/map/named.js | 13 +------------ 3 files changed, 13 insertions(+), 24 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/check-json-content-type.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index a8b7f353..d90820c4 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -9,6 +9,7 @@ const credentials = require('../../middleware/credentials'); const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); const initProfiler = require('./middlewares/init-profiler'); +const checkJsonContentType = require('./middlewares/check-json-content-type'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -110,18 +111,6 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { ]; }; -function checkJsonContentType () { - return function checkJsonContentTypeMiddleware(req, res, next) { - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('POST data must be of type application/json')); - } - - req.profiler.done('checkJsonContentTypeMiddleware'); - - next(); - }; -} - function checkCreateLayergroup () { return function checkCreateLayergroupMiddleware (req, res, next) { if (req.method === 'GET') { diff --git a/lib/cartodb/controllers/map/middlewares/check-json-content-type.js b/lib/cartodb/controllers/map/middlewares/check-json-content-type.js new file mode 100644 index 00000000..05aa6f26 --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/check-json-content-type.js @@ -0,0 +1,11 @@ +module.exports = function checkJsonContentType () { + return function checkJsonContentTypeMiddleware(req, res, next) { + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('POST data must be of type application/json')); + } + + req.profiler.done('checkJsonContentTypeMiddleware'); + + next(); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index ecaf3917..73822421 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -5,6 +5,7 @@ const credentials = require('../../middleware/credentials'); const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); const initProfiler = require('./middlewares/init-profiler'); +const checkJsonContentType = require('./middlewares/check-json-content-type'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -120,18 +121,6 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () ]; }; -function checkJsonContentType () { - return function checkJsonContentTypeMiddleware(req, res, next) { - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('POST data must be of type application/json')); - } - - req.profiler.done('checkJsonContentTypeMiddleware'); - - next(); - }; -} - function checkInstantiteLayergroup () { return function checkInstantiteLayergroupMiddleware(req, res, next) { if (req.method === 'GET') { From 1d54a8dccda6bb9ca61993c462b447dc5733ac81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 13:01:23 +0200 Subject: [PATCH 008/185] Extract increment map view count middleware --- lib/cartodb/controllers/map/anonymous.js | 18 +----------------- .../middlewares/increment-map-view-count.js | 16 ++++++++++++++++ lib/cartodb/controllers/map/named.js | 18 +----------------- 3 files changed, 18 insertions(+), 34 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/increment-map-view-count.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index d90820c4..02c6807e 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -10,6 +10,7 @@ const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); const initProfiler = require('./middlewares/init-profiler'); const checkJsonContentType = require('./middlewares/check-json-content-type'); +const incrementMapViewCount = require('./middlewares/increment-map-view-count'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -215,23 +216,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function incrementMapViewCount (metadataBackend) { - return function incrementMapViewCountMiddleware(req, res, next) { - const { mapConfig, user } = res.locals; - - // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { - req.profiler.done('incMapviewCount'); - - if (err) { - global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); - } - - next(); - }); - }; -} - function augmentLayergroupData () { return function augmentLayergroupDataMiddleware (req, res, next) { const layergroup = res.body; diff --git a/lib/cartodb/controllers/map/middlewares/increment-map-view-count.js b/lib/cartodb/controllers/map/middlewares/increment-map-view-count.js new file mode 100644 index 00000000..acf241fd --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/increment-map-view-count.js @@ -0,0 +1,16 @@ +module.exports = function incrementMapViewCount (metadataBackend) { + return function incrementMapViewCountMiddleware(req, res, next) { + const { mapConfig, user } = res.locals; + + // Error won't blow up, just be logged. + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { + req.profiler.done('incMapviewCount'); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + } + + next(); + }); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index 73822421..f880587c 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -6,6 +6,7 @@ const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); const initProfiler = require('./middlewares/init-profiler'); const checkJsonContentType = require('./middlewares/check-json-content-type'); +const incrementMapViewCount = require('./middlewares/increment-map-view-count'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -221,23 +222,6 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }; } -function incrementMapViewCount (metadataBackend) { - return function incrementMapViewCountMiddleware(req, res, next) { - const { mapConfig, user } = res.locals; - - // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { - req.profiler.done('incMapviewCount'); - - if (err) { - global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); - } - - next(); - }); - }; -} - function augmentLayergroupData () { return function augmentLayergroupDataMiddleware (req, res, next) { const layergroup = res.body; From 16e84511665a18c064642a8b6c9663f9b8a11133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 13:10:47 +0200 Subject: [PATCH 009/185] Extract augment layergroup data middleware --- lib/cartodb/controllers/map/anonymous.js | 15 +-------------- .../map/middlewares/augment-layergroup-data.js | 14 ++++++++++++++ lib/cartodb/controllers/map/named.js | 14 +------------- 3 files changed, 16 insertions(+), 27 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/augment-layergroup-data.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 02c6807e..c5fa19a5 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -1,4 +1,3 @@ -const _ = require('underscore'); const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; @@ -11,6 +10,7 @@ const authorize = require('../../middleware/authorize'); const initProfiler = require('./middlewares/init-profiler'); const checkJsonContentType = require('./middlewares/check-json-content-type'); const incrementMapViewCount = require('./middlewares/increment-map-view-count'); +const augmentLayergroupData = require('./middlewares/augment-layergroup-data'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -216,19 +216,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function augmentLayergroupData () { - return function augmentLayergroupDataMiddleware (req, res, next) { - const layergroup = res.body; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - next(); - }; -} - function setLastUpdatedTimeToLayergroup () { return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { const { mapConfigProvider, analysesResults } = res.locals; diff --git a/lib/cartodb/controllers/map/middlewares/augment-layergroup-data.js b/lib/cartodb/controllers/map/middlewares/augment-layergroup-data.js new file mode 100644 index 00000000..65a6702d --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/augment-layergroup-data.js @@ -0,0 +1,14 @@ +const _ = require('underscore'); + +module.exports = function augmentLayergroupData () { + return function augmentLayergroupDataMiddleware (req, res, next) { + const layergroup = res.body; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + next(); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index f880587c..68504fdc 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -7,6 +7,7 @@ const authorize = require('../../middleware/authorize'); const initProfiler = require('./middlewares/init-profiler'); const checkJsonContentType = require('./middlewares/check-json-content-type'); const incrementMapViewCount = require('./middlewares/increment-map-view-count'); +const augmentLayergroupData = require('./middlewares/augment-layergroup-data'); const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); @@ -222,19 +223,6 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }; } -function augmentLayergroupData () { - return function augmentLayergroupDataMiddleware (req, res, next) { - const layergroup = res.body; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - next(); - }; -} - function setLastUpdatedTimeToLayergroup () { return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { const { mapConfigProvider, analysesResults } = res.locals; From e79d9ec2f9687d94e1ad4bbec7c6f03221a89d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 13:27:01 +0200 Subject: [PATCH 010/185] Extract lastUpdatedTimeLayergroup middleware --- lib/cartodb/controllers/map/anonymous.js | 43 +------------------ .../last-updated-time-layergroup.js | 39 +++++++++++++++++ lib/cartodb/controllers/map/named.js | 43 +------------------ 3 files changed, 43 insertions(+), 82 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index c5fa19a5..073b8982 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -15,6 +15,7 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); +const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../../utils/layergroup-metadata'); @@ -99,7 +100,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader({ now: true }), - setLastUpdatedTimeToLayergroup(), + lastUpdatedTimeLayergroup(), setLayerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), @@ -216,46 +217,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { mapConfigProvider, analysesResults } = res.locals; - const layergroup = res.body; - - mapConfigProvider.getAffectedTables((err, affectedTables) => { - if (err) { - return next(err); - } - - if (!affectedTables) { - return next(); - } - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }); - }; -} - -function getLastUpdatedTime(analysesResults, lastUpdateTime) { - if (!Array.isArray(analysesResults)) { - return lastUpdateTime; - } - return analysesResults.reduce(function(lastUpdateTime, analysis) { - return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { - var nodeUpdatedAtDate = node.getUpdatedAt(); - var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; - return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; - }, lastUpdateTime); - }, lastUpdateTime); -} - function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { const { user, mapConfig } = res.locals; diff --git a/lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js b/lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js new file mode 100644 index 00000000..5b7f091b --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js @@ -0,0 +1,39 @@ +module.exports = function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + return next(err); + } + + if (!affectedTables) { + return next(); + } + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }); + }; +}; + +function getLastUpdatedTime(analysesResults, lastUpdateTime) { + if (!Array.isArray(analysesResults)) { + return lastUpdateTime; + } + return analysesResults.reduce(function(lastUpdateTime, analysis) { + return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { + var nodeUpdatedAtDate = node.getUpdatedAt(); + var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; + return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; + }, lastUpdateTime); + }, lastUpdateTime); +} diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index 68504fdc..00a8f8d2 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -12,6 +12,7 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); +const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); @@ -110,7 +111,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader({ now: true }), - setLastUpdatedTimeToLayergroup(), + lastUpdatedTimeLayergroup(), setLayerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), @@ -223,46 +224,6 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }; } -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { mapConfigProvider, analysesResults } = res.locals; - const layergroup = res.body; - - mapConfigProvider.getAffectedTables((err, affectedTables) => { - if (err) { - return next(err); - } - - if (!affectedTables) { - return next(); - } - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }); - }; -} - -function getLastUpdatedTime(analysesResults, lastUpdateTime) { - if (!Array.isArray(analysesResults)) { - return lastUpdateTime; - } - return analysesResults.reduce(function(lastUpdateTime, analysis) { - return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { - var nodeUpdatedAtDate = node.getUpdatedAt(); - var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; - return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; - }, lastUpdateTime); - }, lastUpdateTime); -} - function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { const { user, mapConfig } = res.locals; From 947a367865ffd3f02fbb6450b41de84760e725b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 13:31:37 +0200 Subject: [PATCH 011/185] Extract layerStats middleware --- lib/cartodb/controllers/map/anonymous.js | 30 ++----------------- .../map/middlewares/layer-stats.js | 26 ++++++++++++++++ lib/cartodb/controllers/map/named.js | 30 ++----------------- 3 files changed, 30 insertions(+), 56 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/layer-stats.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 073b8982..bf672a15 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -16,6 +16,7 @@ const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); +const layerStats = require('./middlewares/layer-stats'); const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../../utils/layergroup-metadata'); @@ -101,7 +102,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader({ now: true }), lastUpdatedTimeLayergroup(), - setLayerStats(this.pgConnection, this.statsBackend), + layerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), @@ -217,33 +218,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function setLayerStats (pgConnection, statsBackend) { - return function setLayerStatsMiddleware(req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - statsBackend.getStats(mapConfig, connection, function(err, layersStats) { - if (err) { - return next(err); - } - - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - - next(); - }); - }); - }; -} - function setLayergroupIdHeader (templateMaps, useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { const { user, template } = res.locals; diff --git a/lib/cartodb/controllers/map/middlewares/layer-stats.js b/lib/cartodb/controllers/map/middlewares/layer-stats.js new file mode 100644 index 00000000..e6ca00e6 --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/layer-stats.js @@ -0,0 +1,26 @@ +module.exports = function setLayerStats (pgConnection, statsBackend) { + return function setLayerStatsMiddleware(req, res, next) { + const { user, mapConfig } = res.locals; + const layergroup = res.body; + + pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { + if (err) { + return next(err); + } + + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + next(); + }); + }); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index 00a8f8d2..cdff9367 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -13,6 +13,7 @@ const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); +const layerStats = require('./middlewares/layer-stats'); const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); @@ -112,7 +113,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader({ now: true }), lastUpdatedTimeLayergroup(), - setLayerStats(this.pgConnection, this.statsBackend), + layerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), @@ -224,33 +225,6 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }; } -function setLayerStats (pgConnection, statsBackend) { - return function setLayerStatsMiddleware(req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - statsBackend.getStats(mapConfig, connection, function(err, layersStats) { - if (err) { - return next(err); - } - - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - - next(); - }); - }); - }; -} - function setLayergroupIdHeader (templateMaps, useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { const { user, template } = res.locals; From 78356ab29810b21d40212ab0130063af3b8f0f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 13:41:41 +0200 Subject: [PATCH 012/185] Extract layergroupIdHeader middleware --- lib/cartodb/controllers/map/anonymous.js | 19 ++----------------- .../map/middlewares/layergroup-id-header.js | 15 +++++++++++++++ lib/cartodb/controllers/map/named.js | 19 ++----------------- 3 files changed, 19 insertions(+), 34 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/layergroup-id-header.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index bf672a15..2ddd12c7 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -17,6 +17,7 @@ const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const layerStats = require('./middlewares/layer-stats'); +const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../../utils/layergroup-metadata'); @@ -103,7 +104,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { lastModifiedHeader({ now: true }), lastUpdatedTimeLayergroup(), layerStats(this.pgConnection, this.statsBackend), - setLayergroupIdHeader(this.templateMaps ,useTemplateHash), + layergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), setTurboCartoMetadataToLayergroup(this.layergroupMetadata), @@ -218,22 +219,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function setLayergroupIdHeader (templateMaps, useTemplateHash) { - return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { user, template } = res.locals; - const layergroup = res.body; - - if (useTemplateHash) { - var templateHash = templateMaps.fingerPrint(template).substring(0, 8); - layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; - } - - res.set('X-Layergroup-Id', layergroup.layergroupid); - - next(); - }; -} - function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { const { user, mapConfig } = res.locals; diff --git a/lib/cartodb/controllers/map/middlewares/layergroup-id-header.js b/lib/cartodb/controllers/map/middlewares/layergroup-id-header.js new file mode 100644 index 00000000..8141d6a6 --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/layergroup-id-header.js @@ -0,0 +1,15 @@ +module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash) { + return function setLayergroupIdHeaderMiddleware (req, res, next) { + const { user, template } = res.locals; + const layergroup = res.body; + + if (useTemplateHash) { + var templateHash = templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + } + + res.set('X-Layergroup-Id', layergroup.layergroupid); + + next(); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index cdff9367..e599212e 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -14,6 +14,7 @@ const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const layerStats = require('./middlewares/layer-stats'); +const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); @@ -114,7 +115,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () lastModifiedHeader({ now: true }), lastUpdatedTimeLayergroup(), layerStats(this.pgConnection, this.statsBackend), - setLayergroupIdHeader(this.templateMaps ,useTemplateHash), + layergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), setTurboCartoMetadataToLayergroup(this.layergroupMetadata), @@ -225,22 +226,6 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }; } -function setLayergroupIdHeader (templateMaps, useTemplateHash) { - return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { user, template } = res.locals; - const layergroup = res.body; - - if (useTemplateHash) { - var templateHash = templateMaps.fingerPrint(template).substring(0, 8); - layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; - } - - res.set('X-Layergroup-Id', layergroup.layergroupid); - - next(); - }; -} - function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { const { user, mapConfig } = res.locals; From e5aff3f3663b8828e2208e21b6134d6e000b3519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 14:06:23 +0200 Subject: [PATCH 013/185] Extract layergroup-metadata middleware --- lib/cartodb/controllers/map/anonymous.js | 62 +------------------ .../map/middlewares/layergroup-metadata.js | 14 +++++ lib/cartodb/controllers/map/named.js | 62 +------------------ 3 files changed, 18 insertions(+), 120 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/layergroup-metadata.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 2ddd12c7..665ba799 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -18,6 +18,7 @@ const lastModifiedHeader = require('../../middleware/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); +const layergroupMetadata = require('./middlewares/layergroup-metadata'); const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../../utils/layergroup-metadata'); @@ -105,11 +106,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { lastUpdatedTimeLayergroup(), layerStats(this.pgConnection, this.statsBackend), layergroupIdHeader(this.templateMaps ,useTemplateHash), - setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), - setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), - setTurboCartoMetadataToLayergroup(this.layergroupMetadata), - setAggregationMetadataToLayergroup(this.layergroupMetadata), - setTilejsonMetadataToLayergroup(this.layergroupMetadata), + layergroupMetadata(this.layergroupMetadata, includeQuery), sendResponse(), augmentError({ label, addContext }) ]; @@ -219,61 +216,6 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }; } -function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { - return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); - - next(); - }; -} - -function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { - return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { user, analysesResults = [] } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - - next(); - }; -} - -function setTurboCartoMetadataToLayergroup (layergroupMetadata) { - return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setAggregationMetadataToLayergroup (layergroupMetadata) { - return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setTilejsonMetadataToLayergroup (layergroupMetadata) { - return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); - - next(); - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; diff --git a/lib/cartodb/controllers/map/middlewares/layergroup-metadata.js b/lib/cartodb/controllers/map/middlewares/layergroup-metadata.js new file mode 100644 index 00000000..301089df --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/layergroup-metadata.js @@ -0,0 +1,14 @@ +module.exports = function setMetadataToLayergroup (layergroupMetadata, includeQuery) { + return function setMetadataToLayergroupMiddleware (req, res, next) { + const { user, mapConfig, analysesResults = [], context } = res.locals; + const layergroup = res.body; + + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); + layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); + + next(); + }; +}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index e599212e..5707053a 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -15,6 +15,7 @@ const lastModifiedHeader = require('../../middleware/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); +const layergroupMetadata = require('./middlewares/layergroup-metadata'); const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); @@ -116,11 +117,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () lastUpdatedTimeLayergroup(), layerStats(this.pgConnection, this.statsBackend), layergroupIdHeader(this.templateMaps ,useTemplateHash), - setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), - setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), - setTurboCartoMetadataToLayergroup(this.layergroupMetadata), - setAggregationMetadataToLayergroup(this.layergroupMetadata), - setTilejsonMetadataToLayergroup(this.layergroupMetadata), + layergroupMetadata(this.layergroupMetadata, includeQuery), sendResponse(), augmentError({ label, addContext }) ]; @@ -226,61 +223,6 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }; } -function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { - return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); - - next(); - }; -} - -function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { - return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { user, analysesResults = [] } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - - next(); - }; -} - -function setTurboCartoMetadataToLayergroup (layergroupMetadata) { - return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setAggregationMetadataToLayergroup (layergroupMetadata) { - return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setTilejsonMetadataToLayergroup (layergroupMetadata) { - return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); - - next(); - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; From 9377b73aa30201c3d8c983b056b004474191b153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 14:12:21 +0200 Subject: [PATCH 014/185] Extract map error middleware --- lib/cartodb/controllers/map/anonymous.js | 39 +----------------- .../controllers/map/middlewares/map-error.js | 35 ++++++++++++++++ lib/cartodb/controllers/map/named.js | 40 +------------------ 3 files changed, 39 insertions(+), 75 deletions(-) create mode 100644 lib/cartodb/controllers/map/middlewares/map-error.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 665ba799..b90c6d0b 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -19,6 +19,7 @@ const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layer const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const layergroupMetadata = require('./middlewares/layergroup-metadata'); +const mapError = require('./middlewares/map-error'); const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../../utils/layergroup-metadata'); @@ -108,7 +109,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { layergroupIdHeader(this.templateMaps ,useTemplateHash), layergroupMetadata(this.layergroupMetadata, includeQuery), sendResponse(), - augmentError({ label, addContext }) + mapError({ label, addContext }) ]; }; @@ -215,39 +216,3 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl }); }; } - -function augmentError (options) { - const { addContext = false, label = 'MAPS CONTROLLER' } = options; - - return function augmentErrorMiddleware (err, req, res, next) { - req.profiler.done('error'); - const { mapConfig } = res.locals; - - if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - } - - err.label = label; - - next(err); - }; -} - -function populateError(err, mapConfig) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - return error; -} diff --git a/lib/cartodb/controllers/map/middlewares/map-error.js b/lib/cartodb/controllers/map/middlewares/map-error.js new file mode 100644 index 00000000..6d87a93d --- /dev/null +++ b/lib/cartodb/controllers/map/middlewares/map-error.js @@ -0,0 +1,35 @@ +module.exports = function mapError (options) { + const { addContext = false, label = 'MAPS CONTROLLER' } = options; + + return function mapErrorMiddleware (err, req, res, next) { + req.profiler.done('error'); + const { mapConfig } = res.locals; + + if (addContext) { + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; + } + + err.label = label; + + next(err); + }; +}; + +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index 5707053a..b29cb06f 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -1,4 +1,3 @@ -const _ = require('underscore'); const ResourceLocator = require('../../models/resource-locator'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -16,6 +15,7 @@ const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layer const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const layergroupMetadata = require('./middlewares/layergroup-metadata'); +const mapError = require('./middlewares/map-error'); const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); @@ -119,7 +119,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () layergroupIdHeader(this.templateMaps ,useTemplateHash), layergroupMetadata(this.layergroupMetadata, includeQuery), sendResponse(), - augmentError({ label, addContext }) + mapError({ label, addContext }) ]; }; @@ -222,39 +222,3 @@ function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affecte }); }; } - -function augmentError (options) { - const { addContext = false, label = 'MAPS CONTROLLER' } = options; - - return function augmentErrorMiddleware (err, req, res, next) { - req.profiler.done('error'); - const { mapConfig } = res.locals; - - if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - } - - err.label = label; - - next(err); - }; -} - -function populateError(err, mapConfig) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - return error; -} From 3576eb8081e5a755b1f34d1cfc0619bbf63f7bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 14:16:13 +0200 Subject: [PATCH 015/185] Do not use template strings --- lib/cartodb/controllers/map/anonymous.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index b90c6d0b..e64b57ea 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -70,9 +70,9 @@ function AnonymousMapController ( module.exports = AnonymousMapController; AnonymousMapController.prototype.register = function (mapRouter) { - mapRouter.get(`/`, this.composeCreateMapMiddleware()); - mapRouter.post(`/`, this.composeCreateMapMiddleware()); - mapRouter.options(`/`, cors('Content-Type')); + mapRouter.get('/', this.composeCreateMapMiddleware()); + mapRouter.post('/', this.composeCreateMapMiddleware()); + mapRouter.options('/', cors('Content-Type')); }; AnonymousMapController.prototype.composeCreateMapMiddleware = function () { From 04b65c7c0d0e2183440f59d0c623c8b0ea792d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 14:21:20 +0200 Subject: [PATCH 016/185] Please jshint --- test/unit/cartodb/ported/windshaft_server.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/cartodb/ported/windshaft_server.test.js b/test/unit/cartodb/ported/windshaft_server.test.js index 3f71ff76..ad9ed1d9 100644 --- a/test/unit/cartodb/ported/windshaft_server.test.js +++ b/test/unit/cartodb/ported/windshaft_server.test.js @@ -1,6 +1,5 @@ require('../../../support/test_helper.js'); -var _ = require('underscore'); var assert = require('assert'); var cartodbServer = require('../../../../lib/cartodb/server'); var serverOptions = require('../../../../lib/cartodb/server_options'); From 51fade6bd314ba856c8813f069772abbec9878d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 19:11:19 +0200 Subject: [PATCH 017/185] Ensure each controller only receives one router --- lib/cartodb/controllers/index.js | 2 - .../admin.js} | 20 +-- .../middlewares/named-map-provider.js | 32 +++++ .../{named_maps.js => template/preview.js} | 116 +++--------------- lib/cartodb/controllers/template/tile.js | 91 ++++++++++++++ lib/cartodb/server.js | 25 +++- 6 files changed, 168 insertions(+), 118 deletions(-) rename lib/cartodb/controllers/{named_maps_admin.js => template/admin.js} (92%) create mode 100644 lib/cartodb/controllers/template/middlewares/named-map-provider.js rename lib/cartodb/controllers/{named_maps.js => template/preview.js} (72%) create mode 100644 lib/cartodb/controllers/template/tile.js diff --git a/lib/cartodb/controllers/index.js b/lib/cartodb/controllers/index.js index 429a4b87..bb73af0f 100644 --- a/lib/cartodb/controllers/index.js +++ b/lib/cartodb/controllers/index.js @@ -2,7 +2,5 @@ module.exports = { Analyses: require('./analyses'), Layergroup: require('./layergroup'), Map: require('./map'), - NamedMaps: require('./named_maps'), - NamedMapsAdmin: require('./named_maps_admin'), ServerInfo: require('./server_info') }; diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/template/admin.js similarity index 92% rename from lib/cartodb/controllers/named_maps_admin.js rename to lib/cartodb/controllers/template/admin.js index 95ac0d51..f5f2fec7 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/template/admin.js @@ -1,9 +1,9 @@ -const { templateName } = require('../backends/template_maps'); -const cors = require('../middleware/cors'); -const credentials = require('../middleware/credentials'); -const rateLimit = require('../middleware/rate-limit'); +const { templateName } = require('../../backends/template_maps'); +const cors = require('../../middleware/cors'); +const credentials = require('../../middleware/credentials'); +const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const sendResponse = require('../middleware/send-response'); +const sendResponse = require('../../middleware/send-response'); /** * @param {AuthApi} authApi @@ -11,21 +11,21 @@ const sendResponse = require('../middleware/send-response'); * @param {TemplateMaps} templateMaps * @constructor */ -function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { +function AdminTemplateController(authApi, templateMaps, userLimitsApi) { this.authApi = authApi; this.templateMaps = templateMaps; this.userLimitsApi = userLimitsApi; } -module.exports = NamedMapsAdminController; +module.exports = AdminTemplateController; -NamedMapsAdminController.prototype.register = function (templateRouter) { +AdminTemplateController.prototype.register = function (templateRouter) { templateRouter.post( `/`, credentials(), - checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), + checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), createTemplate({ templateMaps: this.templateMaps }), sendResponse() ); @@ -33,9 +33,9 @@ NamedMapsAdminController.prototype.register = function (templateRouter) { templateRouter.put( `/:template_id`, credentials(), - checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), + checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), updateTemplate({ templateMaps: this.templateMaps }), sendResponse() ); diff --git a/lib/cartodb/controllers/template/middlewares/named-map-provider.js b/lib/cartodb/controllers/template/middlewares/named-map-provider.js new file mode 100644 index 00000000..08f0cad5 --- /dev/null +++ b/lib/cartodb/controllers/template/middlewares/named-map-provider.js @@ -0,0 +1,32 @@ +module.exports = function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { + return function getNamedMapProviderMiddleware (req, res, next) { + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, layer: layerFromParams, z, x, y, format } = req.params; + const { layer: layerFromQuery } = req.query; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format + }; + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; + } + + const { config, auth_token } = req.query; + + namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { + if (err) { + err.label = label; + return next(err); + } + + res.locals.mapConfigProvider = namedMapProvider; + + next(); + }); + }; +}; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/template/preview.js similarity index 72% rename from lib/cartodb/controllers/named_maps.js rename to lib/cartodb/controllers/template/preview.js index c5347190..d1abc587 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/template/preview.js @@ -1,14 +1,14 @@ -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const credentials = require('../middleware/credentials'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const authorize = require('../middleware/authorize'); -const cacheControlHeader = require('../middleware/cache-control-header'); -const cacheChannelHeader = require('../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../middleware/last-modified-header'); -const sendResponse = require('../middleware/send-response'); -const vectorError = require('../middleware/vector-error'); -const rateLimit = require('../middleware/rate-limit'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const namedMapProvider = require('./middlewares/named-map-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); +const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DEFAULT_ZOOM_CENTER = { @@ -23,9 +23,8 @@ function numMapper(n) { return +n; } -function NamedMapsController ( +function PreviewTemplateController ( namedMapProviderCache, - tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, @@ -35,7 +34,6 @@ function NamedMapsController ( userLimitsApi ) { this.namedMapProviderCache = namedMapProviderCache; - this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; @@ -45,34 +43,9 @@ function NamedMapsController ( this.userLimitsApi = userLimitsApi; } -module.exports = NamedMapsController; - -NamedMapsController.prototype.register = function(mapRouter, templateRouter) { - - templateRouter.get( - `/:template_id/:layer/:z/:x/:y.(:format)`, - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), - cleanUpQueryParams(), - getNamedMapProvider({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'NAMED_MAP_TILE' - }), - getTile({ - tileBackend: this.tileBackend, - label: 'NAMED_MAP_TILE' - }), - setContentTypeHeader(), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse(), - vectorError() - ); +module.exports = PreviewTemplateController; +PreviewTemplateController.prototype.register = function (mapRouter) { mapRouter.get( `/static/named/:template_id/:width/:height.:format`, credentials(), @@ -80,7 +53,7 @@ NamedMapsController.prototype.register = function(mapRouter, templateRouter) { dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), - getNamedMapProvider({ + namedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP', forcedFormat: 'png' }), @@ -101,39 +74,6 @@ NamedMapsController.prototype.register = function(mapRouter, templateRouter) { ); }; -function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { - return function getNamedMapProviderMiddleware (req, res, next) { - const { user, token, cache_buster, api_key } = res.locals; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const { template_id, layer: layerFromParams, z, x, y, format } = req.params; - const { layer: layerFromQuery } = req.query; - - const params = { - user, token, cache_buster, api_key, - dbuser, dbname, dbpassword, dbhost, dbport, - template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format - }; - - if (forcedFormat) { - params.format = forcedFormat; - params.layer = params.layer || 'all'; - } - - const { config, auth_token } = req.query; - - namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { - if (err) { - err.label = label; - return next(err); - } - - res.locals.mapConfigProvider = namedMapProvider; - - next(); - }); - }; -} - function getTemplate ({ label }) { return function getTemplateMiddleware (req, res, next) { const { mapConfigProvider } = res.locals; @@ -200,32 +140,6 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) }; } -function getTile ({ tileBackend, label }) { - return function getTileMiddleware (req, res, next) { - const { mapConfigProvider } = res.locals; - const { layer, z, x, y, format } = req.params; - const params = { layer, z, x, y, format }; - - tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { - req.profiler.add(stats); - req.profiler.done('render-' + format); - - if (err) { - err.label = label; - return next(err); - } - - if (headers) { - res.set(headers); - } - - res.body = tile; - - next(); - }); - }; -} - function getStaticImageOptions ({ tablesExtentApi }) { return function getStaticImageOptionsMiddleware(req, res, next) { const { user, mapConfigProvider, template } = res.locals; diff --git a/lib/cartodb/controllers/template/tile.js b/lib/cartodb/controllers/template/tile.js new file mode 100644 index 00000000..29b7f852 --- /dev/null +++ b/lib/cartodb/controllers/template/tile.js @@ -0,0 +1,91 @@ +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const namedMapProvider = require('./middlewares/named-map-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); +const vectorError = require('../../middleware/vector-error'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; + +function TileTemplateController ( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authApi, + userLimitsApi +) { + this.namedMapProviderCache = namedMapProviderCache; + this.tileBackend = tileBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.pgConnection = pgConnection; + this.authApi = authApi; + this.userLimitsApi = userLimitsApi; +} + +module.exports = TileTemplateController; + +TileTemplateController.prototype.register = function (templateRouter) { + templateRouter.get( + `/:template_id/:layer/:z/:x/:y.(:format)`, + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + cleanUpQueryParams(), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'NAMED_MAP_TILE' + }), + getTile({ + tileBackend: this.tileBackend, + label: 'NAMED_MAP_TILE' + }), + setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse(), + vectorError() + ); +}; + +function getTile ({ tileBackend, label }) { + return function getTileMiddleware (req, res, next) { + const { mapConfigProvider } = res.locals; + const { layer, z, x, y, format } = req.params; + const params = { layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { + req.profiler.add(stats); + req.profiler.done('render-' + format); + + if (err) { + err.label = label; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.body = tile; + + next(); + }); + }; +} + +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index f9161b21..0641629c 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -7,6 +7,9 @@ var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); var controller = require('./controllers'); +const AdminTemplateController = require('./controllers/template/admin'); +const PreviewTemplateController = require('./controllers/template/preview'); +const TileTemplateController = require('./controllers/template/tile'); var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); @@ -224,8 +227,6 @@ module.exports = function(serverOptions) { const templateRouter = express.Router(); const monitorRouter = express.Router(); - const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = serverOptions; - new controller.Layergroup( pgConnection, mapStore, @@ -252,9 +253,17 @@ module.exports = function(serverOptions) { authApi ).register(mapRouter, templateRouter); - new controller.NamedMaps( + new TileTemplateController( namedMapProviderCache, tileBackend, + surrogateKeysCache, + pgConnection, + authApi, + userLimitsApi + ).register(templateRouter); + + new PreviewTemplateController( + namedMapProviderCache, previewBackend, surrogateKeysCache, tablesExtentApi, @@ -262,14 +271,20 @@ module.exports = function(serverOptions) { pgConnection, authApi, userLimitsApi - ).register(mapRouter, templateRouter); + ).register(mapRouter); - new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(templateRouter); + new AdminTemplateController( + authApi, + templateMaps, + userLimitsApi + ).register(templateRouter); new controller.Analyses(pgConnection, authApi, userLimitsApi).register(mapRouter); new controller.ServerInfo(versions).register(monitorRouter); + const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = serverOptions; + app.use(mapConfigBasePath, mapRouter); app.use(templateBasePath, templateRouter); app.use('/', monitorRouter); From 57e10a8d2b13a87deb94927a5360a3dc4fc9fe9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 19:37:31 +0200 Subject: [PATCH 018/185] Create layergroup controllers in server construction --- lib/cartodb/controllers/index.js | 1 - .../controllers/layergroup/analysis.js | 2 +- .../controllers/layergroup/attributes.js | 2 +- .../controllers/layergroup/dataview.js | 2 +- lib/cartodb/controllers/layergroup/index.js | 112 ------------------ .../layergroup/{static.js => preview.js} | 2 +- lib/cartodb/controllers/layergroup/tile.js | 2 +- lib/cartodb/server.js | 62 ++++++++-- 8 files changed, 60 insertions(+), 125 deletions(-) delete mode 100644 lib/cartodb/controllers/layergroup/index.js rename lib/cartodb/controllers/layergroup/{static.js => preview.js} (99%) diff --git a/lib/cartodb/controllers/index.js b/lib/cartodb/controllers/index.js index bb73af0f..11bf6a2f 100644 --- a/lib/cartodb/controllers/index.js +++ b/lib/cartodb/controllers/index.js @@ -1,6 +1,5 @@ module.exports = { Analyses: require('./analyses'), - Layergroup: require('./layergroup'), Map: require('./map'), ServerInfo: require('./server_info') }; diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js index 9d58a53f..30371950 100644 --- a/lib/cartodb/controllers/layergroup/analysis.js +++ b/lib/cartodb/controllers/layergroup/analysis.js @@ -8,7 +8,7 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const sendResponse = require('../../middleware/send-response'); const dbParamsFromResLocals = require('../../utils/database-params'); -module.exports = class AnalysisController { +module.exports = class AnalysisLayergroupController { constructor ( analysisStatusBackend, pgConnection, diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js index 4965fe44..9e1aa24f 100644 --- a/lib/cartodb/controllers/layergroup/attributes.js +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -12,7 +12,7 @@ const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const sendResponse = require('../../middleware/send-response'); -module.exports = class AttributesController { +module.exports = class AttributesLayergroupController { constructor ( attributesBackend, pgConnection, diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js index b768ee89..04390d23 100644 --- a/lib/cartodb/controllers/layergroup/dataview.js +++ b/lib/cartodb/controllers/layergroup/dataview.js @@ -27,7 +27,7 @@ const ALLOWED_DATAVIEW_QUERY_PARAMS = [ 'categories', // number ]; -module.exports = class DataviewController { +module.exports = class DataviewLayergroupController { constructor ( dataviewBackend, pgConnection, diff --git a/lib/cartodb/controllers/layergroup/index.js b/lib/cartodb/controllers/layergroup/index.js deleted file mode 100644 index fbbe1fd8..00000000 --- a/lib/cartodb/controllers/layergroup/index.js +++ /dev/null @@ -1,112 +0,0 @@ -const DataviewBackend = require('../../backends/dataview'); -const AnalysisStatusBackend = require('../../backends/analysis-status'); - -const TileController = require('./tile'); -const AttributesController = require('./attributes'); -const StaticController = require('./static'); -const DataviewController = require('./dataview'); -const AnalysisController = require('./analysis'); - -/** - * @param {prepareContext} prepareContext - * @param {PgConnection} pgConnection - * @param {MapStore} mapStore - * @param {TileBackend} tileBackend - * @param {PreviewBackend} previewBackend - * @param {AttributesBackend} attributesBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {AnalysisBackend} analysisBackend - * @constructor - */ -function LayergroupController( - pgConnection, - mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - analysisBackend, - authApi -) { - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.attributesBackend = attributesBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - - this.dataviewBackend = new DataviewBackend(analysisBackend); - this.analysisStatusBackend = new AnalysisStatusBackend(); - this.authApi = authApi; -} - -module.exports = LayergroupController; - -LayergroupController.prototype.register = function(app) { - - const tileController = new TileController( - this.tileBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - tileController.register(app); - - const attributesController = new AttributesController( - this.attributesBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - attributesController.register(app); - - const staticController = new StaticController( - this.previewBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - staticController.register(app); - - const dataviewController = new DataviewController( - this.dataviewBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - dataviewController.register(app); - - const analysisController = new AnalysisController( - this.analysisStatusBackend, - this.pgConnection, - this.mapStore, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.authApi, - this.surrogateKeysCache - ); - - analysisController.register(app); -}; diff --git a/lib/cartodb/controllers/layergroup/static.js b/lib/cartodb/controllers/layergroup/preview.js similarity index 99% rename from lib/cartodb/controllers/layergroup/static.js rename to lib/cartodb/controllers/layergroup/preview.js index 70957b53..1918fd72 100644 --- a/lib/cartodb/controllers/layergroup/static.js +++ b/lib/cartodb/controllers/layergroup/preview.js @@ -12,7 +12,7 @@ const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); const sendResponse = require('../../middleware/send-response'); -module.exports = class StaticController { +module.exports = class PreviewLayergroupController { constructor ( previewBackend, pgConnection, diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js index 091a8f35..39c0e5f7 100644 --- a/lib/cartodb/controllers/layergroup/tile.js +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -22,7 +22,7 @@ const SUPPORTED_FORMATS = { mvt: true }; -module.exports = class TileController { +module.exports = class TileLayergroupController { constructor ( tileBackend, pgConnection, diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 0641629c..bb5564b8 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -7,6 +7,12 @@ var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); var controller = require('./controllers'); +const AnalysisLayergroupController = require('./controllers/layergroup/analysis'); +const AttributesLayergroupController = require('./controllers/layergroup/attributes'); +const DataviewLayergroupController = require('./controllers/layergroup/dataview'); +const PreviewLayergroupController = require('./controllers/layergroup/preview'); +const TileLayergroupController = require('./controllers/layergroup/tile'); + const AdminTemplateController = require('./controllers/template/admin'); const PreviewTemplateController = require('./controllers/template/preview'); const TileTemplateController = require('./controllers/template/tile'); @@ -35,6 +41,8 @@ var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); var AnalysisBackend = require('./backends/analysis'); +const AnalysisStatusBackend = require('./backends/analysis-status'); +const DataviewBackend = require('./backends/dataview'); var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); @@ -182,6 +190,9 @@ module.exports = function(serverOptions) { var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + const analysisStatusBackend = new AnalysisStatusBackend(); + + const dataviewBackend = new DataviewBackend(analysisBackend); var statsBackend = new StatsBackend(); @@ -227,17 +238,54 @@ module.exports = function(serverOptions) { const templateRouter = express.Router(); const monitorRouter = express.Router(); - new controller.Layergroup( + new AnalysisLayergroupController( + analysisStatusBackend, pgConnection, mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - analysisBackend, - authApi + authApi, + surrogateKeysCache + ).register(mapRouter); + + new AttributesLayergroupController( + attributesBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ).register(mapRouter); + + new DataviewLayergroupController( + dataviewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ).register(mapRouter); + + new PreviewLayergroupController( + previewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ).register(mapRouter); + + new TileLayergroupController( + tileBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache ).register(mapRouter); new controller.Map( From 5bb5bc42eea49bba2a2fc333d51af38d6c87e030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 19:58:30 +0200 Subject: [PATCH 019/185] Create Map Controller while building server --- lib/cartodb/controllers/index.js | 1 - lib/cartodb/controllers/map/anonymous.js | 12 ++--- lib/cartodb/controllers/map/index.js | 60 ------------------------ lib/cartodb/controllers/map/named.js | 10 ++-- lib/cartodb/server.js | 31 ++++++++++-- 5 files changed, 35 insertions(+), 79 deletions(-) delete mode 100644 lib/cartodb/controllers/map/index.js diff --git a/lib/cartodb/controllers/index.js b/lib/cartodb/controllers/index.js index 11bf6a2f..b3623cb5 100644 --- a/lib/cartodb/controllers/index.js +++ b/lib/cartodb/controllers/index.js @@ -1,5 +1,4 @@ module.exports = { Analyses: require('./analyses'), - Map: require('./map'), ServerInfo: require('./server_info') }; diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index e64b57ea..87f9bb84 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -1,7 +1,6 @@ const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const ResourceLocator = require('../../models/resource-locator'); const cors = require('../../middleware/cors'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); @@ -22,7 +21,6 @@ const layergroupMetadata = require('./middlewares/layergroup-metadata'); const mapError = require('./middlewares/map-error'); const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); -const LayergroupMetadata = require('../../utils/layergroup-metadata'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -49,7 +47,8 @@ function AnonymousMapController ( layergroupAffectedTables, mapConfigAdapter, statsBackend, - authApi + authApi, + layergroupMetadata ) { this.pgConnection = pgConnection; this.templateMaps = templateMaps; @@ -58,13 +57,10 @@ function AnonymousMapController ( this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; - this.mapConfigAdapter = mapConfigAdapter; - const resourceLocator = new ResourceLocator(global.environment); - this.layergroupMetadata = new LayergroupMetadata(resourceLocator); - this.statsBackend = statsBackend; this.authApi = authApi; + this.layergroupMetadata = layergroupMetadata; } module.exports = AnonymousMapController; @@ -106,7 +102,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { lastModifiedHeader({ now: true }), lastUpdatedTimeLayergroup(), layerStats(this.pgConnection, this.statsBackend), - layergroupIdHeader(this.templateMaps ,useTemplateHash), + layergroupIdHeader(this.templateMaps, useTemplateHash), layergroupMetadata(this.layergroupMetadata, includeQuery), sendResponse(), mapError({ label, addContext }) diff --git a/lib/cartodb/controllers/map/index.js b/lib/cartodb/controllers/map/index.js deleted file mode 100644 index 70d45fe9..00000000 --- a/lib/cartodb/controllers/map/index.js +++ /dev/null @@ -1,60 +0,0 @@ -const AnonymousMapController = require('./anonymous'); -const NamedMapController = require('./named'); - -module.exports = class MapController { - constructor ( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - mapConfigAdapter, - statsBackend, - authApi - ) { - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; - this.mapBackend = mapBackend; - this.metadataBackend = metadataBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.mapConfigAdapter = mapConfigAdapter; - this.statsBackend = statsBackend; - this.authApi = authApi; - } - - register (mapRouter, templateRouter) { - const anonymousMapController = new AnonymousMapController( - this.pgConnection, - this.templateMaps, - this.mapBackend, - this.metadataBackend, - this.surrogateKeysCache, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.mapConfigAdapter, - this.statsBackend, - this.authApi - ); - - anonymousMapController.register(mapRouter); - - const namedMapController = new NamedMapController( - this.pgConnection, - this.templateMaps, - this.mapBackend, - this.metadataBackend, - this.surrogateKeysCache, - this.userLimitsApi, - this.layergroupAffectedTablesCache, - this.mapConfigAdapter, - this.statsBackend, - this.authApi - ); - - namedMapController.register(templateRouter); - } -}; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index b29cb06f..e59f2d03 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -1,4 +1,3 @@ -const ResourceLocator = require('../../models/resource-locator'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); const dbConnSetup = require('../../middleware/db-conn-setup'); @@ -19,7 +18,6 @@ const mapError = require('./middlewares/map-error'); const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); -const LayergroupMetadata = require('../../utils/layergroup-metadata'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -46,7 +44,8 @@ function NamedMapController ( layergroupAffectedTables, mapConfigAdapter, statsBackend, - authApi + authApi, + layergroupMetadata ) { this.pgConnection = pgConnection; this.templateMaps = templateMaps; @@ -55,13 +54,10 @@ function NamedMapController ( this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; - this.mapConfigAdapter = mapConfigAdapter; - const resourceLocator = new ResourceLocator(global.environment); - this.layergroupMetadata = new LayergroupMetadata(resourceLocator); - this.statsBackend = statsBackend; this.authApi = authApi; + this.layergroupMetadata = layergroupMetadata; } module.exports = NamedMapController; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index bb5564b8..4ef54339 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -7,12 +7,16 @@ var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); var controller = require('./controllers'); + const AnalysisLayergroupController = require('./controllers/layergroup/analysis'); const AttributesLayergroupController = require('./controllers/layergroup/attributes'); const DataviewLayergroupController = require('./controllers/layergroup/dataview'); const PreviewLayergroupController = require('./controllers/layergroup/preview'); const TileLayergroupController = require('./controllers/layergroup/tile'); +const AnonymousMapController = require('./controllers/map/anonymous'); +const NamedMapController = require('./controllers/map/named'); + const AdminTemplateController = require('./controllers/template/admin'); const PreviewTemplateController = require('./controllers/template/preview'); const TileTemplateController = require('./controllers/template/tile'); @@ -59,6 +63,9 @@ var MapConfigAdapter = require('./models/mapconfig/adapter'); var StatsBackend = require('./backends/stats'); +const ResourceLocator = require('./models/resource-locator'); +const LayergroupMetadata = require('./utils/layergroup-metadata'); + const lzmaMiddleware = require('./middleware/lzma'); const errorMiddleware = require('./middleware/error-middleware'); const servedByHostHeader = require('./middleware/served-by-host-header'); @@ -228,6 +235,9 @@ module.exports = function(serverOptions) { var TablesExtentApi = require('./api/tables_extent_api'); var tablesExtentApi = new TablesExtentApi(pgQueryRunner); + const resourceLocator = new ResourceLocator(global.environment); + const layergroupMetadata = new LayergroupMetadata(resourceLocator); + var versions = getAndValidateVersions(serverOptions); /******************************************************************************************************************* @@ -288,7 +298,7 @@ module.exports = function(serverOptions) { surrogateKeysCache ).register(mapRouter); - new controller.Map( + new AnonymousMapController( pgConnection, templateMaps, mapBackend, @@ -298,8 +308,23 @@ module.exports = function(serverOptions) { layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, - authApi - ).register(mapRouter, templateRouter); + authApi, + layergroupMetadata + ).register(mapRouter); + + new NamedMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi, + layergroupMetadata + ).register(templateRouter); new TileTemplateController( namedMapProviderCache, From 6bf06116df33e09adf9dc640afe48a58f36ffe16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Mar 2018 20:12:11 +0200 Subject: [PATCH 020/185] Build controllers (analysis and server info) in server --- lib/cartodb/controllers/index.js | 4 ---- .../controllers/{server_info.js => server-info.js} | 0 lib/cartodb/server.js | 14 ++++++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) delete mode 100644 lib/cartodb/controllers/index.js rename lib/cartodb/controllers/{server_info.js => server-info.js} (100%) diff --git a/lib/cartodb/controllers/index.js b/lib/cartodb/controllers/index.js deleted file mode 100644 index b3623cb5..00000000 --- a/lib/cartodb/controllers/index.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - Analyses: require('./analyses'), - ServerInfo: require('./server_info') -}; diff --git a/lib/cartodb/controllers/server_info.js b/lib/cartodb/controllers/server-info.js similarity index 100% rename from lib/cartodb/controllers/server_info.js rename to lib/cartodb/controllers/server-info.js diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 4ef54339..d5410369 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -6,8 +6,6 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); -var controller = require('./controllers'); - const AnalysisLayergroupController = require('./controllers/layergroup/analysis'); const AttributesLayergroupController = require('./controllers/layergroup/attributes'); const DataviewLayergroupController = require('./controllers/layergroup/dataview'); @@ -21,6 +19,10 @@ const AdminTemplateController = require('./controllers/template/admin'); const PreviewTemplateController = require('./controllers/template/preview'); const TileTemplateController = require('./controllers/template/tile'); +const AnalysesController = require('./controllers/analyses'); + +const ServerInfoController = require('./controllers/server-info'); + var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); @@ -352,9 +354,13 @@ module.exports = function(serverOptions) { userLimitsApi ).register(templateRouter); - new controller.Analyses(pgConnection, authApi, userLimitsApi).register(mapRouter); + new AnalysesController( + pgConnection, + authApi, + userLimitsApi + ).register(mapRouter); - new controller.ServerInfo(versions).register(monitorRouter); + new ServerInfoController(versions).register(monitorRouter); const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = serverOptions; From b6989ac82a874120375c011a42ad742770f11f85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 2 Apr 2018 19:02:31 +0200 Subject: [PATCH 021/185] Create a controllers factory where all collaborators are created and controllers are mounted afterwards --- lib/cartodb/controllers/factory.js | 446 +++++++++++++++++ lib/cartodb/middleware/syntax-error.js | 10 + lib/cartodb/server.js | 459 +----------------- test/acceptance/multilayer_server.js | 2 +- .../cartodb/ported/windshaft_server.test.js | 2 +- 5 files changed, 483 insertions(+), 436 deletions(-) create mode 100644 lib/cartodb/controllers/factory.js create mode 100644 lib/cartodb/middleware/syntax-error.js diff --git a/lib/cartodb/controllers/factory.js b/lib/cartodb/controllers/factory.js new file mode 100644 index 00000000..ce341fbd --- /dev/null +++ b/lib/cartodb/controllers/factory.js @@ -0,0 +1,446 @@ +const { Router: router } = require('express'); + +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); + +const windshaft = require('windshaft'); + +const PgConnection = require('../backends/pg_connection'); +const AnalysisBackend = require('../backends/analysis'); +const AnalysisStatusBackend = require('../backends/analysis-status'); +const DataviewBackend = require('../backends/dataview'); +const TemplateMaps = require('../backends/template_maps.js'); +const PgQueryRunner = require('../backends/pg_query_runner'); +const StatsBackend = require('../backends/stats'); + +const AuthApi = require('../api/auth_api'); +const UserLimitsApi = require('../api/user_limits_api'); +const OverviewsMetadataApi = require('../api/overviews_metadata_api'); +const FilterStatsApi = require('../api/filter_stats_api'); +const TablesExtentApi = require('../api/tables_extent_api'); + +const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); +const SurrogateKeysCache = require('../cache/surrogate_keys_cache'); +const VarnishHttpCacheBackend = require('../cache/backend/varnish_http'); +const FastlyCacheBackend = require('../cache/backend/fastly'); +const NamedMapProviderCache = require('../cache/named_map_provider_cache'); +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); + +const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); +const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter'); +const MapConfigBufferSizeAdapter = require('../models/mapconfig/adapter/mapconfig-buffer-size-adapter'); +const AnalysisMapConfigAdapter = require('../models/mapconfig/adapter/analysis-mapconfig-adapter'); +const MapConfigOverviewsAdapter = require('../models/mapconfig/adapter/mapconfig-overviews-adapter'); +const TurboCartoAdapter = require('../models/mapconfig/adapter/turbo-carto-adapter'); +const DataviewsWidgetsAdapter = require('../models/mapconfig/adapter/dataviews-widgets-adapter'); +const AggregationMapConfigAdapter = require('../models/mapconfig/adapter/aggregation-mapconfig-adapter'); +const MapConfigAdapter = require('../models/mapconfig/adapter'); + +const ResourceLocator = require('../models/resource-locator'); +const LayergroupMetadata = require('../utils/layergroup-metadata'); +const RendererStatsReporter = require('../stats/reporter/renderer'); + +const AnalysisLayergroupController = require('./layergroup/analysis'); +const AttributesLayergroupController = require('./layergroup/attributes'); +const DataviewLayergroupController = require('./layergroup/dataview'); +const PreviewLayergroupController = require('./layergroup/preview'); +const TileLayergroupController = require('./layergroup/tile'); + +const AnonymousMapController = require('./map/anonymous'); +const NamedMapController = require('./map/named'); + +const AdminTemplateController = require('./template/admin'); +const PreviewTemplateController = require('./template/preview'); +const TileTemplateController = require('./template/tile'); + +const AnalysesController = require('./analyses'); + +const ServerInfoController = require('./server-info'); + +module.exports = class ControllersFactory { + constructor ({ serverOptions, environmentOptions }) { + const redisOptions = Object.assign({}, environmentOptions.redis, { + name: 'windshaft-server', + unwatchOnRelease: false, + noReadyCheck: true + }); + + const redisPool = new RedisPool(redisOptions); + + redisPool.on('status', function(status) { + var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; + global.statsClient.gauge(keyPrefix + 'count', status.count); + global.statsClient.gauge(keyPrefix + 'unused', status.unused); + global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); + }); + + const metadataBackend = cartodbRedis({ pool: redisPool }); + const pgConnection = new PgConnection(metadataBackend); + + const mapStore = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: serverOptions.grainstore.default_layergroup_ttl + }); + + const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); + + const rendererCacheOpts = Object.assign({}, serverOptions.renderCache || {}, { + ttl: 60000, // 60 seconds TTL by default + statsInterval: 60000 // reports stats every milliseconds defined here + }); + const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); + const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); + rendererStatsReporter.start(); + + const tileBackend = new windshaft.backend.Tile(rendererCache); + const attributesBackend = new windshaft.backend.Attributes(); + const previewBackend = new windshaft.backend.Preview(rendererCache); + const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); + const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + + const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions); + const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends); + const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache }); + + const analysisStatusBackend = new AnalysisStatusBackend(); + const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + const dataviewBackend = new DataviewBackend(analysisBackend); + const statsBackend = new StatsBackend(); + + const userLimitsApi = new UserLimitsApi(metadataBackend, { + limits: { + cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, + render: serverOptions.renderer.mapnik.limits.render || 0, + rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled + } + }); + const authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); + + const layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); + + if (process.env.NODE_ENV === 'test') { + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + } + + const pgQueryRunner = new PgQueryRunner(pgConnection); + const overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + + const filterStatsApi = new FilterStatsApi(pgQueryRunner); + const tablesExtentApi = new TablesExtentApi(pgQueryRunner); + + const mapConfigAdapter = new MapConfigAdapter( + new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new MapConfigBufferSizeAdapter(), + new SqlWrapMapConfigAdapter(), + new DataviewsWidgetsAdapter(), + new AnalysisMapConfigAdapter(analysisBackend), + new AggregationMapConfigAdapter(pgConnection), + new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), + new TurboCartoAdapter() + ); + + const resourceLocator = new ResourceLocator(global.environment); + const layergroupMetadata = new LayergroupMetadata(resourceLocator); + + const namedMapProviderCache = new NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + layergroupAffectedTablesCache + ); + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); + }); + + const versions = getAndValidateVersions(serverOptions); + + this.mapConfigBasePath = serverOptions.base_url_mapconfig; + this.templateBasePath = serverOptions.base_url_templated; + + this.analysisLayergroupController = new AnalysisLayergroupController( + analysisStatusBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.attributesLayergroupController = new AttributesLayergroupController( + attributesBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.dataviewLayergroupController = new DataviewLayergroupController( + dataviewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.previewLayergroupController = new PreviewLayergroupController( + previewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.tileLayergroupController = new TileLayergroupController( + tileBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.anonymousMapController = new AnonymousMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi, + layergroupMetadata + ); + + this.namedMapController = new NamedMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi, + layergroupMetadata + ); + + this.tileTemplateController = new TileTemplateController( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authApi, + userLimitsApi + ); + + this.previewTemplateController = new PreviewTemplateController( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentApi, + metadataBackend, + pgConnection, + authApi, + userLimitsApi + ); + + this.adminTemplateController = new AdminTemplateController( + authApi, + templateMaps, + userLimitsApi + ); + + this.analysesController = new AnalysesController( + pgConnection, + authApi, + userLimitsApi + ); + + this.serverInfoController = new ServerInfoController(versions); + } + + regist (app) { + if (process.env.NODE_ENV === 'test') { + app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; + } + + const mapRouter = router(); + + this.analysisLayergroupController.register(mapRouter); + this.attributesLayergroupController.register(mapRouter); + this.dataviewLayergroupController.register(mapRouter); + this.previewLayergroupController.register(mapRouter); + this.tileLayergroupController.register(mapRouter); + this.anonymousMapController.register(mapRouter); + this.previewTemplateController.register(mapRouter); + this.analysesController.register(mapRouter); + + app.use(this.mapConfigBasePath, mapRouter); + + const templateRouter = router(); + + this.namedMapController.register(templateRouter); + this.tileTemplateController.register(templateRouter); + this.adminTemplateController.register(templateRouter); + + app.use(this.templateBasePath, templateRouter); + + const monitorRouter = router(); + + this.serverInfoController.register(monitorRouter); + + app.use('/', monitorRouter); + } +}; + + +function createTemplateMaps ({ redisPool, surrogateKeysCache }) { + const templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + + function invalidateNamedMap (owner, templateName) { + var startTime = Date.now(); + surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { + var logMessage = JSON.stringify({ + username: owner, + type: 'named_map_invalidation', + elapsed: Date.now() - startTime, + error: !!err ? JSON.stringify(err.message) : undefined + }); + if (err) { + global.logger.warn(logMessage); + } else { + global.logger.info(logMessage); + } + }); + } + + + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, invalidateNamedMap); + }); + + return templateMaps; +} + +function createSurrogateKeysCacheBackends(serverOptions) { + var cacheBackends = []; + + if (serverOptions.varnish_purge_enabled) { + cacheBackends.push( + new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) + ); + } + + if (serverOptions.fastly && + !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { + cacheBackends.push( + new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) + ); + } + + return cacheBackends; +} + +const timeoutErrorTilePath = __dirname + '/../../../assets/render-timeout-fallback.png'; +const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); + +function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) { + var onTileErrorStrategy; + if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) { + onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + + function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; + } + + function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg'; + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return callback(null, timeoutErrorTile, { + 'Content-Type': 'image/png', + }, {}); + } else { + return callback(err, tile, headers, stats); + } + }; + } + + const rendererFactory = new windshaft.renderer.Factory({ + onTileErrorStrategy: onTileErrorStrategy, + mapnik: { + redisPool: redisPool, + grainstore: serverOptions.grainstore, + mapnik: serverOptions.renderer.mapnik + }, + http: serverOptions.renderer.http, + mvt: serverOptions.renderer.mvt + }); + + + return rendererFactory; +} + +function getAndValidateVersions(options) { + // jshint undef:false + var warn = console.warn.bind(console); + // jshint undef:true + + var packageDefinition = require('../../../package.json'); + + var declaredDependencies = packageDefinition.dependencies || {}; + var installedDependenciesVersions = { + camshaft: require('camshaft').version, + grainstore: windshaft.grainstore.version(), + mapnik: windshaft.mapnik.versions.mapnik, + node_mapnik: windshaft.mapnik.version, + 'turbo-carto': require('turbo-carto').version, + windshaft: windshaft.version, + windshaft_cartodb: packageDefinition.version + }; + + var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft']; + dependenciesToValidate.forEach(function(depName) { + var declaredDependencyVersion = declaredDependencies[depName]; + var installedDependencyVersion = installedDependenciesVersions[depName]; + if (declaredDependencyVersion !== installedDependencyVersion) { + warn( + 'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.', + depName, installedDependencyVersion, declaredDependencyVersion + ); + } + }); + + // Be nice and warn if configured mapnik version is != installed mapnik version + if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) { + warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' + + ' != configured mapnik version (' + options.grainstore.mapnik_version + ')'); + } + + return installedDependenciesVersions; +} diff --git a/lib/cartodb/middleware/syntax-error.js b/lib/cartodb/middleware/syntax-error.js new file mode 100644 index 00000000..ad4df4e8 --- /dev/null +++ b/lib/cartodb/middleware/syntax-error.js @@ -0,0 +1,10 @@ +module.exports = function syntaxError () { + return function syntaxErrorMiddleware (err, req, res, next) { + if (err.name === 'SyntaxError') { + err.http_status = 400; + err.message = `${err.name}: ${err.message}`; + } + + next(err); + }; +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index d5410369..3e860d12 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -1,377 +1,55 @@ -var express = require('express'); +const express = require('express'); const cors = require('./middleware/cors'); const user = require('./middleware/user'); -var bodyParser = require('body-parser'); -var RedisPool = require('redis-mpool'); -var cartodbRedis = require('cartodb-redis'); -var _ = require('underscore'); +const bodyParser = require('body-parser'); +const _ = require('underscore'); -const AnalysisLayergroupController = require('./controllers/layergroup/analysis'); -const AttributesLayergroupController = require('./controllers/layergroup/attributes'); -const DataviewLayergroupController = require('./controllers/layergroup/dataview'); -const PreviewLayergroupController = require('./controllers/layergroup/preview'); -const TileLayergroupController = require('./controllers/layergroup/tile'); - -const AnonymousMapController = require('./controllers/map/anonymous'); -const NamedMapController = require('./controllers/map/named'); - -const AdminTemplateController = require('./controllers/template/admin'); -const PreviewTemplateController = require('./controllers/template/preview'); -const TileTemplateController = require('./controllers/template/tile'); - -const AnalysesController = require('./controllers/analyses'); - -const ServerInfoController = require('./controllers/server-info'); - -var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); -var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); -var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); -var FastlyCacheBackend = require('./cache/backend/fastly'); - -var StatsClient = require('./stats/client'); +const StatsClient = require('./stats/client'); const stats = require('./middleware/stats'); -var RendererStatsReporter = require('./stats/reporter/renderer'); - -var windshaft = require('windshaft'); -var mapnik = windshaft.mapnik; - -var TemplateMaps = require('./backends/template_maps.js'); -var OverviewsMetadataApi = require('./api/overviews_metadata_api'); -var FilterStatsApi = require('./api/filter_stats_api'); -var UserLimitsApi = require('./api/user_limits_api'); -var AuthApi = require('./api/auth_api'); -var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); -var NamedMapProviderCache = require('./cache/named_map_provider_cache'); -var PgQueryRunner = require('./backends/pg_query_runner'); -var PgConnection = require('./backends/pg_connection'); - -var AnalysisBackend = require('./backends/analysis'); -const AnalysisStatusBackend = require('./backends/analysis-status'); -const DataviewBackend = require('./backends/dataview'); - -var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; -var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); - -var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); -var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); -var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter'); -var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); -var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); -var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); -var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); -var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter'); -var MapConfigAdapter = require('./models/mapconfig/adapter'); - -var StatsBackend = require('./backends/stats'); - -const ResourceLocator = require('./models/resource-locator'); -const LayergroupMetadata = require('./utils/layergroup-metadata'); +const { mapnik } = require('windshaft'); const lzmaMiddleware = require('./middleware/lzma'); const errorMiddleware = require('./middleware/error-middleware'); +const syntaxError = require('./middleware/syntax-error'); const servedByHostHeader = require('./middleware/served-by-host-header'); -module.exports = function(serverOptions) { +const ControllersFactory = require('./controllers/factory'); + +module.exports = function createServer (serverOptions) { + validateOptions(serverOptions); + // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); - var redisPool = new RedisPool(_.defaults(global.environment.redis, { - name: 'windshaft-server', - unwatchOnRelease: false, - noReadyCheck: true - })); - - redisPool.on('status', function(status) { - var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; - global.statsClient.gauge(keyPrefix + 'count', status.count); - global.statsClient.gauge(keyPrefix + 'unused', status.unused); - global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); - }); - - var metadataBackend = cartodbRedis({pool: redisPool}); - var pgConnection = new PgConnection(metadataBackend); - var pgQueryRunner = new PgQueryRunner(pgConnection); - var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); - var filterStatsApi = new FilterStatsApi(pgQueryRunner); - var userLimitsApi = new UserLimitsApi(metadataBackend, { - limits: { - cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, - render: serverOptions.renderer.mapnik.limits.render || 0, - rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled - } - }); - - var templateMaps = new TemplateMaps(redisPool, { - max_user_templates: global.environment.maxUserTemplates - }); - - var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions)); - - function invalidateNamedMap (owner, templateName) { - var startTime = Date.now(); - surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { - var logMessage = JSON.stringify({ - username: owner, - type: 'named_map_invalidation', - elapsed: Date.now() - startTime, - error: !!err ? JSON.stringify(err.message) : undefined - }); - if (err) { - global.logger.warn(logMessage); - } else { - global.logger.info(logMessage); - } - }); - } - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, invalidateNamedMap); - }); - serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions); - validateOptions(serverOptions); - bootstrapFonts(serverOptions); - // initialize express server - var app = bootstrap(serverOptions); + const app = bootstrap(serverOptions); - var mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: serverOptions.grainstore.default_layergroup_ttl - }); + app.use(bodyParser.json()); - var onTileErrorStrategy; - if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { - onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + app.use(servedByHostHeader()); - function isRenderTimeoutError (err) { - return err.message === 'Render timed out'; - } + app.use(stats({ + enabled: serverOptions.useProfiler, + statsClient: global.statsClient + })); - function isDatasourceTimeoutError (err) { - return err.message && err.message.match(/canceling statement due to statement timeout/i); - } + app.use(lzmaMiddleware()); - function isTimeoutError (err) { - return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); - } - function isRasterFormat (format) { - return format === 'png' || format === 'jpg'; - } + setupLogger(app, serverOptions); - if (isTimeoutError(err) && isRasterFormat(format)) { - return callback(null, timeoutErrorTile, { - 'Content-Type': 'image/png', - }, {}); - } else { - return callback(err, tile, headers, stats); - } - }; - } + app.use(cors()); + app.use(user()); - var rendererFactory = new windshaft.renderer.Factory({ - onTileErrorStrategy: onTileErrorStrategy, - mapnik: { - redisPool: redisPool, - grainstore: serverOptions.grainstore, - mapnik: serverOptions.renderer.mapnik - }, - http: serverOptions.renderer.http, - mvt: serverOptions.renderer.mvt - }); + const controllers = new ControllersFactory({ serverOptions, environmentOptions: global.environment }); - // initialize render cache - var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, { - ttl: 60000, // 60 seconds TTL by default - statsInterval: 60000 // reports stats every milliseconds defined here - }); - var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); - var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); - rendererStatsReporter.start(); - - var attributesBackend = new windshaft.backend.Attributes(); - var previewBackend = new windshaft.backend.Preview(rendererCache); - var tileBackend = new windshaft.backend.Tile(rendererCache); - var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); - var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); - - var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); - const analysisStatusBackend = new AnalysisStatusBackend(); - - const dataviewBackend = new DataviewBackend(analysisBackend); - - var statsBackend = new StatsBackend(); - - var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); - app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - - var mapConfigAdapter = new MapConfigAdapter( - new MapConfigNamedLayersAdapter(templateMaps, pgConnection), - new MapConfigBufferSizeAdapter(), - new SqlWrapMapConfigAdapter(), - new DataviewsWidgetsAdapter(), - new AnalysisMapConfigAdapter(analysisBackend), - new AggregationMapConfigAdapter(pgConnection), - new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), - new TurboCartoAdapter() - ); - - var namedMapProviderCache = new NamedMapProviderCache( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - layergroupAffectedTablesCache - ); - - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); - }); - - var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); - - var TablesExtentApi = require('./api/tables_extent_api'); - var tablesExtentApi = new TablesExtentApi(pgQueryRunner); - - const resourceLocator = new ResourceLocator(global.environment); - const layergroupMetadata = new LayergroupMetadata(resourceLocator); - - var versions = getAndValidateVersions(serverOptions); - - /******************************************************************************************************************* - * Routing - ******************************************************************************************************************/ - - const mapRouter = express.Router(); - const templateRouter = express.Router(); - const monitorRouter = express.Router(); - - new AnalysisLayergroupController( - analysisStatusBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ).register(mapRouter); - - new AttributesLayergroupController( - attributesBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ).register(mapRouter); - - new DataviewLayergroupController( - dataviewBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ).register(mapRouter); - - new PreviewLayergroupController( - previewBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ).register(mapRouter); - - new TileLayergroupController( - tileBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ).register(mapRouter); - - new AnonymousMapController( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - mapConfigAdapter, - statsBackend, - authApi, - layergroupMetadata - ).register(mapRouter); - - new NamedMapController( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - mapConfigAdapter, - statsBackend, - authApi, - layergroupMetadata - ).register(templateRouter); - - new TileTemplateController( - namedMapProviderCache, - tileBackend, - surrogateKeysCache, - pgConnection, - authApi, - userLimitsApi - ).register(templateRouter); - - new PreviewTemplateController( - namedMapProviderCache, - previewBackend, - surrogateKeysCache, - tablesExtentApi, - metadataBackend, - pgConnection, - authApi, - userLimitsApi - ).register(mapRouter); - - new AdminTemplateController( - authApi, - templateMaps, - userLimitsApi - ).register(templateRouter); - - new AnalysesController( - pgConnection, - authApi, - userLimitsApi - ).register(mapRouter); - - new ServerInfoController(versions).register(monitorRouter); - - const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = serverOptions; - - app.use(mapConfigBasePath, mapRouter); - app.use(templateBasePath, templateRouter); - app.use('/', monitorRouter); - - /******************************************************************************************************************* - * END Routing - ******************************************************************************************************************/ + controllers.regist(app); + app.use(syntaxError()); app.use(errorMiddleware()); return app; @@ -383,45 +61,6 @@ function validateOptions(opts) { } } -function getAndValidateVersions(options) { - // jshint undef:false - var warn = console.warn.bind(console); - // jshint undef:true - - var packageDefinition = require('../../package.json'); - - var declaredDependencies = packageDefinition.dependencies || {}; - var installedDependenciesVersions = { - camshaft: require('camshaft').version, - grainstore: windshaft.grainstore.version(), - mapnik: windshaft.mapnik.versions.mapnik, - node_mapnik: windshaft.mapnik.version, - 'turbo-carto': require('turbo-carto').version, - windshaft: windshaft.version, - windshaft_cartodb: packageDefinition.version - }; - - var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft']; - dependenciesToValidate.forEach(function(depName) { - var declaredDependencyVersion = declaredDependencies[depName]; - var installedDependencyVersion = installedDependenciesVersions[depName]; - if (declaredDependencyVersion !== installedDependencyVersion) { - warn( - 'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.', - depName, installedDependencyVersion, declaredDependencyVersion - ); - } - }); - - // Be nice and warn if configured mapnik version is != installed mapnik version - if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) { - warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' + - ' != configured mapnik version (' + options.grainstore.mapnik_version + ')'); - } - - return installedDependenciesVersions; -} - function bootstrapFonts(opts) { // Set carto renderer configuration for MMLStore opts.grainstore.carto_env = opts.grainstore.carto_env || {}; @@ -465,35 +104,6 @@ function bootstrap(opts) { return value; }); - app.use(bodyParser.json()); - - app.use(servedByHostHeader()); - - app.use(stats({ - enabled: opts.useProfiler, - statsClient: global.statsClient - })); - - app.use(lzmaMiddleware()); - - // temporary measure until we upgrade to newer version expressjs so we can check err.status - app.use(function(err, req, res, next) { - if (err) { - if (err.name === 'SyntaxError') { - res.status(400).json({ errors: [err.name + ': ' + err.message] }); - } else { - next(err); - } - } else { - next(); - } - }); - - setupLogger(app, opts); - - app.use(cors()); - app.use(user()); - return app; } @@ -513,25 +123,6 @@ function setupLogger(app, opts) { } } -function surrogateKeysCacheBackends(serverOptions) { - var cacheBackends = []; - - if (serverOptions.varnish_purge_enabled) { - cacheBackends.push( - new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) - ); - } - - if (serverOptions.fastly && - !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { - cacheBackends.push( - new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) - ); - } - - return cacheBackends; -} - function mapnikVersion(opts) { return opts.grainstore.mapnik_version || mapnik.versions.mapnik; } diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index b599cf9c..2131fd1a 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -230,7 +230,7 @@ describe('tests from old api translated to multilayer', function() { var parsed = JSON.parse(res.body); assert.ok(parsed.errors); assert.equal(parsed.errors.length, 1); - assert.ok(parsed.errors[0].match(/^Unexpected token W/)); + assert.ok(parsed.errors[0].match(/Unexpected token W/)); done(); } diff --git a/test/unit/cartodb/ported/windshaft_server.test.js b/test/unit/cartodb/ported/windshaft_server.test.js index ad9ed1d9..2bc1423c 100644 --- a/test/unit/cartodb/ported/windshaft_server.test.js +++ b/test/unit/cartodb/ported/windshaft_server.test.js @@ -29,7 +29,7 @@ describe('windshaft', function() { function(){ var ws = cartodbServer({unbuffered_logging:true}); ws.listen(); - }, /Cannot read property 'mapnik' of undefined/ + }, /Must initialise server with/ ); }); }); From 3316c2ded32b145c37dfc81c0dd6e5c4ab39db6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 12:26:35 +0200 Subject: [PATCH 022/185] Create logger middleware to encapsulate its configuration from app --- lib/cartodb/middleware/logger.js | 22 ++++++++++++++++++++++ lib/cartodb/server.js | 6 +++--- 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 lib/cartodb/middleware/logger.js diff --git a/lib/cartodb/middleware/logger.js b/lib/cartodb/middleware/logger.js new file mode 100644 index 00000000..ccd51d45 --- /dev/null +++ b/lib/cartodb/middleware/logger.js @@ -0,0 +1,22 @@ +module.exports = function logger (options) { + if (!global.log4js || !options.log_format) { + return function dummyLoggerMiddleware (req, res, next) { + next(); + }; + } + + const opts = { + level: 'info', + // Allowing for unbuffered logging is mainly + // used to avoid hanging during unit testing. + // TODO: provide an explicit teardown function instead, + // releasing any event handler or timer set by + // this component. + buffer: !options.unbuffered_logging, + // optional log format + format: options.log_format + }; + const logger = global.log4js.getLogger(); + + return global.log4js.connectLogger(logger, opts); +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 3e860d12..bcb53dc1 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -13,6 +13,7 @@ const lzmaMiddleware = require('./middleware/lzma'); const errorMiddleware = require('./middleware/error-middleware'); const syntaxError = require('./middleware/syntax-error'); const servedByHostHeader = require('./middleware/served-by-host-header'); +const logger = require('./middleware/logger'); const ControllersFactory = require('./controllers/factory'); @@ -28,6 +29,8 @@ module.exports = function createServer (serverOptions) { const app = bootstrap(serverOptions); + app.use(logger(serverOptions)); + app.use(bodyParser.json()); app.use(servedByHostHeader()); @@ -39,9 +42,6 @@ module.exports = function createServer (serverOptions) { app.use(lzmaMiddleware()); - - setupLogger(app, serverOptions); - app.use(cors()); app.use(user()); From cc61a89c684702badbbd0513376192d106fcbf98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 12:27:45 +0200 Subject: [PATCH 023/185] Remove function --- lib/cartodb/server.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index bcb53dc1..12f3cf42 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -107,22 +107,6 @@ function bootstrap(opts) { return app; } -function setupLogger(app, opts) { - if (global.log4js && opts.log_format) { - var loggerOpts = { - // Allowing for unbuffered logging is mainly - // used to avoid hanging during unit testing. - // TODO: provide an explicit teardown function instead, - // releasing any event handler or timer set by - // this component. - buffer: !opts.unbuffered_logging, - // optional log format - format: opts.log_format - }; - app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'}))); - } -} - function mapnikVersion(opts) { return opts.grainstore.mapnik_version || mapnik.versions.mapnik; } From a21d7db3901544f406c7839ad5fa3192e02b06d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 13:21:41 +0200 Subject: [PATCH 024/185] Extract json-replacer --- lib/cartodb/server.js | 72 +++++++++--------------------- lib/cartodb/utils/json-replacer.js | 19 ++++++++ 2 files changed, 39 insertions(+), 52 deletions(-) create mode 100644 lib/cartodb/utils/json-replacer.js diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 12f3cf42..25193a7e 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -1,22 +1,24 @@ -const express = require('express'); -const cors = require('./middleware/cors'); -const user = require('./middleware/user'); -const bodyParser = require('body-parser'); const _ = require('underscore'); - -const StatsClient = require('./stats/client'); -const stats = require('./middleware/stats'); - +const express = require('express'); const { mapnik } = require('windshaft'); -const lzmaMiddleware = require('./middleware/lzma'); -const errorMiddleware = require('./middleware/error-middleware'); -const syntaxError = require('./middleware/syntax-error'); -const servedByHostHeader = require('./middleware/served-by-host-header'); +const jsonReplacer = require('./utils/json-replacer'); + const logger = require('./middleware/logger'); +const bodyParser = require('body-parser'); +const servedByHostHeader = require('./middleware/served-by-host-header'); +const stats = require('./middleware/stats'); +const lzmaMiddleware = require('./middleware/lzma'); +const cors = require('./middleware/cors'); +const user = require('./middleware/user'); const ControllersFactory = require('./controllers/factory'); +const syntaxError = require('./middleware/syntax-error'); +const errorMiddleware = require('./middleware/error-middleware'); + +const StatsClient = require('./stats/client'); + module.exports = function createServer (serverOptions) { validateOptions(serverOptions); @@ -27,21 +29,21 @@ module.exports = function createServer (serverOptions) { bootstrapFonts(serverOptions); - const app = bootstrap(serverOptions); + const app = express(); + + app.enable('jsonp callback'); + app.disable('x-powered-by'); + app.disable('etag'); + app.set('json replacer', jsonReplacer()); app.use(logger(serverOptions)); - app.use(bodyParser.json()); - app.use(servedByHostHeader()); - app.use(stats({ enabled: serverOptions.useProfiler, statsClient: global.statsClient })); - app.use(lzmaMiddleware()); - app.use(cors()); app.use(user()); @@ -73,40 +75,6 @@ function bootstrapFonts(opts) { } } -function bootstrap(opts) { - var app; - if (_.isObject(opts.https)) { - // use https if possible - app = express.createServer(opts.https); - } else { - // fall back to http by default - app = express(); - } - app.enable('jsonp callback'); - app.disable('x-powered-by'); - app.disable('etag'); - - // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 - // See: http://expressjs.com/en/4x/api.html#app.set - app.set('json replacer', function (key, value) { - if (value !== value) { - return 'NaN'; - } - - if (value === Infinity) { - return 'Infinity'; - } - - if (value === -Infinity) { - return '-Infinity'; - } - - return value; - }); - - return app; -} - function mapnikVersion(opts) { return opts.grainstore.mapnik_version || mapnik.versions.mapnik; } diff --git a/lib/cartodb/utils/json-replacer.js b/lib/cartodb/utils/json-replacer.js new file mode 100644 index 00000000..cccac1f6 --- /dev/null +++ b/lib/cartodb/utils/json-replacer.js @@ -0,0 +1,19 @@ +module.exports = function jsonReplacerFactory () { + // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 + // See: http://expressjs.com/en/4x/api.html#app.set + return function jsonReplacer (key, value) { + if (value !== value) { + return 'NaN'; + } + + if (value === Infinity) { + return 'Infinity'; + } + + if (value === -Infinity) { + return '-Infinity'; + } + + return value; + }; +}; From 5cd073c96fd8d5267e8bf45071c71f4bc9a2779e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 14:50:24 +0200 Subject: [PATCH 025/185] Remove old controller from bad merge --- lib/cartodb/controllers/map.js | 592 --------------------------------- 1 file changed, 592 deletions(-) delete mode 100644 lib/cartodb/controllers/map.js diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js deleted file mode 100644 index 9fd67710..00000000 --- a/lib/cartodb/controllers/map.js +++ /dev/null @@ -1,592 +0,0 @@ -const _ = require('underscore'); -const windshaft = require('windshaft'); -const MapConfig = windshaft.model.MapConfig; -const Datasource = windshaft.model.Datasource; -const ResourceLocator = require('../models/resource-locator'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const credentials = require('../middleware/credentials'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const authorize = require('../middleware/authorize'); -const cacheControlHeader = require('../middleware/cache-control-header'); -const cacheChannelHeader = require('../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../middleware/last-modified-header'); -const sendResponse = require('../middleware/send-response'); -const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); -const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); -const LayergroupMetadata = require('../utils/layergroup-metadata'); -const rateLimit = require('../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; - -/** - * @param {AuthApi} authApi - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @param {MapBackend} mapBackend - * @param metadataBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {MapConfigAdapter} mapConfigAdapter - * @param {StatsBackend} statsBackend - * @constructor - */ -function MapController ( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTables, - mapConfigAdapter, - statsBackend, - authApi -) { - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; - this.mapBackend = mapBackend; - this.metadataBackend = metadataBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; - - this.mapConfigAdapter = mapConfigAdapter; - const resourceLocator = new ResourceLocator(global.environment); - this.layergroupMetadata = new LayergroupMetadata(resourceLocator); - - this.statsBackend = statsBackend; - this.authApi = authApi; -} - -module.exports = MapController; - -MapController.prototype.register = function(app) { - const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; - - app.get( - `${mapConfigBasePath}`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - - app.post( - `${mapConfigBasePath}`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - - const useTemplate = true; - - app.get( - `${templateBasePath}/:template_id/jsonp`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) - ); - - app.post( - `${templateBasePath}/:template_id`, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) - ); - - app.options(`${mapConfigBasePath}`, cors('Content-Type')); -}; - -MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { - const isTemplateInstantiation = useTemplate; - const useTemplateHash = useTemplate; - const includeQuery = !useTemplate; - const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; - const addContext = !useTemplate; - - return [ - cors(), - user(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, endpointGroup), - cleanUpQueryParams(['aggregation']), - initProfiler(isTemplateInstantiation), - checkJsonContentType(), - this.getCreateMapMiddlewares(useTemplate), - incrementMapViewCount(this.metadataBackend), - augmentLayergroupData(), - cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader({ now: true }), - setLastUpdatedTimeToLayergroup(), - setLayerStats(this.pgConnection, this.statsBackend), - setLayergroupIdHeader(this.templateMaps ,useTemplateHash), - setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), - setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery), - setTurboCartoMetadataToLayergroup(this.layergroupMetadata), - setAggregationMetadataToLayergroup(this.layergroupMetadata), - setTilejsonMetadataToLayergroup(this.layergroupMetadata), - sendResponse(), - augmentError({ label, addContext }) - ]; -}; - -MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { - if (useTemplate) { - return [ - checkInstantiteLayergroup(), - getTemplate( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsApi, - this.mapConfigAdapter, - this.layergroupAffectedTables - ), - instantiateLayergroup( - this.mapBackend, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTables - ) - ]; - } - - return [ - checkCreateLayergroup(), - prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup ( - this.mapBackend, - this.userLimitsApi, - this.pgConnection, - this.layergroupAffectedTables - ) - ]; -}; - -function initProfiler (isTemplateInstantiation) { - const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; - - return function initProfilerMiddleware (req, res, next) { - req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); - req.profiler.done(`${operation}.initProfilerMiddleware`); - next(); - }; -} - -function checkJsonContentType () { - return function checkJsonContentTypeMiddleware(req, res, next) { - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('POST data must be of type application/json')); - } - - req.profiler.done('checkJsonContentTypeMiddleware'); - - next(); - }; -} - -function checkInstantiteLayergroup () { - return function checkInstantiteLayergroupMiddleware(req, res, next) { - if (req.method === 'GET') { - const { callback, config } = req.query; - - if (callback === undefined || callback.length === 0) { - return next(new Error('callback parameter should be present and be a function name')); - } - - if (config) { - try { - req.body = JSON.parse(config); - } catch(e) { - return next(new Error('Invalid config parameter, should be a valid JSON')); - } - } - } - - req.profiler.done('checkInstantiteLayergroup'); - - return next(); - }; -} - -function checkCreateLayergroup () { - return function checkCreateLayergroupMiddleware (req, res, next) { - if (req.method === 'GET') { - const { config } = req.query; - - if (!config) { - return next(new Error('layergroup GET needs a "config" parameter')); - } - - try { - req.body = JSON.parse(config); - } catch (err) { - return next(err); - } - } - - req.profiler.done('checkCreateLayergroup'); - return next(); - }; -} - -function getTemplate ( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - affectedTablesCache -) { - return function getTemplateMiddleware (req, res, next) { - const templateParams = req.body; - const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const { template_id } = req.params; - const { auth_token } = req.query; - - const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); - - const mapConfigProvider = new NamedMapMapConfigProvider( - templateMaps, - pgConnection, - metadataBackend, - userLimitsApi, - mapConfigAdapter, - affectedTablesCache, - user, - template_id, - templateParams, - auth_token, - params - ); - - mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { - req.profiler.done('named.getMapConfig'); - if (err) { - return next(err); - } - - res.locals.mapConfig = mapConfig; - res.locals.rendererParams = rendererParams; - res.locals.mapConfigProvider = mapConfigProvider; - - next(); - }); - }; -} - -function prepareAdapterMapConfig (mapConfigAdapter) { - return function prepareAdapterMapConfigMiddleware(req, res, next) { - const requestMapConfig = req.body; - - const { user, api_key } = res.locals; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); - - const context = { - analysisConfiguration: { - user, - db: { - host: dbhost, - port: dbport, - dbname: dbname, - user: dbuser, - pass: dbpassword - }, - batch: { - username: user, - apiKey: api_key - } - } - }; - - mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => { - req.profiler.done('anonymous.getMapConfig'); - if (err) { - return next(err); - } - - req.body = requestMapConfig; - res.locals.context = context; - - next(); - }); - }; -} - -function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { - return function createLayergroupMiddleware (req, res, next) { - const requestMapConfig = req.body; - - const { context } = res.locals; - const { user, cache_buster, api_key } = res.locals; - const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - - const params = { - cache_buster, api_key, - dbuser, dbname, dbpassword, dbhost, dbport - }; - - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapConfig = new MapConfig(requestMapConfig, datasource); - - const mapConfigProvider = new CreateLayergroupMapConfigProvider( - mapConfig, - user, - userLimitsApi, - pgConnection, - affectedTablesCache, - params - ); - - res.locals.mapConfig = mapConfig; - res.locals.analysesResults = context.analysesResults; - - const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; - - mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.body = layergroup; - res.locals.mapConfigProvider = mapConfigProvider; - - next(); - }); - }; -} - -function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { - return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapConfig, rendererParams } = res.locals; - const mapConfigProvider = new CreateLayergroupMapConfigProvider( - mapConfig, - user, - userLimitsApi, - pgConnection, - affectedTablesCache, - rendererParams - ); - - mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { - req.profiler.done('createLayergroup'); - if (err) { - return next(err); - } - - res.body = layergroup; - - const { mapConfigProvider } = res.locals; - - res.locals.analysesResults = mapConfigProvider.analysesResults; - res.locals.template = mapConfigProvider.template; - res.locals.context = mapConfigProvider.context; - - next(); - }); - }; -} - -function incrementMapViewCount (metadataBackend) { - return function incrementMapViewCountMiddleware(req, res, next) { - const { mapConfig, user } = res.locals; - - // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { - req.profiler.done('incMapviewCount'); - - if (err) { - global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); - } - - next(); - }); - }; -} - -function augmentLayergroupData () { - return function augmentLayergroupDataMiddleware (req, res, next) { - const layergroup = res.body; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - next(); - }; -} - -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { mapConfigProvider, analysesResults } = res.locals; - const layergroup = res.body; - - mapConfigProvider.getAffectedTables((err, affectedTables) => { - if (err) { - return next(err); - } - - if (!affectedTables) { - return next(); - } - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }); - }; -} - -function getLastUpdatedTime(analysesResults, lastUpdateTime) { - if (!Array.isArray(analysesResults)) { - return lastUpdateTime; - } - return analysesResults.reduce(function(lastUpdateTime, analysis) { - return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { - var nodeUpdatedAtDate = node.getUpdatedAt(); - var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; - return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; - }, lastUpdateTime); - }, lastUpdateTime); -} - -function setLayerStats (pgConnection, statsBackend) { - return function setLayerStatsMiddleware(req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - statsBackend.getStats(mapConfig, connection, function(err, layersStats) { - if (err) { - return next(err); - } - - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - - next(); - }); - }); - }; -} - -function setLayergroupIdHeader (templateMaps, useTemplateHash) { - return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { user, template } = res.locals; - const layergroup = res.body; - - if (useTemplateHash) { - var templateHash = templateMaps.fingerPrint(template).substring(0, 8); - layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; - } - - res.set('X-Layergroup-Id', layergroup.layergroupid); - - next(); - }; -} - -function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { - return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); - - next(); - }; -} - -function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { - return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { user, analysesResults = [] } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - - next(); - }; -} - -function setTurboCartoMetadataToLayergroup (layergroupMetadata) { - return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setAggregationMetadataToLayergroup (layergroupMetadata) { - return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { mapConfig, context } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); - - next(); - }; -} - -function setTilejsonMetadataToLayergroup (layergroupMetadata) { - return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { user, mapConfig } = res.locals; - const layergroup = res.body; - - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); - - next(); - }; -} - -function augmentError (options) { - const { addContext = false, label = 'MAPS CONTROLLER' } = options; - - return function augmentErrorMiddleware (err, req, res, next) { - req.profiler.done('error'); - const { mapConfig } = res.locals; - - if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - } - - err.label = label; - - next(err); - }; -} - -function populateError(err, mapConfig) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - return error; -} From 462ba626569ac1063b04e5821e7cf128d132b567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 15:32:29 +0200 Subject: [PATCH 026/185] Make cors middleware more generic and link it just to application level --- lib/cartodb/controllers/map/anonymous.js | 3 +-- lib/cartodb/controllers/template/admin.js | 5 ++--- lib/cartodb/middleware/cors.js | 14 +++++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 87f9bb84..9575eead 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -1,7 +1,6 @@ const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const cors = require('../../middleware/cors'); const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); const credentials = require('../../middleware/credentials'); const dbConnSetup = require('../../middleware/db-conn-setup'); @@ -66,9 +65,9 @@ function AnonymousMapController ( module.exports = AnonymousMapController; AnonymousMapController.prototype.register = function (mapRouter) { + mapRouter.options('/'); mapRouter.get('/', this.composeCreateMapMiddleware()); mapRouter.post('/', this.composeCreateMapMiddleware()); - mapRouter.options('/', cors('Content-Type')); }; AnonymousMapController.prototype.composeCreateMapMiddleware = function () { diff --git a/lib/cartodb/controllers/template/admin.js b/lib/cartodb/controllers/template/admin.js index f5f2fec7..d91ca9fd 100644 --- a/lib/cartodb/controllers/template/admin.js +++ b/lib/cartodb/controllers/template/admin.js @@ -1,5 +1,4 @@ const { templateName } = require('../../backends/template_maps'); -const cors = require('../../middleware/cors'); const credentials = require('../../middleware/credentials'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -20,6 +19,8 @@ function AdminTemplateController(authApi, templateMaps, userLimitsApi) { module.exports = AdminTemplateController; AdminTemplateController.prototype.register = function (templateRouter) { + templateRouter.options(`/:template_id`); + templateRouter.post( `/`, credentials(), @@ -66,8 +67,6 @@ AdminTemplateController.prototype.register = function (templateRouter) { listTemplates({ templateMaps: this.templateMaps }), sendResponse() ); - - templateRouter.options(`/:template_id`, cors('Content-Type')); }; function checkContentType ({ action, label }) { diff --git a/lib/cartodb/middleware/cors.js b/lib/cartodb/middleware/cors.js index 65b7cf4f..12db26ac 100644 --- a/lib/cartodb/middleware/cors.js +++ b/lib/cartodb/middleware/cors.js @@ -1,13 +1,17 @@ -module.exports = function cors (extraHeaders) { +module.exports = function cors () { return function corsMiddleware (req, res, next) { - let baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token"; + const headers = [ + 'X-Requested-With', + 'X-Prototype-Version', + 'X-CSRF-Token' + ]; - if(extraHeaders) { - baseHeaders += ", " + extraHeaders; + if (req.method === 'OPTIONS') { + headers.push('Content-Type'); } res.set("Access-Control-Allow-Origin", "*"); - res.set("Access-Control-Allow-Headers", baseHeaders); + res.set("Access-Control-Allow-Headers", headers.join(', ')); next(); }; From f1e421db05b1d1cda9c20883d012d2e79b502d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 19:08:56 +0200 Subject: [PATCH 027/185] Create an api router isolated from heath checks --- lib/cartodb/controllers/factory.js | 60 ++-------------------- lib/cartodb/server.js | 77 ++++++++++++++++++++++++---- test/acceptance/multilayer_server.js | 1 + test/acceptance/templates.js | 2 + 4 files changed, 74 insertions(+), 66 deletions(-) diff --git a/lib/cartodb/controllers/factory.js b/lib/cartodb/controllers/factory.js index ce341fbd..4826d726 100644 --- a/lib/cartodb/controllers/factory.js +++ b/lib/cartodb/controllers/factory.js @@ -55,8 +55,6 @@ const TileTemplateController = require('./template/tile'); const AnalysesController = require('./analyses'); -const ServerInfoController = require('./server-info'); - module.exports = class ControllersFactory { constructor ({ serverOptions, environmentOptions }) { const redisOptions = Object.assign({}, environmentOptions.redis, { @@ -155,8 +153,6 @@ module.exports = class ControllersFactory { templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); }); - const versions = getAndValidateVersions(serverOptions); - this.mapConfigBasePath = serverOptions.base_url_mapconfig; this.templateBasePath = serverOptions.base_url_templated; @@ -269,13 +265,12 @@ module.exports = class ControllersFactory { authApi, userLimitsApi ); - - this.serverInfoController = new ServerInfoController(versions); } - regist (app) { + regist (api) { + // FIXME: we need a better way to reset cache while running tests if (process.env.NODE_ENV === 'test') { - app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; + api.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; } const mapRouter = router(); @@ -289,7 +284,7 @@ module.exports = class ControllersFactory { this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); - app.use(this.mapConfigBasePath, mapRouter); + api.use(this.mapConfigBasePath, mapRouter); const templateRouter = router(); @@ -297,13 +292,7 @@ module.exports = class ControllersFactory { this.tileTemplateController.register(templateRouter); this.adminTemplateController.register(templateRouter); - app.use(this.templateBasePath, templateRouter); - - const monitorRouter = router(); - - this.serverInfoController.register(monitorRouter); - - app.use('/', monitorRouter); + api.use(this.templateBasePath, templateRouter); } }; @@ -405,42 +394,3 @@ function createRendererFactory ({ redisPool, serverOptions, environmentOptions } return rendererFactory; } - -function getAndValidateVersions(options) { - // jshint undef:false - var warn = console.warn.bind(console); - // jshint undef:true - - var packageDefinition = require('../../../package.json'); - - var declaredDependencies = packageDefinition.dependencies || {}; - var installedDependenciesVersions = { - camshaft: require('camshaft').version, - grainstore: windshaft.grainstore.version(), - mapnik: windshaft.mapnik.versions.mapnik, - node_mapnik: windshaft.mapnik.version, - 'turbo-carto': require('turbo-carto').version, - windshaft: windshaft.version, - windshaft_cartodb: packageDefinition.version - }; - - var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft']; - dependenciesToValidate.forEach(function(depName) { - var declaredDependencyVersion = declaredDependencies[depName]; - var installedDependencyVersion = installedDependenciesVersions[depName]; - if (declaredDependencyVersion !== installedDependencyVersion) { - warn( - 'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.', - depName, installedDependencyVersion, declaredDependencyVersion - ); - } - }); - - // Be nice and warn if configured mapnik version is != installed mapnik version - if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) { - warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' + - ' != configured mapnik version (' + options.grainstore.mapnik_version + ')'); - } - - return installedDependenciesVersions; -} diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 25193a7e..87bc2e9f 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -1,6 +1,7 @@ const _ = require('underscore'); const express = require('express'); -const { mapnik } = require('windshaft'); +const windshaft = require('windshaft'); +const { mapnik } = windshaft; const jsonReplacer = require('./utils/json-replacer'); @@ -13,6 +14,7 @@ const cors = require('./middleware/cors'); const user = require('./middleware/user'); const ControllersFactory = require('./controllers/factory'); +const ServerInfoController = require('./controllers/server-info'); const syntaxError = require('./middleware/syntax-error'); const errorMiddleware = require('./middleware/error-middleware'); @@ -36,23 +38,37 @@ module.exports = function createServer (serverOptions) { app.disable('etag'); app.set('json replacer', jsonReplacer()); - app.use(logger(serverOptions)); - app.use(bodyParser.json()); - app.use(servedByHostHeader()); - app.use(stats({ + const api = express.Router(); + + api.use(logger(serverOptions)); + api.use(bodyParser.json()); + api.use(servedByHostHeader()); + api.use(stats({ enabled: serverOptions.useProfiler, statsClient: global.statsClient })); - app.use(lzmaMiddleware()); - app.use(cors()); - app.use(user()); + api.use(lzmaMiddleware()); + api.use(cors()); + api.use(user()); const controllers = new ControllersFactory({ serverOptions, environmentOptions: global.environment }); - controllers.regist(app); + controllers.regist(api); - app.use(syntaxError()); - app.use(errorMiddleware()); + api.use(syntaxError()); + api.use(errorMiddleware()); + + app.use('/', api); + + const versions = getAndValidateVersions(serverOptions); + const serverInfoController = new ServerInfoController(versions); + + serverInfoController.register(app); + + // FIXME: we need a better way to reset cache while running tests + if (process.env.NODE_ENV === 'test') { + app.layergroupAffectedTablesCache = api.layergroupAffectedTablesCache; + } return app; }; @@ -78,3 +94,42 @@ function bootstrapFonts(opts) { function mapnikVersion(opts) { return opts.grainstore.mapnik_version || mapnik.versions.mapnik; } + +function getAndValidateVersions(options) { + // jshint undef:false + var warn = console.warn.bind(console); + // jshint undef:true + + var packageDefinition = require('../../package.json'); + + var declaredDependencies = packageDefinition.dependencies || {}; + var installedDependenciesVersions = { + camshaft: require('camshaft').version, + grainstore: windshaft.grainstore.version(), + mapnik: windshaft.mapnik.versions.mapnik, + node_mapnik: windshaft.mapnik.version, + 'turbo-carto': require('turbo-carto').version, + windshaft: windshaft.version, + windshaft_cartodb: packageDefinition.version + }; + + var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft']; + dependenciesToValidate.forEach(function(depName) { + var declaredDependencyVersion = declaredDependencies[depName]; + var installedDependencyVersion = installedDependenciesVersions[depName]; + if (declaredDependencyVersion !== installedDependencyVersion) { + warn( + 'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.', + depName, installedDependencyVersion, declaredDependencyVersion + ); + } + }); + + // Be nice and warn if configured mapnik version is != installed mapnik version + if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) { + warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' + + ' != configured mapnik version (' + options.grainstore.mapnik_version + ')'); + } + + return installedDependenciesVersions; +} diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index 2131fd1a..3e3bbcc0 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -369,6 +369,7 @@ describe('tests from old api translated to multilayer', function() { }; // reset internal cacheChannel cache + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); assert.response(server, diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 92a95f60..0234059b 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -26,6 +26,7 @@ server.setMaxListeners(0); var LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); describe('template_api', function() { + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); var httpRendererResourcesServer; @@ -1056,6 +1057,7 @@ describe('template_api', function() { assert.ok(cc); assert.equal(cc, expectedCC); // hack simulating restart... + // FIXME: we need a better way to reset cache while running tests server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache var get_request = { url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1', From 93bdbb1c505cb71cd4f9988c80ba9470236aa810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 3 Apr 2018 19:16:37 +0200 Subject: [PATCH 028/185] Create monitor monitor router --- lib/cartodb/server.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 87bc2e9f..e844f4f4 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -60,10 +60,14 @@ module.exports = function createServer (serverOptions) { app.use('/', api); + const monitor = express.Router(); + const versions = getAndValidateVersions(serverOptions); const serverInfoController = new ServerInfoController(versions); - serverInfoController.register(app); + serverInfoController.register(monitor); + + app.use('/', monitor); // FIXME: we need a better way to reset cache while running tests if (process.env.NODE_ENV === 'test') { From cfdff61d0844b7f82e73e9e14202aa06507e72a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 4 Apr 2018 15:52:54 +0200 Subject: [PATCH 029/185] Create map & template routers to skip unneeded middlewares in monitor endpoints --- lib/cartodb/controllers/factory.js | 150 +++---------------- lib/cartodb/controllers/map-router.js | 160 +++++++++++++++++++++ lib/cartodb/controllers/template-router.js | 92 ++++++++++++ lib/cartodb/server.js | 45 +----- 4 files changed, 271 insertions(+), 176 deletions(-) create mode 100644 lib/cartodb/controllers/map-router.js create mode 100644 lib/cartodb/controllers/template-router.js diff --git a/lib/cartodb/controllers/factory.js b/lib/cartodb/controllers/factory.js index 4826d726..4e4365e1 100644 --- a/lib/cartodb/controllers/factory.js +++ b/lib/cartodb/controllers/factory.js @@ -1,5 +1,3 @@ -const { Router: router } = require('express'); - const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); @@ -40,23 +38,13 @@ const ResourceLocator = require('../models/resource-locator'); const LayergroupMetadata = require('../utils/layergroup-metadata'); const RendererStatsReporter = require('../stats/reporter/renderer'); -const AnalysisLayergroupController = require('./layergroup/analysis'); -const AttributesLayergroupController = require('./layergroup/attributes'); -const DataviewLayergroupController = require('./layergroup/dataview'); -const PreviewLayergroupController = require('./layergroup/preview'); -const TileLayergroupController = require('./layergroup/tile'); - -const AnonymousMapController = require('./map/anonymous'); -const NamedMapController = require('./map/named'); - -const AdminTemplateController = require('./template/admin'); -const PreviewTemplateController = require('./template/preview'); -const TileTemplateController = require('./template/tile'); - -const AnalysesController = require('./analyses'); +const MapRouter = require('./map-router'); +const TemplateRouter = require('./template-router'); module.exports = class ControllersFactory { constructor ({ serverOptions, environmentOptions }) { + this.serverOptions = serverOptions; + const redisOptions = Object.assign({}, environmentOptions.redis, { name: 'windshaft-server', unwatchOnRelease: false, @@ -65,7 +53,7 @@ module.exports = class ControllersFactory { const redisPool = new RedisPool(redisOptions); - redisPool.on('status', function(status) { + redisPool.on('status', function (status) { var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.'; global.statsClient.gauge(keyPrefix + 'count', status.count); global.statsClient.gauge(keyPrefix + 'unused', status.unused); @@ -153,146 +141,40 @@ module.exports = class ControllersFactory { templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache)); }); - this.mapConfigBasePath = serverOptions.base_url_mapconfig; - this.templateBasePath = serverOptions.base_url_templated; - - this.analysisLayergroupController = new AnalysisLayergroupController( + const collaborators = { analysisStatusBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ); - - this.attributesLayergroupController = new AttributesLayergroupController( attributesBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ); - - this.dataviewLayergroupController = new DataviewLayergroupController( dataviewBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ); - - this.previewLayergroupController = new PreviewLayergroupController( previewBackend, - pgConnection, - mapStore, - userLimitsApi, - layergroupAffectedTablesCache, - authApi, - surrogateKeysCache - ); - - this.tileLayergroupController = new TileLayergroupController( tileBackend, pgConnection, mapStore, userLimitsApi, layergroupAffectedTablesCache, authApi, - surrogateKeysCache - ); - - this.anonymousMapController = new AnonymousMapController( - pgConnection, + surrogateKeysCache, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, - authApi, - layergroupMetadata - ); - - this.namedMapController = new NamedMapController( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTablesCache, - mapConfigAdapter, - statsBackend, - authApi, - layergroupMetadata - ); - - this.tileTemplateController = new TileTemplateController( + layergroupMetadata, namedMapProviderCache, - tileBackend, - surrogateKeysCache, - pgConnection, - authApi, - userLimitsApi - ); + tablesExtentApi + }; - this.previewTemplateController = new PreviewTemplateController( - namedMapProviderCache, - previewBackend, - surrogateKeysCache, - tablesExtentApi, - metadataBackend, - pgConnection, - authApi, - userLimitsApi - ); - - this.adminTemplateController = new AdminTemplateController( - authApi, - templateMaps, - userLimitsApi - ); - - this.analysesController = new AnalysesController( - pgConnection, - authApi, - userLimitsApi - ); + this.mapRouter = new MapRouter({ collaborators, serverOptions }); + this.templateRouter = new TemplateRouter({ collaborators, serverOptions }); } - regist (api) { + register (app) { // FIXME: we need a better way to reset cache while running tests if (process.env.NODE_ENV === 'test') { - api.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; + app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; } - const mapRouter = router(); - - this.analysisLayergroupController.register(mapRouter); - this.attributesLayergroupController.register(mapRouter); - this.dataviewLayergroupController.register(mapRouter); - this.previewLayergroupController.register(mapRouter); - this.tileLayergroupController.register(mapRouter); - this.anonymousMapController.register(mapRouter); - this.previewTemplateController.register(mapRouter); - this.analysesController.register(mapRouter); - - api.use(this.mapConfigBasePath, mapRouter); - - const templateRouter = router(); - - this.namedMapController.register(templateRouter); - this.tileTemplateController.register(templateRouter); - this.adminTemplateController.register(templateRouter); - - api.use(this.templateBasePath, templateRouter); + this.mapRouter.register(app); + this.templateRouter.register(app); } }; diff --git a/lib/cartodb/controllers/map-router.js b/lib/cartodb/controllers/map-router.js new file mode 100644 index 00000000..0b72572d --- /dev/null +++ b/lib/cartodb/controllers/map-router.js @@ -0,0 +1,160 @@ +const { Router: router } = require('express'); + +const logger = require('../middleware/logger'); +const bodyParser = require('body-parser'); +const servedByHostHeader = require('../middleware/served-by-host-header'); +const stats = require('../middleware/stats'); +const lzmaMiddleware = require('../middleware/lzma'); +const cors = require('../middleware/cors'); +const user = require('../middleware/user'); +const syntaxError = require('../middleware/syntax-error'); +const errorMiddleware = require('../middleware/error-middleware'); + +const AnalysisLayergroupController = require('./layergroup/analysis'); +const AttributesLayergroupController = require('./layergroup/attributes'); +const DataviewLayergroupController = require('./layergroup/dataview'); +const PreviewLayergroupController = require('./layergroup/preview'); +const TileLayergroupController = require('./layergroup/tile'); +const AnonymousMapController = require('./map/anonymous'); +const PreviewTemplateController = require('./template/preview'); +const AnalysesController = require('./analyses'); + +module.exports = class MapRouter { + constructor ({ collaborators, serverOptions }) { + this.serverOptions = serverOptions; + + const { + analysisStatusBackend, + attributesBackend, + dataviewBackend, + previewBackend, + tileBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache, + templateMaps, + mapBackend, + metadataBackend, + mapConfigAdapter, + statsBackend, + layergroupMetadata, + namedMapProviderCache, + tablesExtentApi + } = collaborators; + + this.analysisLayergroupController = new AnalysisLayergroupController( + analysisStatusBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.attributesLayergroupController = new AttributesLayergroupController( + attributesBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.dataviewLayergroupController = new DataviewLayergroupController( + dataviewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.previewLayergroupController = new PreviewLayergroupController( + previewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.tileLayergroupController = new TileLayergroupController( + tileBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ); + + this.anonymousMapController = new AnonymousMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi, + layergroupMetadata + ); + + this.previewTemplateController = new PreviewTemplateController( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentApi, + metadataBackend, + pgConnection, + authApi, + userLimitsApi + ); + + this.analysesController = new AnalysesController( + pgConnection, + authApi, + userLimitsApi + ); + } + + register (app) { + const mapConfigBasePath = this.serverOptions.base_url_mapconfig; + + const mapRouter = router(); + + mapRouter.use(logger(this.serverOptions)); + mapRouter.use(bodyParser.json()); + mapRouter.use(servedByHostHeader()); + mapRouter.use(stats({ + enabled: this.serverOptions.useProfiler, + statsClient: global.statsClient + })); + mapRouter.use(lzmaMiddleware()); + mapRouter.use(cors()); + mapRouter.use(user()); + + this.analysisLayergroupController.register(mapRouter); + this.attributesLayergroupController.register(mapRouter); + this.dataviewLayergroupController.register(mapRouter); + this.previewLayergroupController.register(mapRouter); + this.tileLayergroupController.register(mapRouter); + this.anonymousMapController.register(mapRouter); + this.previewTemplateController.register(mapRouter); + this.analysesController.register(mapRouter); + + mapRouter.use(syntaxError()); + mapRouter.use(errorMiddleware()); + + app.use(mapConfigBasePath, mapRouter); + } +}; diff --git a/lib/cartodb/controllers/template-router.js b/lib/cartodb/controllers/template-router.js new file mode 100644 index 00000000..b7334c9b --- /dev/null +++ b/lib/cartodb/controllers/template-router.js @@ -0,0 +1,92 @@ +const { Router: router } = require('express'); + +const logger = require('../middleware/logger'); +const bodyParser = require('body-parser'); +const servedByHostHeader = require('../middleware/served-by-host-header'); +const stats = require('../middleware/stats'); +const lzmaMiddleware = require('../middleware/lzma'); +const cors = require('../middleware/cors'); +const user = require('../middleware/user'); +const syntaxError = require('../middleware/syntax-error'); +const errorMiddleware = require('../middleware/error-middleware'); + +const NamedMapController = require('./map/named'); +const AdminTemplateController = require('./template/admin'); +const TileTemplateController = require('./template/tile'); + +module.exports = class TemplateRouter { + constructor ({ collaborators, serverOptions }) { + this.serverOptions = serverOptions; + + const { + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi, + layergroupMetadata, + namedMapProviderCache, + tileBackend, + } = collaborators; + + this.namedMapController = new NamedMapController( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + mapConfigAdapter, + statsBackend, + authApi, + layergroupMetadata + ); + + this.tileTemplateController = new TileTemplateController( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authApi, + userLimitsApi + ); + + this.adminTemplateController = new AdminTemplateController( + authApi, + templateMaps, + userLimitsApi + ); + } + + register (app) { + const templateBasePath = this.serverOptions.base_url_templated; + + const templateRouter = router(); + + templateRouter.use(logger(this.serverOptions)); + templateRouter.use(bodyParser.json()); + templateRouter.use(servedByHostHeader()); + templateRouter.use(stats({ + enabled: this.serverOptions.useProfiler, + statsClient: global.statsClient + })); + templateRouter.use(lzmaMiddleware()); + templateRouter.use(cors()); + templateRouter.use(user()); + + this.namedMapController.register(templateRouter); + this.tileTemplateController.register(templateRouter); + this.adminTemplateController.register(templateRouter); + + templateRouter.use(syntaxError()); + templateRouter.use(errorMiddleware()); + + app.use(templateBasePath, templateRouter); + } +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e844f4f4..847b7090 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -5,20 +5,9 @@ const { mapnik } = windshaft; const jsonReplacer = require('./utils/json-replacer'); -const logger = require('./middleware/logger'); -const bodyParser = require('body-parser'); -const servedByHostHeader = require('./middleware/served-by-host-header'); -const stats = require('./middleware/stats'); -const lzmaMiddleware = require('./middleware/lzma'); -const cors = require('./middleware/cors'); -const user = require('./middleware/user'); - const ControllersFactory = require('./controllers/factory'); const ServerInfoController = require('./controllers/server-info'); -const syntaxError = require('./middleware/syntax-error'); -const errorMiddleware = require('./middleware/error-middleware'); - const StatsClient = require('./stats/client'); module.exports = function createServer (serverOptions) { @@ -38,41 +27,13 @@ module.exports = function createServer (serverOptions) { app.disable('etag'); app.set('json replacer', jsonReplacer()); - const api = express.Router(); - - api.use(logger(serverOptions)); - api.use(bodyParser.json()); - api.use(servedByHostHeader()); - api.use(stats({ - enabled: serverOptions.useProfiler, - statsClient: global.statsClient - })); - api.use(lzmaMiddleware()); - api.use(cors()); - api.use(user()); - const controllers = new ControllersFactory({ serverOptions, environmentOptions: global.environment }); - - controllers.regist(api); - - api.use(syntaxError()); - api.use(errorMiddleware()); - - app.use('/', api); - - const monitor = express.Router(); + controllers.register(app); const versions = getAndValidateVersions(serverOptions); + const serverInfoController = new ServerInfoController(versions); - - serverInfoController.register(monitor); - - app.use('/', monitor); - - // FIXME: we need a better way to reset cache while running tests - if (process.env.NODE_ENV === 'test') { - app.layergroupAffectedTablesCache = api.layergroupAffectedTablesCache; - } + serverInfoController.register(app); return app; }; From ef22c46199233058ed624ce65eecdc4829ff7703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 4 Apr 2018 19:15:51 +0200 Subject: [PATCH 030/185] Make send-response middleware generic to the router --- lib/cartodb/controllers/analyses.js | 2 -- lib/cartodb/controllers/factory.js | 2 +- lib/cartodb/controllers/layergroup/analysis.js | 4 +--- .../controllers/layergroup/attributes.js | 4 +--- lib/cartodb/controllers/layergroup/dataview.js | 13 ++++--------- lib/cartodb/controllers/layergroup/preview.js | 7 ++----- lib/cartodb/controllers/layergroup/tile.js | 18 ++++++++---------- lib/cartodb/controllers/map-router.js | 2 ++ lib/cartodb/controllers/map/anonymous.js | 2 -- lib/cartodb/controllers/map/named.js | 2 -- lib/cartodb/controllers/template-router.js | 2 ++ lib/cartodb/controllers/template/admin.js | 16 +++++----------- lib/cartodb/controllers/template/preview.js | 4 +--- lib/cartodb/controllers/template/tile.js | 2 -- 14 files changed, 27 insertions(+), 53 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 9c0e915c..cc6dd3a6 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -6,7 +6,6 @@ const dbConnSetup = require('../middleware/db-conn-setup'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const cacheControlHeader = require('../middleware/cache-control-header'); -const sendResponse = require('../middleware/send-response'); const dbParamsFromResLocals = require('../utils/database-params'); function AnalysesController(pgConnection, authApi, userLimitsApi) { @@ -30,7 +29,6 @@ AnalysesController.prototype.register = function (mapRouter) { getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), prepareResponse(), cacheControlHeader({ ttl: 10, revalidate: true }), - sendResponse(), unauthorizedError() ); }; diff --git a/lib/cartodb/controllers/factory.js b/lib/cartodb/controllers/factory.js index 4e4365e1..33d76c2a 100644 --- a/lib/cartodb/controllers/factory.js +++ b/lib/cartodb/controllers/factory.js @@ -173,8 +173,8 @@ module.exports = class ControllersFactory { app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; } - this.mapRouter.register(app); this.templateRouter.register(app); + this.mapRouter.register(app); } }; diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js index 30371950..36f20991 100644 --- a/lib/cartodb/controllers/layergroup/analysis.js +++ b/lib/cartodb/controllers/layergroup/analysis.js @@ -5,7 +5,6 @@ const dbConnSetup = require('../../middleware/db-conn-setup'); const authorize = require('../../middleware/authorize'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const sendResponse = require('../../middleware/send-response'); const dbParamsFromResLocals = require('../../utils/database-params'); module.exports = class AnalysisLayergroupController { @@ -36,8 +35,7 @@ module.exports = class AnalysisLayergroupController { dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), cleanUpQueryParams(), - analysisNodeStatus(this.analysisStatusBackend), - sendResponse() + analysisNodeStatus(this.analysisStatusBackend) ); } diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js index 9e1aa24f..c4c5af38 100644 --- a/lib/cartodb/controllers/layergroup/attributes.js +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -10,7 +10,6 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); module.exports = class AttributesLayergroupController { constructor ( @@ -50,8 +49,7 @@ module.exports = class AttributesLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); } }; diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js index 04390d23..c13e0eaa 100644 --- a/lib/cartodb/controllers/layergroup/dataview.js +++ b/lib/cartodb/controllers/layergroup/dataview.js @@ -10,7 +10,6 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); const ALLOWED_DATAVIEW_QUERY_PARAMS = [ 'filters', // json @@ -68,8 +67,7 @@ module.exports = class DataviewLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); mapRouter.get( @@ -90,8 +88,7 @@ module.exports = class DataviewLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); mapRouter.get( @@ -112,8 +109,7 @@ module.exports = class DataviewLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); mapRouter.get( @@ -134,8 +130,7 @@ module.exports = class DataviewLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); } }; diff --git a/lib/cartodb/controllers/layergroup/preview.js b/lib/cartodb/controllers/layergroup/preview.js index 1918fd72..8885e45c 100644 --- a/lib/cartodb/controllers/layergroup/preview.js +++ b/lib/cartodb/controllers/layergroup/preview.js @@ -10,7 +10,6 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); module.exports = class PreviewLayergroupController { constructor ( @@ -53,8 +52,7 @@ module.exports = class PreviewLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); mapRouter.get( @@ -76,8 +74,7 @@ module.exports = class PreviewLayergroupController { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); } }; diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js index 39c0e5f7..17c35377 100644 --- a/lib/cartodb/controllers/layergroup/tile.js +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -10,7 +10,6 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); const vectorError = require('../../middleware/vector-error'); const SUPPORTED_FORMATS = { @@ -42,8 +41,10 @@ module.exports = class TileLayergroupController { } register (mapRouter) { + const number = (param) => `${param}(-?\\d+)`; + mapRouter.get( - `/:token/:z/:x/:y@:scale_factor?x.:format`, + `/:token/:${number('z')}/:${number('x')}/:${number('y')}@:${number('scale_factor')}?x.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -64,12 +65,11 @@ module.exports = class TileLayergroupController { incrementSuccessMetrics(global.statsClient), incrementErrorMetrics(global.statsClient), tileError(), - vectorError(), - sendResponse() + vectorError() ); mapRouter.get( - `/:token/:z/:x/:y.:format`, + `/:token/:${number('z')}/:${number('x')}/:${number('y')}.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -90,12 +90,11 @@ module.exports = class TileLayergroupController { incrementSuccessMetrics(global.statsClient), incrementErrorMetrics(global.statsClient), tileError(), - vectorError(), - sendResponse() + vectorError() ); mapRouter.get( - `/:token/:layer/:z/:x/:y.(:format)`, + `/:token/:layer/:${number('z')}/:${number('x')}/:${number('y')}.(:format)`, distinguishLayergroupFromStaticRoute(), layergroupToken(), credentials(), @@ -117,8 +116,7 @@ module.exports = class TileLayergroupController { incrementSuccessMetrics(global.statsClient), incrementErrorMetrics(global.statsClient), tileError(), - vectorError(), - sendResponse() + vectorError() ); } }; diff --git a/lib/cartodb/controllers/map-router.js b/lib/cartodb/controllers/map-router.js index 0b72572d..5cc2bc7b 100644 --- a/lib/cartodb/controllers/map-router.js +++ b/lib/cartodb/controllers/map-router.js @@ -7,6 +7,7 @@ const stats = require('../middleware/stats'); const lzmaMiddleware = require('../middleware/lzma'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); +const sendResponse = require('../middleware/send-response'); const syntaxError = require('../middleware/syntax-error'); const errorMiddleware = require('../middleware/error-middleware'); @@ -152,6 +153,7 @@ module.exports = class MapRouter { this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); + mapRouter.use(sendResponse()); mapRouter.use(syntaxError()); mapRouter.use(errorMiddleware()); diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/controllers/map/anonymous.js index 9575eead..433da3e4 100644 --- a/lib/cartodb/controllers/map/anonymous.js +++ b/lib/cartodb/controllers/map/anonymous.js @@ -18,7 +18,6 @@ const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const layergroupMetadata = require('./middlewares/layergroup-metadata'); const mapError = require('./middlewares/map-error'); -const sendResponse = require('../../middleware/send-response'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -103,7 +102,6 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { layerStats(this.pgConnection, this.statsBackend), layergroupIdHeader(this.templateMaps, useTemplateHash), layergroupMetadata(this.layergroupMetadata, includeQuery), - sendResponse(), mapError({ label, addContext }) ]; }; diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/controllers/map/named.js index 4c99fc09..2258b62f 100644 --- a/lib/cartodb/controllers/map/named.js +++ b/lib/cartodb/controllers/map/named.js @@ -15,7 +15,6 @@ const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const layergroupMetadata = require('./middlewares/layergroup-metadata'); const mapError = require('./middlewares/map-error'); -const sendResponse = require('../../middleware/send-response'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const rateLimit = require('../../middleware/rate-limit'); @@ -114,7 +113,6 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () layerStats(this.pgConnection, this.statsBackend), layergroupIdHeader(this.templateMaps ,useTemplateHash), layergroupMetadata(this.layergroupMetadata, includeQuery), - sendResponse(), mapError({ label, addContext }) ]; }; diff --git a/lib/cartodb/controllers/template-router.js b/lib/cartodb/controllers/template-router.js index b7334c9b..50287b43 100644 --- a/lib/cartodb/controllers/template-router.js +++ b/lib/cartodb/controllers/template-router.js @@ -7,6 +7,7 @@ const stats = require('../middleware/stats'); const lzmaMiddleware = require('../middleware/lzma'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); +const sendResponse = require('../middleware/send-response'); const syntaxError = require('../middleware/syntax-error'); const errorMiddleware = require('../middleware/error-middleware'); @@ -84,6 +85,7 @@ module.exports = class TemplateRouter { this.tileTemplateController.register(templateRouter); this.adminTemplateController.register(templateRouter); + templateRouter.use(sendResponse()); templateRouter.use(syntaxError()); templateRouter.use(errorMiddleware()); diff --git a/lib/cartodb/controllers/template/admin.js b/lib/cartodb/controllers/template/admin.js index d91ca9fd..169f6b20 100644 --- a/lib/cartodb/controllers/template/admin.js +++ b/lib/cartodb/controllers/template/admin.js @@ -2,7 +2,6 @@ const { templateName } = require('../../backends/template_maps'); const credentials = require('../../middleware/credentials'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const sendResponse = require('../../middleware/send-response'); /** * @param {AuthApi} authApi @@ -27,8 +26,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), - createTemplate({ templateMaps: this.templateMaps }), - sendResponse() + createTemplate({ templateMaps: this.templateMaps }) ); templateRouter.put( @@ -37,8 +35,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), - updateTemplate({ templateMaps: this.templateMaps }), - sendResponse() + updateTemplate({ templateMaps: this.templateMaps }) ); templateRouter.get( @@ -46,8 +43,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), - retrieveTemplate({ templateMaps: this.templateMaps }), - sendResponse() + retrieveTemplate({ templateMaps: this.templateMaps }) ); templateRouter.delete( @@ -55,8 +51,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), - destroyTemplate({ templateMaps: this.templateMaps }), - sendResponse() + destroyTemplate({ templateMaps: this.templateMaps }) ); templateRouter.get( @@ -64,8 +59,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), - listTemplates({ templateMaps: this.templateMaps }), - sendResponse() + listTemplates({ templateMaps: this.templateMaps }) ); }; diff --git a/lib/cartodb/controllers/template/preview.js b/lib/cartodb/controllers/template/preview.js index d1abc587..53f583de 100644 --- a/lib/cartodb/controllers/template/preview.js +++ b/lib/cartodb/controllers/template/preview.js @@ -7,7 +7,6 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -69,8 +68,7 @@ PreviewTemplateController.prototype.register = function (mapRouter) { cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - sendResponse() + lastModifiedHeader() ); }; diff --git a/lib/cartodb/controllers/template/tile.js b/lib/cartodb/controllers/template/tile.js index 29b7f852..86404100 100644 --- a/lib/cartodb/controllers/template/tile.js +++ b/lib/cartodb/controllers/template/tile.js @@ -7,7 +7,6 @@ const cacheControlHeader = require('../../middleware/cache-control-header'); const cacheChannelHeader = require('../../middleware/cache-channel-header'); const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); const lastModifiedHeader = require('../../middleware/last-modified-header'); -const sendResponse = require('../../middleware/send-response'); const vectorError = require('../../middleware/vector-error'); const rateLimit = require('../../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -51,7 +50,6 @@ TileTemplateController.prototype.register = function (templateRouter) { cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), - sendResponse(), vectorError() ); }; From 11099c88dc759d75da15a66d9c7858b4d4fa50ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 4 Apr 2018 20:00:59 +0200 Subject: [PATCH 031/185] Improve routing via regular expressions --- lib/cartodb/controllers/layergroup/tile.js | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js index 17c35377..4008afa4 100644 --- a/lib/cartodb/controllers/layergroup/tile.js +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -41,10 +41,11 @@ module.exports = class TileLayergroupController { } register (mapRouter) { - const number = (param) => `${param}(-?\\d+)`; + const number = () => `(-?\\d+)`; + const not = (val) => `(?!${val})([^\/]+?)`; mapRouter.get( - `/:token/:${number('z')}/:${number('x')}/:${number('y')}@:${number('scale_factor')}?x.:format`, + `/:token/:z${number()}/:x${number()}/:y${number()}@:scale_factor${number()}?x.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -69,7 +70,7 @@ module.exports = class TileLayergroupController { ); mapRouter.get( - `/:token/:${number('z')}/:${number('x')}/:${number('y')}.:format`, + `/:token/:z${number()}/:x${number()}/:y${number()}.:format`, layergroupToken(), credentials(), authorize(this.authApi), @@ -94,8 +95,7 @@ module.exports = class TileLayergroupController { ); mapRouter.get( - `/:token/:layer/:${number('z')}/:${number('x')}/:${number('y')}.(:format)`, - distinguishLayergroupFromStaticRoute(), + `/:token${not('static')}/:layer/:z${number()}/:x${number()}/:y${number()}.(:format)`, layergroupToken(), credentials(), authorize(this.authApi), @@ -121,16 +121,6 @@ module.exports = class TileLayergroupController { } }; -function distinguishLayergroupFromStaticRoute () { - return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { - if (req.params.token === 'static') { - return next('route'); - } - - next(); - }; -} - function parseFormat (format = '') { const prettyFormat = format.replace('.', '_'); return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; From 5aa63d35ce0eb594f53dfddcfa4986e1de48d1a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 4 Apr 2018 20:07:53 +0200 Subject: [PATCH 032/185] Add comments --- lib/cartodb/controllers/layergroup/tile.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js index 4008afa4..7837d116 100644 --- a/lib/cartodb/controllers/layergroup/tile.js +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -41,7 +41,9 @@ module.exports = class TileLayergroupController { } register (mapRouter) { + // REGEXP matches with positive and negative numnbers const number = () => `(-?\\d+)`; + // REGEXP doesn't match with `val` const not = (val) => `(?!${val})([^\/]+?)`; mapRouter.get( From 4e754e0d865b175276cdba21c2e658ac49fe99f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 5 Apr 2018 19:42:20 +0200 Subject: [PATCH 033/185] Create API router to handle common stuff among child routers --- .../controllers/{factory.js => api-router.js} | 49 +++++++++++++++++-- lib/cartodb/controllers/map-router.js | 39 +++------------ lib/cartodb/controllers/template-router.js | 39 +++------------ lib/cartodb/server.js | 6 +-- 4 files changed, 61 insertions(+), 72 deletions(-) rename lib/cartodb/controllers/{factory.js => api-router.js} (88%) diff --git a/lib/cartodb/controllers/factory.js b/lib/cartodb/controllers/api-router.js similarity index 88% rename from lib/cartodb/controllers/factory.js rename to lib/cartodb/controllers/api-router.js index 33d76c2a..fe701985 100644 --- a/lib/cartodb/controllers/factory.js +++ b/lib/cartodb/controllers/api-router.js @@ -1,3 +1,5 @@ +const { Router: router } = require('express'); + const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); @@ -38,10 +40,21 @@ const ResourceLocator = require('../models/resource-locator'); const LayergroupMetadata = require('../utils/layergroup-metadata'); const RendererStatsReporter = require('../stats/reporter/renderer'); +const logger = require('../middleware/logger'); +const bodyParser = require('body-parser'); +const servedByHostHeader = require('../middleware/served-by-host-header'); +const stats = require('../middleware/stats'); +const lzmaMiddleware = require('../middleware/lzma'); +const cors = require('../middleware/cors'); +const user = require('../middleware/user'); +const sendResponse = require('../middleware/send-response'); +const syntaxError = require('../middleware/syntax-error'); +const errorMiddleware = require('../middleware/error-middleware'); + const MapRouter = require('./map-router'); const TemplateRouter = require('./template-router'); -module.exports = class ControllersFactory { +module.exports = class ApiRouter { constructor ({ serverOptions, environmentOptions }) { this.serverOptions = serverOptions; @@ -163,8 +176,8 @@ module.exports = class ControllersFactory { tablesExtentApi }; - this.mapRouter = new MapRouter({ collaborators, serverOptions }); - this.templateRouter = new TemplateRouter({ collaborators, serverOptions }); + this.mapRouter = new MapRouter({ collaborators }); + this.templateRouter = new TemplateRouter({ collaborators }); } register (app) { @@ -173,8 +186,34 @@ module.exports = class ControllersFactory { app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; } - this.templateRouter.register(app); - this.mapRouter.register(app); + const apiRouter = router(); + + apiRouter.use(logger(this.serverOptions)); + apiRouter.use(bodyParser.json()); + apiRouter.use(servedByHostHeader()); + apiRouter.use(stats({ + enabled: this.serverOptions.useProfiler, + statsClient: global.statsClient + })); + apiRouter.use(lzmaMiddleware()); + apiRouter.use(cors()); + apiRouter.use(user()); + + this.templateRouter.register(apiRouter); + this.mapRouter.register(apiRouter); + + apiRouter.use(sendResponse()); + apiRouter.use(syntaxError()); + apiRouter.use(errorMiddleware()); + + const paths = [ + '/api/v1', + '/user/:user/api/v1', + '/tiles', + '/database/:dbname' + ]; + + app.use(`(?:${paths.join('|')})`, apiRouter); } }; diff --git a/lib/cartodb/controllers/map-router.js b/lib/cartodb/controllers/map-router.js index 5cc2bc7b..d02cd563 100644 --- a/lib/cartodb/controllers/map-router.js +++ b/lib/cartodb/controllers/map-router.js @@ -1,16 +1,5 @@ const { Router: router } = require('express'); -const logger = require('../middleware/logger'); -const bodyParser = require('body-parser'); -const servedByHostHeader = require('../middleware/served-by-host-header'); -const stats = require('../middleware/stats'); -const lzmaMiddleware = require('../middleware/lzma'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const sendResponse = require('../middleware/send-response'); -const syntaxError = require('../middleware/syntax-error'); -const errorMiddleware = require('../middleware/error-middleware'); - const AnalysisLayergroupController = require('./layergroup/analysis'); const AttributesLayergroupController = require('./layergroup/attributes'); const DataviewLayergroupController = require('./layergroup/dataview'); @@ -21,9 +10,7 @@ const PreviewTemplateController = require('./template/preview'); const AnalysesController = require('./analyses'); module.exports = class MapRouter { - constructor ({ collaborators, serverOptions }) { - this.serverOptions = serverOptions; - + constructor ({ collaborators }) { const { analysisStatusBackend, attributesBackend, @@ -128,22 +115,9 @@ module.exports = class MapRouter { ); } - register (app) { - const mapConfigBasePath = this.serverOptions.base_url_mapconfig; - + register (apiRouter) { const mapRouter = router(); - mapRouter.use(logger(this.serverOptions)); - mapRouter.use(bodyParser.json()); - mapRouter.use(servedByHostHeader()); - mapRouter.use(stats({ - enabled: this.serverOptions.useProfiler, - statsClient: global.statsClient - })); - mapRouter.use(lzmaMiddleware()); - mapRouter.use(cors()); - mapRouter.use(user()); - this.analysisLayergroupController.register(mapRouter); this.attributesLayergroupController.register(mapRouter); this.dataviewLayergroupController.register(mapRouter); @@ -153,10 +127,11 @@ module.exports = class MapRouter { this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); - mapRouter.use(sendResponse()); - mapRouter.use(syntaxError()); - mapRouter.use(errorMiddleware()); + const paths = [ + '/map', + '/layergroup' + ]; - app.use(mapConfigBasePath, mapRouter); + apiRouter.use(`(?:${paths.join('|')})`, mapRouter); } }; diff --git a/lib/cartodb/controllers/template-router.js b/lib/cartodb/controllers/template-router.js index 50287b43..d549dafd 100644 --- a/lib/cartodb/controllers/template-router.js +++ b/lib/cartodb/controllers/template-router.js @@ -1,24 +1,11 @@ const { Router: router } = require('express'); -const logger = require('../middleware/logger'); -const bodyParser = require('body-parser'); -const servedByHostHeader = require('../middleware/served-by-host-header'); -const stats = require('../middleware/stats'); -const lzmaMiddleware = require('../middleware/lzma'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const sendResponse = require('../middleware/send-response'); -const syntaxError = require('../middleware/syntax-error'); -const errorMiddleware = require('../middleware/error-middleware'); - const NamedMapController = require('./map/named'); const AdminTemplateController = require('./template/admin'); const TileTemplateController = require('./template/tile'); module.exports = class TemplateRouter { - constructor ({ collaborators, serverOptions }) { - this.serverOptions = serverOptions; - + constructor ({ collaborators }) { const { pgConnection, templateMaps, @@ -65,30 +52,18 @@ module.exports = class TemplateRouter { ); } - register (app) { - const templateBasePath = this.serverOptions.base_url_templated; - + register (apiRouter) { const templateRouter = router(); - templateRouter.use(logger(this.serverOptions)); - templateRouter.use(bodyParser.json()); - templateRouter.use(servedByHostHeader()); - templateRouter.use(stats({ - enabled: this.serverOptions.useProfiler, - statsClient: global.statsClient - })); - templateRouter.use(lzmaMiddleware()); - templateRouter.use(cors()); - templateRouter.use(user()); - this.namedMapController.register(templateRouter); this.tileTemplateController.register(templateRouter); this.adminTemplateController.register(templateRouter); - templateRouter.use(sendResponse()); - templateRouter.use(syntaxError()); - templateRouter.use(errorMiddleware()); + const paths = [ + '/map/named', + '/template' + ]; - app.use(templateBasePath, templateRouter); + apiRouter.use(`(?:${paths.join('|')})`, templateRouter); } }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 847b7090..ab6fcc62 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -5,7 +5,7 @@ const { mapnik } = windshaft; const jsonReplacer = require('./utils/json-replacer'); -const ControllersFactory = require('./controllers/factory'); +const ApiRouter = require('./controllers/api-router'); const ServerInfoController = require('./controllers/server-info'); const StatsClient = require('./stats/client'); @@ -27,8 +27,8 @@ module.exports = function createServer (serverOptions) { app.disable('etag'); app.set('json replacer', jsonReplacer()); - const controllers = new ControllersFactory({ serverOptions, environmentOptions: global.environment }); - controllers.register(app); + const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment }); + apiRouter.register(app); const versions = getAndValidateVersions(serverOptions); From 61fc15cec07c291d953e4c02704e430ed2592f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 6 Apr 2018 11:13:34 +0200 Subject: [PATCH 034/185] Add comment to indicate deprecated paths --- lib/cartodb/controllers/api-router.js | 4 ++-- lib/cartodb/server_options.js | 3 +++ test/acceptance/ported/support/ported_server_options.js | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/api-router.js b/lib/cartodb/controllers/api-router.js index fe701985..537be27b 100644 --- a/lib/cartodb/controllers/api-router.js +++ b/lib/cartodb/controllers/api-router.js @@ -209,8 +209,8 @@ module.exports = class ApiRouter { const paths = [ '/api/v1', '/user/:user/api/v1', - '/tiles', - '/database/:dbname' + '/tiles', // Deprecated + '/database/:dbname' // Deprecated: used in ported test from windshaft ]; app.use(`(?:${paths.join('|')})`, apiRouter); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 1891e208..adc03003 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -49,12 +49,14 @@ module.exports = { port: global.environment.port, host: global.environment.host }, + // FIXME: Remove it. This is no longer needed, paths are defined in routers // This is for inline maps and table maps base_url: global.environment.base_url_legacy || '/tiles/:table', /// @deprecated with Windshaft-0.17.0 ///base_url_notable: '/tiles', + // FIXME: Remove it. This is no longer needed, paths are defined in routers // This is for Detached maps // // "maps" is the official, while @@ -62,6 +64,7 @@ module.exports = { // base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)', + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)', grainstore: { diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 8b8ec339..1ce262af 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -7,7 +7,9 @@ var overviewsQueryRewriter = new OverviewsQueryRewriter({ }); module.exports = _.extend({}, serverOptions, { + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url: '/database/:dbname/table/:table', + // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_mapconfig: '/database/:dbname/layergroup', grainstore: { datasource: { From 3bfc7d3d23f5da8d2c9646acd794c04f6f7a7b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 6 Apr 2018 13:53:50 +0200 Subject: [PATCH 035/185] Rename controllers folder by routers --- lib/cartodb/{controllers => routers}/analyses.js | 0 lib/cartodb/{controllers => routers}/api-router.js | 0 lib/cartodb/{controllers => routers}/layergroup/analysis.js | 0 lib/cartodb/{controllers => routers}/layergroup/attributes.js | 0 lib/cartodb/{controllers => routers}/layergroup/dataview.js | 0 .../layergroup/middlewares/map-store-map-config-provider.js | 0 lib/cartodb/{controllers => routers}/layergroup/preview.js | 0 lib/cartodb/{controllers => routers}/layergroup/tile.js | 0 lib/cartodb/{controllers => routers}/map-router.js | 0 lib/cartodb/{controllers => routers}/map/anonymous.js | 0 .../map/middlewares/augment-layergroup-data.js | 0 .../map/middlewares/check-json-content-type.js | 0 .../map/middlewares/increment-map-view-count.js | 0 .../{controllers => routers}/map/middlewares/init-profiler.js | 0 .../map/middlewares/last-updated-time-layergroup.js | 0 .../{controllers => routers}/map/middlewares/layer-stats.js | 0 .../map/middlewares/layergroup-id-header.js | 0 .../map/middlewares/layergroup-metadata.js | 0 .../{controllers => routers}/map/middlewares/map-error.js | 0 lib/cartodb/{controllers => routers}/map/named.js | 0 lib/cartodb/{controllers => routers}/server-info.js | 0 lib/cartodb/{controllers => routers}/template-router.js | 0 lib/cartodb/{controllers => routers}/template/admin.js | 0 .../template/middlewares/named-map-provider.js | 0 lib/cartodb/{controllers => routers}/template/preview.js | 0 lib/cartodb/{controllers => routers}/template/tile.js | 0 lib/cartodb/server.js | 4 ++-- 27 files changed, 2 insertions(+), 2 deletions(-) rename lib/cartodb/{controllers => routers}/analyses.js (100%) rename lib/cartodb/{controllers => routers}/api-router.js (100%) rename lib/cartodb/{controllers => routers}/layergroup/analysis.js (100%) rename lib/cartodb/{controllers => routers}/layergroup/attributes.js (100%) rename lib/cartodb/{controllers => routers}/layergroup/dataview.js (100%) rename lib/cartodb/{controllers => routers}/layergroup/middlewares/map-store-map-config-provider.js (100%) rename lib/cartodb/{controllers => routers}/layergroup/preview.js (100%) rename lib/cartodb/{controllers => routers}/layergroup/tile.js (100%) rename lib/cartodb/{controllers => routers}/map-router.js (100%) rename lib/cartodb/{controllers => routers}/map/anonymous.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/augment-layergroup-data.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/check-json-content-type.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/increment-map-view-count.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/init-profiler.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/last-updated-time-layergroup.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/layer-stats.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/layergroup-id-header.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/layergroup-metadata.js (100%) rename lib/cartodb/{controllers => routers}/map/middlewares/map-error.js (100%) rename lib/cartodb/{controllers => routers}/map/named.js (100%) rename lib/cartodb/{controllers => routers}/server-info.js (100%) rename lib/cartodb/{controllers => routers}/template-router.js (100%) rename lib/cartodb/{controllers => routers}/template/admin.js (100%) rename lib/cartodb/{controllers => routers}/template/middlewares/named-map-provider.js (100%) rename lib/cartodb/{controllers => routers}/template/preview.js (100%) rename lib/cartodb/{controllers => routers}/template/tile.js (100%) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/routers/analyses.js similarity index 100% rename from lib/cartodb/controllers/analyses.js rename to lib/cartodb/routers/analyses.js diff --git a/lib/cartodb/controllers/api-router.js b/lib/cartodb/routers/api-router.js similarity index 100% rename from lib/cartodb/controllers/api-router.js rename to lib/cartodb/routers/api-router.js diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/routers/layergroup/analysis.js similarity index 100% rename from lib/cartodb/controllers/layergroup/analysis.js rename to lib/cartodb/routers/layergroup/analysis.js diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/routers/layergroup/attributes.js similarity index 100% rename from lib/cartodb/controllers/layergroup/attributes.js rename to lib/cartodb/routers/layergroup/attributes.js diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/routers/layergroup/dataview.js similarity index 100% rename from lib/cartodb/controllers/layergroup/dataview.js rename to lib/cartodb/routers/layergroup/dataview.js diff --git a/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js b/lib/cartodb/routers/layergroup/middlewares/map-store-map-config-provider.js similarity index 100% rename from lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js rename to lib/cartodb/routers/layergroup/middlewares/map-store-map-config-provider.js diff --git a/lib/cartodb/controllers/layergroup/preview.js b/lib/cartodb/routers/layergroup/preview.js similarity index 100% rename from lib/cartodb/controllers/layergroup/preview.js rename to lib/cartodb/routers/layergroup/preview.js diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/routers/layergroup/tile.js similarity index 100% rename from lib/cartodb/controllers/layergroup/tile.js rename to lib/cartodb/routers/layergroup/tile.js diff --git a/lib/cartodb/controllers/map-router.js b/lib/cartodb/routers/map-router.js similarity index 100% rename from lib/cartodb/controllers/map-router.js rename to lib/cartodb/routers/map-router.js diff --git a/lib/cartodb/controllers/map/anonymous.js b/lib/cartodb/routers/map/anonymous.js similarity index 100% rename from lib/cartodb/controllers/map/anonymous.js rename to lib/cartodb/routers/map/anonymous.js diff --git a/lib/cartodb/controllers/map/middlewares/augment-layergroup-data.js b/lib/cartodb/routers/map/middlewares/augment-layergroup-data.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/augment-layergroup-data.js rename to lib/cartodb/routers/map/middlewares/augment-layergroup-data.js diff --git a/lib/cartodb/controllers/map/middlewares/check-json-content-type.js b/lib/cartodb/routers/map/middlewares/check-json-content-type.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/check-json-content-type.js rename to lib/cartodb/routers/map/middlewares/check-json-content-type.js diff --git a/lib/cartodb/controllers/map/middlewares/increment-map-view-count.js b/lib/cartodb/routers/map/middlewares/increment-map-view-count.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/increment-map-view-count.js rename to lib/cartodb/routers/map/middlewares/increment-map-view-count.js diff --git a/lib/cartodb/controllers/map/middlewares/init-profiler.js b/lib/cartodb/routers/map/middlewares/init-profiler.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/init-profiler.js rename to lib/cartodb/routers/map/middlewares/init-profiler.js diff --git a/lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js b/lib/cartodb/routers/map/middlewares/last-updated-time-layergroup.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js rename to lib/cartodb/routers/map/middlewares/last-updated-time-layergroup.js diff --git a/lib/cartodb/controllers/map/middlewares/layer-stats.js b/lib/cartodb/routers/map/middlewares/layer-stats.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/layer-stats.js rename to lib/cartodb/routers/map/middlewares/layer-stats.js diff --git a/lib/cartodb/controllers/map/middlewares/layergroup-id-header.js b/lib/cartodb/routers/map/middlewares/layergroup-id-header.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/layergroup-id-header.js rename to lib/cartodb/routers/map/middlewares/layergroup-id-header.js diff --git a/lib/cartodb/controllers/map/middlewares/layergroup-metadata.js b/lib/cartodb/routers/map/middlewares/layergroup-metadata.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/layergroup-metadata.js rename to lib/cartodb/routers/map/middlewares/layergroup-metadata.js diff --git a/lib/cartodb/controllers/map/middlewares/map-error.js b/lib/cartodb/routers/map/middlewares/map-error.js similarity index 100% rename from lib/cartodb/controllers/map/middlewares/map-error.js rename to lib/cartodb/routers/map/middlewares/map-error.js diff --git a/lib/cartodb/controllers/map/named.js b/lib/cartodb/routers/map/named.js similarity index 100% rename from lib/cartodb/controllers/map/named.js rename to lib/cartodb/routers/map/named.js diff --git a/lib/cartodb/controllers/server-info.js b/lib/cartodb/routers/server-info.js similarity index 100% rename from lib/cartodb/controllers/server-info.js rename to lib/cartodb/routers/server-info.js diff --git a/lib/cartodb/controllers/template-router.js b/lib/cartodb/routers/template-router.js similarity index 100% rename from lib/cartodb/controllers/template-router.js rename to lib/cartodb/routers/template-router.js diff --git a/lib/cartodb/controllers/template/admin.js b/lib/cartodb/routers/template/admin.js similarity index 100% rename from lib/cartodb/controllers/template/admin.js rename to lib/cartodb/routers/template/admin.js diff --git a/lib/cartodb/controllers/template/middlewares/named-map-provider.js b/lib/cartodb/routers/template/middlewares/named-map-provider.js similarity index 100% rename from lib/cartodb/controllers/template/middlewares/named-map-provider.js rename to lib/cartodb/routers/template/middlewares/named-map-provider.js diff --git a/lib/cartodb/controllers/template/preview.js b/lib/cartodb/routers/template/preview.js similarity index 100% rename from lib/cartodb/controllers/template/preview.js rename to lib/cartodb/routers/template/preview.js diff --git a/lib/cartodb/controllers/template/tile.js b/lib/cartodb/routers/template/tile.js similarity index 100% rename from lib/cartodb/controllers/template/tile.js rename to lib/cartodb/routers/template/tile.js diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index ab6fcc62..40f1195c 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -5,8 +5,8 @@ const { mapnik } = windshaft; const jsonReplacer = require('./utils/json-replacer'); -const ApiRouter = require('./controllers/api-router'); -const ServerInfoController = require('./controllers/server-info'); +const ApiRouter = require('./routers/api-router'); +const ServerInfoController = require('./routers/server-info'); const StatsClient = require('./stats/client'); From 7e68f5270d821f151526c0e82c336483264f1e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 6 Apr 2018 18:20:33 +0200 Subject: [PATCH 036/185] Move application middlewares to routers folder --- lib/cartodb/routers/analyses.js | 12 ++--- lib/cartodb/routers/api-router.js | 18 ++++---- lib/cartodb/routers/layergroup/analysis.js | 12 ++--- lib/cartodb/routers/layergroup/attributes.js | 20 ++++----- lib/cartodb/routers/layergroup/dataview.js | 20 ++++----- lib/cartodb/routers/layergroup/preview.js | 20 ++++----- lib/cartodb/routers/layergroup/tile.js | 22 +++++----- lib/cartodb/routers/map/anonymous.js | 18 ++++---- lib/cartodb/routers/map/named.js | 18 ++++---- .../middlewares}/authorize.js | 0 .../middlewares}/cache-channel-header.js | 0 .../middlewares}/cache-control-header.js | 0 .../middlewares}/clean-up-query-params.js | 0 .../middlewares}/cors.js | 0 .../middlewares}/credentials.js | 0 .../middlewares}/db-conn-setup.js | 0 .../middlewares}/error-middleware.js | 0 .../middlewares}/last-modified-header.js | 0 .../middlewares}/layergroup-token.js | 2 +- .../middlewares}/logger.js | 0 .../middlewares}/lzma.js | 0 .../middlewares}/rate-limit.js | 0 .../middlewares}/send-response.js | 0 .../middlewares}/served-by-host-header.js | 0 .../middlewares}/stats.js | 2 +- .../middlewares}/surrogate-key-header.js | 4 +- .../middlewares}/syntax-error.js | 0 .../middlewares}/user.js | 2 +- .../middlewares}/vector-error.js | 2 +- lib/cartodb/routers/template/admin.js | 4 +- lib/cartodb/routers/template/preview.js | 18 ++++---- lib/cartodb/routers/template/tile.js | 20 ++++----- test/acceptance/rate-limit.test.js | 44 +++++++++---------- test/unit/cartodb/error-middleware.test.js | 10 ++--- test/unit/cartodb/error_messages.test.js | 2 +- test/unit/cartodb/lzmaMiddleware.test.js | 2 +- test/unit/cartodb/prepare-context.test.js | 8 ++-- 37 files changed, 140 insertions(+), 140 deletions(-) rename lib/cartodb/{middleware => routers/middlewares}/authorize.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/cache-channel-header.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/cache-control-header.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/clean-up-query-params.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/cors.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/credentials.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/db-conn-setup.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/error-middleware.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/last-modified-header.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/layergroup-token.js (93%) rename lib/cartodb/{middleware => routers/middlewares}/logger.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/lzma.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/rate-limit.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/send-response.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/served-by-host-header.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/stats.js (93%) rename lib/cartodb/{middleware => routers/middlewares}/surrogate-key-header.js (84%) rename lib/cartodb/{middleware => routers/middlewares}/syntax-error.js (100%) rename lib/cartodb/{middleware => routers/middlewares}/user.js (79%) rename lib/cartodb/{middleware => routers/middlewares}/vector-error.js (95%) diff --git a/lib/cartodb/routers/analyses.js b/lib/cartodb/routers/analyses.js index cc6dd3a6..145d6a0d 100644 --- a/lib/cartodb/routers/analyses.js +++ b/lib/cartodb/routers/analyses.js @@ -1,11 +1,11 @@ const PSQL = require('cartodb-psql'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const credentials = require('../middleware/credentials'); -const authorize = require('../middleware/authorize'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const rateLimit = require('../middleware/rate-limit'); +const cleanUpQueryParams = require('./middlewares/clean-up-query-params'); +const credentials = require('./middlewares/credentials'); +const authorize = require('./middlewares/authorize'); +const dbConnSetup = require('./middlewares/db-conn-setup'); +const rateLimit = require('./middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheControlHeader = require('./middlewares/cache-control-header'); const dbParamsFromResLocals = require('../utils/database-params'); function AnalysesController(pgConnection, authApi, userLimitsApi) { diff --git a/lib/cartodb/routers/api-router.js b/lib/cartodb/routers/api-router.js index 537be27b..ca46d6ae 100644 --- a/lib/cartodb/routers/api-router.js +++ b/lib/cartodb/routers/api-router.js @@ -40,16 +40,16 @@ const ResourceLocator = require('../models/resource-locator'); const LayergroupMetadata = require('../utils/layergroup-metadata'); const RendererStatsReporter = require('../stats/reporter/renderer'); -const logger = require('../middleware/logger'); +const logger = require('./middlewares/logger'); const bodyParser = require('body-parser'); -const servedByHostHeader = require('../middleware/served-by-host-header'); -const stats = require('../middleware/stats'); -const lzmaMiddleware = require('../middleware/lzma'); -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const sendResponse = require('../middleware/send-response'); -const syntaxError = require('../middleware/syntax-error'); -const errorMiddleware = require('../middleware/error-middleware'); +const servedByHostHeader = require('./middlewares/served-by-host-header'); +const stats = require('./middlewares/stats'); +const lzmaMiddleware = require('./middlewares/lzma'); +const cors = require('./middlewares/cors'); +const user = require('./middlewares/user'); +const sendResponse = require('./middlewares/send-response'); +const syntaxError = require('./middlewares/syntax-error'); +const errorMiddleware = require('./middlewares/error-middleware'); const MapRouter = require('./map-router'); const TemplateRouter = require('./template-router'); diff --git a/lib/cartodb/routers/layergroup/analysis.js b/lib/cartodb/routers/layergroup/analysis.js index 36f20991..b2164140 100644 --- a/lib/cartodb/routers/layergroup/analysis.js +++ b/lib/cartodb/routers/layergroup/analysis.js @@ -1,9 +1,9 @@ -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const dbParamsFromResLocals = require('../../utils/database-params'); diff --git a/lib/cartodb/routers/layergroup/attributes.js b/lib/cartodb/routers/layergroup/attributes.js index c4c5af38..3666483a 100644 --- a/lib/cartodb/routers/layergroup/attributes.js +++ b/lib/cartodb/routers/layergroup/attributes.js @@ -1,15 +1,15 @@ -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); module.exports = class AttributesLayergroupController { constructor ( diff --git a/lib/cartodb/routers/layergroup/dataview.js b/lib/cartodb/routers/layergroup/dataview.js index c13e0eaa..559a51d2 100644 --- a/lib/cartodb/routers/layergroup/dataview.js +++ b/lib/cartodb/routers/layergroup/dataview.js @@ -1,15 +1,15 @@ -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); const ALLOWED_DATAVIEW_QUERY_PARAMS = [ 'filters', // json diff --git a/lib/cartodb/routers/layergroup/preview.js b/lib/cartodb/routers/layergroup/preview.js index 8885e45c..d22efffd 100644 --- a/lib/cartodb/routers/layergroup/preview.js +++ b/lib/cartodb/routers/layergroup/preview.js @@ -1,15 +1,15 @@ -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); module.exports = class PreviewLayergroupController { constructor ( diff --git a/lib/cartodb/routers/layergroup/tile.js b/lib/cartodb/routers/layergroup/tile.js index 7837d116..0443ca4e 100644 --- a/lib/cartodb/routers/layergroup/tile.js +++ b/lib/cartodb/routers/layergroup/tile.js @@ -1,16 +1,16 @@ -const layergroupToken = require('../../middleware/layergroup-token'); -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); -const rateLimit = require('../../middleware/rate-limit'); +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const vectorError = require('../../middleware/vector-error'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const vectorError = require('../middlewares/vector-error'); const SUPPORTED_FORMATS = { grid_json: true, diff --git a/lib/cartodb/routers/map/anonymous.js b/lib/cartodb/routers/map/anonymous.js index 433da3e4..3e2a3582 100644 --- a/lib/cartodb/routers/map/anonymous.js +++ b/lib/cartodb/routers/map/anonymous.js @@ -1,25 +1,25 @@ const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); const initProfiler = require('./middlewares/init-profiler'); const checkJsonContentType = require('./middlewares/check-json-content-type'); const incrementMapViewCount = require('./middlewares/increment-map-view-count'); const augmentLayergroupData = require('./middlewares/augment-layergroup-data'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); const layergroupMetadata = require('./middlewares/layergroup-metadata'); const mapError = require('./middlewares/map-error'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); -const rateLimit = require('../../middleware/rate-limit'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** diff --git a/lib/cartodb/routers/map/named.js b/lib/cartodb/routers/map/named.js index 2258b62f..47699e83 100644 --- a/lib/cartodb/routers/map/named.js +++ b/lib/cartodb/routers/map/named.js @@ -1,15 +1,15 @@ -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); const initProfiler = require('./middlewares/init-profiler'); const checkJsonContentType = require('./middlewares/check-json-content-type'); const incrementMapViewCount = require('./middlewares/increment-map-view-count'); const augmentLayergroupData = require('./middlewares/augment-layergroup-data'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); const layerStats = require('./middlewares/layer-stats'); const layergroupIdHeader = require('./middlewares/layergroup-id-header'); @@ -17,7 +17,7 @@ const layergroupMetadata = require('./middlewares/layergroup-metadata'); const mapError = require('./middlewares/map-error'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); -const rateLimit = require('../../middleware/rate-limit'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** diff --git a/lib/cartodb/middleware/authorize.js b/lib/cartodb/routers/middlewares/authorize.js similarity index 100% rename from lib/cartodb/middleware/authorize.js rename to lib/cartodb/routers/middlewares/authorize.js diff --git a/lib/cartodb/middleware/cache-channel-header.js b/lib/cartodb/routers/middlewares/cache-channel-header.js similarity index 100% rename from lib/cartodb/middleware/cache-channel-header.js rename to lib/cartodb/routers/middlewares/cache-channel-header.js diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/routers/middlewares/cache-control-header.js similarity index 100% rename from lib/cartodb/middleware/cache-control-header.js rename to lib/cartodb/routers/middlewares/cache-control-header.js diff --git a/lib/cartodb/middleware/clean-up-query-params.js b/lib/cartodb/routers/middlewares/clean-up-query-params.js similarity index 100% rename from lib/cartodb/middleware/clean-up-query-params.js rename to lib/cartodb/routers/middlewares/clean-up-query-params.js diff --git a/lib/cartodb/middleware/cors.js b/lib/cartodb/routers/middlewares/cors.js similarity index 100% rename from lib/cartodb/middleware/cors.js rename to lib/cartodb/routers/middlewares/cors.js diff --git a/lib/cartodb/middleware/credentials.js b/lib/cartodb/routers/middlewares/credentials.js similarity index 100% rename from lib/cartodb/middleware/credentials.js rename to lib/cartodb/routers/middlewares/credentials.js diff --git a/lib/cartodb/middleware/db-conn-setup.js b/lib/cartodb/routers/middlewares/db-conn-setup.js similarity index 100% rename from lib/cartodb/middleware/db-conn-setup.js rename to lib/cartodb/routers/middlewares/db-conn-setup.js diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/routers/middlewares/error-middleware.js similarity index 100% rename from lib/cartodb/middleware/error-middleware.js rename to lib/cartodb/routers/middlewares/error-middleware.js diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/routers/middlewares/last-modified-header.js similarity index 100% rename from lib/cartodb/middleware/last-modified-header.js rename to lib/cartodb/routers/middlewares/last-modified-header.js diff --git a/lib/cartodb/middleware/layergroup-token.js b/lib/cartodb/routers/middlewares/layergroup-token.js similarity index 93% rename from lib/cartodb/middleware/layergroup-token.js rename to lib/cartodb/routers/middlewares/layergroup-token.js index 1a32e413..0e6553d2 100644 --- a/lib/cartodb/middleware/layergroup-token.js +++ b/lib/cartodb/routers/middlewares/layergroup-token.js @@ -1,4 +1,4 @@ -const LayergroupToken = require('../models/layergroup-token'); +const LayergroupToken = require('../../models/layergroup-token'); const authErrorMessageTemplate = function (signer, user) { return `Cannot use map signature of user "${signer}" on db of user "${user}"`; }; diff --git a/lib/cartodb/middleware/logger.js b/lib/cartodb/routers/middlewares/logger.js similarity index 100% rename from lib/cartodb/middleware/logger.js rename to lib/cartodb/routers/middlewares/logger.js diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/routers/middlewares/lzma.js similarity index 100% rename from lib/cartodb/middleware/lzma.js rename to lib/cartodb/routers/middlewares/lzma.js diff --git a/lib/cartodb/middleware/rate-limit.js b/lib/cartodb/routers/middlewares/rate-limit.js similarity index 100% rename from lib/cartodb/middleware/rate-limit.js rename to lib/cartodb/routers/middlewares/rate-limit.js diff --git a/lib/cartodb/middleware/send-response.js b/lib/cartodb/routers/middlewares/send-response.js similarity index 100% rename from lib/cartodb/middleware/send-response.js rename to lib/cartodb/routers/middlewares/send-response.js diff --git a/lib/cartodb/middleware/served-by-host-header.js b/lib/cartodb/routers/middlewares/served-by-host-header.js similarity index 100% rename from lib/cartodb/middleware/served-by-host-header.js rename to lib/cartodb/routers/middlewares/served-by-host-header.js diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/routers/middlewares/stats.js similarity index 93% rename from lib/cartodb/middleware/stats.js rename to lib/cartodb/routers/middlewares/stats.js index 83ff3054..53d2cf9a 100644 --- a/lib/cartodb/middleware/stats.js +++ b/lib/cartodb/routers/middlewares/stats.js @@ -1,4 +1,4 @@ -const Profiler = require('../stats/profiler_proxy'); +const Profiler = require('../../stats/profiler_proxy'); const debug = require('debug')('windshaft:cartodb:stats'); const onHeaders = require('on-headers'); diff --git a/lib/cartodb/middleware/surrogate-key-header.js b/lib/cartodb/routers/middlewares/surrogate-key-header.js similarity index 84% rename from lib/cartodb/middleware/surrogate-key-header.js rename to lib/cartodb/routers/middlewares/surrogate-key-header.js index 51cec1c1..d2a3243c 100644 --- a/lib/cartodb/middleware/surrogate-key-header.js +++ b/lib/cartodb/routers/middlewares/surrogate-key-header.js @@ -1,5 +1,5 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); +const NamedMapsCacheEntry = require('../../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { return function setSurrogateKeyHeaderMiddleware(req, res, next) { diff --git a/lib/cartodb/middleware/syntax-error.js b/lib/cartodb/routers/middlewares/syntax-error.js similarity index 100% rename from lib/cartodb/middleware/syntax-error.js rename to lib/cartodb/routers/middlewares/syntax-error.js diff --git a/lib/cartodb/middleware/user.js b/lib/cartodb/routers/middlewares/user.js similarity index 79% rename from lib/cartodb/middleware/user.js rename to lib/cartodb/routers/middlewares/user.js index 9c7968bc..551fa001 100644 --- a/lib/cartodb/middleware/user.js +++ b/lib/cartodb/routers/middlewares/user.js @@ -1,4 +1,4 @@ -const CdbRequest = require('../models/cdb_request'); +const CdbRequest = require('../../models/cdb_request'); module.exports = function user () { const cdbRequest = new CdbRequest(); diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/routers/middlewares/vector-error.js similarity index 95% rename from lib/cartodb/middleware/vector-error.js rename to lib/cartodb/routers/middlewares/vector-error.js index 8268ec1f..7e2ec75f 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/routers/middlewares/vector-error.js @@ -1,5 +1,5 @@ const fs = require('fs'); -const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt'); +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../../assets/render-timeout-fallback.mvt'); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { diff --git a/lib/cartodb/routers/template/admin.js b/lib/cartodb/routers/template/admin.js index 169f6b20..b5d1b964 100644 --- a/lib/cartodb/routers/template/admin.js +++ b/lib/cartodb/routers/template/admin.js @@ -1,6 +1,6 @@ const { templateName } = require('../../backends/template_maps'); -const credentials = require('../../middleware/credentials'); -const rateLimit = require('../../middleware/rate-limit'); +const credentials = require('../middlewares/credentials'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** diff --git a/lib/cartodb/routers/template/preview.js b/lib/cartodb/routers/template/preview.js index 53f583de..224579ad 100644 --- a/lib/cartodb/routers/template/preview.js +++ b/lib/cartodb/routers/template/preview.js @@ -1,13 +1,13 @@ -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); const namedMapProvider = require('./middlewares/named-map-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const rateLimit = require('../../middleware/rate-limit'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const DEFAULT_ZOOM_CENTER = { diff --git a/lib/cartodb/routers/template/tile.js b/lib/cartodb/routers/template/tile.js index 86404100..730f7e9d 100644 --- a/lib/cartodb/routers/template/tile.js +++ b/lib/cartodb/routers/template/tile.js @@ -1,14 +1,14 @@ -const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); -const credentials = require('../../middleware/credentials'); -const dbConnSetup = require('../../middleware/db-conn-setup'); -const authorize = require('../../middleware/authorize'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); const namedMapProvider = require('./middlewares/named-map-provider'); -const cacheControlHeader = require('../../middleware/cache-control-header'); -const cacheChannelHeader = require('../../middleware/cache-channel-header'); -const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); -const lastModifiedHeader = require('../../middleware/last-modified-header'); -const vectorError = require('../../middleware/vector-error'); -const rateLimit = require('../../middleware/rate-limit'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); +const vectorError = require('../middlewares/vector-error'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; function TileTemplateController ( diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js index eb9f8954..568db07b 100644 --- a/test/acceptance/rate-limit.test.js +++ b/test/acceptance/rate-limit.test.js @@ -6,10 +6,10 @@ const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const TestClient = require('../support/test-client'); const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api'); -const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit'); +const rateLimitMiddleware = require('../../lib/cartodb/routers/middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; -let userLimitsApi; +let userLimitsApi; let rateLimit; let redisClient; let testClient; @@ -112,11 +112,11 @@ function assertGetLayergroupRequest (status, limit, remaining, reset, retry, don 'Carto-Rate-Limit-Reset': reset } }; - + if(retry) { response.headers['Retry-After'] = retry; } - + testClient.getLayergroup({ response }, err => { assert.ifError(err); if (done) { @@ -133,11 +133,11 @@ function assertRateLimitRequest (status, limit, remaining, reset, retry, done) { "Carto-Rate-Limit-Remaining": remaining, "Carto-Rate-Limit-Reset": reset }; - + if(retry) { expectedHeaders['Retry-After'] = retry; } - + assert.deepEqual(res.headers, expectedHeaders); if(status === 200) { @@ -160,7 +160,7 @@ describe('rate limit', function() { before(function() { global.environment.enabledFeatures.rateLimitsEnabled = true; global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true; - + redisClient = redis.createClient(global.environment.redis.port); testClient = new TestClient(createMapConfig(), 1234); }); @@ -183,7 +183,7 @@ describe('rate limit', function() { done(); }); }); - }); + }); it('should not be rate limited', function (done) { const count = 1; @@ -257,9 +257,9 @@ describe('rate limit middleware', function () { it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { userLimitsApi.metadataBackend.redisCmd( - 8, - 'SCRIPT', - ['FLUSH'], + 8, + 'SCRIPT', + ['FLUSH'], function () { assertRateLimitRequest(200, 1, 0, 1); setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); @@ -277,7 +277,7 @@ describe('rate limit and vector tiles', function () { before(function(done) { global.environment.enabledFeatures.rateLimitsEnabled = true; global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true; - + redisClient = redis.createClient(global.environment.redis.port); const count = 1; const period = 1; @@ -287,9 +287,9 @@ describe('rate limit and vector tiles', function () { testClient = new TestClient(createMapConfig(), 1234); testClient.getLayergroup({status: 200}, (err, res) => { assert.ifError(err); - + layergroupid = res.layergroupid; - + done(); }); }); @@ -336,12 +336,12 @@ describe('rate limit and vector tiles', function () { testClient.getTile(0, 0, 0, tileParams(204, '1', '0', '1'), (err) => { assert.ifError(err); - + testClient.getTile( - 0, - 0, - 0, - tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'), + 0, + 0, + 0, + tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'), (err, res, tile) => { assert.ifError(err); @@ -350,11 +350,11 @@ describe('rate limit and vector tiles', function () { assert.equal(tileJSON.length, 2); assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - + done(); } - ); + ); }); - + }); }); diff --git a/test/unit/cartodb/error-middleware.test.js b/test/unit/cartodb/error-middleware.test.js index 861bec54..74cb8fa5 100644 --- a/test/unit/cartodb/error-middleware.test.js +++ b/test/unit/cartodb/error-middleware.test.js @@ -1,7 +1,7 @@ require('../../support/test_helper.js'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/routers/middlewares/error-middleware'); describe('error-middleware', function() { @@ -28,7 +28,7 @@ describe('error-middleware', function() { error.subtype = 'test subtype'; const errors = [error, error]; - + const req = {}; const res = { headers: {}, @@ -43,7 +43,7 @@ describe('error-middleware', function() { send () {} }; - const errorHeader = { + const errorHeader = { mainError: { statusCode: 400, message: error.message, @@ -133,7 +133,7 @@ describe('error-middleware', function() { error.subtype = badString; const errors = [error, error]; - + const req = {}; const res = { headers: {}, @@ -148,7 +148,7 @@ describe('error-middleware', function() { send () {} }; - const errorHeader = { + const errorHeader = { mainError: { statusCode: 400, message: escapedString, diff --git a/test/unit/cartodb/error_messages.test.js b/test/unit/cartodb/error_messages.test.js index bfe0b03a..dfc489de 100644 --- a/test/unit/cartodb/error_messages.test.js +++ b/test/unit/cartodb/error_messages.test.js @@ -2,7 +2,7 @@ require('../../support/test_helper'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/routers/middlewares/error-middleware'); describe('error messages clean up', function() { diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js index 3ad81962..06219a4d 100644 --- a/test/unit/cartodb/lzmaMiddleware.test.js +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -1,7 +1,7 @@ var assert = require('assert'); var testHelper = require('../../support/test_helper'); -var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma'); +var lzmaMiddleware = require('../../../lib/cartodb/routers/middlewares/lzma'); describe('lzma-middleware', function() { diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index ff311cc6..37265f23 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -7,10 +7,10 @@ var PgConnection = require('../../../lib/cartodb/backends/pg_connection'); var AuthApi = require('../../../lib/cartodb/api/auth_api'); var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); -const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/clean-up-query-params'); -const authorizeMiddleware = require('../../../lib/cartodb/middleware/authorize'); -const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/db-conn-setup'); -const credentialsMiddleware = require('../../../lib/cartodb/middleware/credentials'); +const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/routers/middlewares/clean-up-query-params'); +const authorizeMiddleware = require('../../../lib/cartodb/routers/middlewares/authorize'); +const dbConnSetupMiddleware = require('../../../lib/cartodb/routers/middlewares/db-conn-setup'); +const credentialsMiddleware = require('../../../lib/cartodb/routers/middlewares/credentials'); var windshaft = require('windshaft'); From 6167562758bcbb171f1a5020e46635285c4dbc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 9 Apr 2018 16:18:30 +0200 Subject: [PATCH 037/185] Meet application logic and folder struture --- lib/cartodb/{routers => api}/api-router.js | 14 +++++++------- .../map/analyses-catalog-controller.js} | 14 +++++++------- .../map/analysis-layergroup-controller.js} | 0 .../map/anonymous-map-controller.js} | 18 +++++++++--------- .../map/attributes-layergroup-controller.js} | 2 +- .../map/dataview-layergroup-controller.js} | 2 +- lib/cartodb/{routers => api/map}/map-router.js | 18 +++++++++--------- .../map/preview-layergroup-controller.js} | 2 +- .../map/preview-template-controller.js} | 2 +- .../map/tile-layergroup-controller.js} | 2 +- .../middlewares/augment-layergroup-data.js | 0 .../{routers => api}/middlewares/authorize.js | 0 .../middlewares/cache-channel-header.js | 0 .../middlewares/cache-control-header.js | 0 .../middlewares/check-json-content-type.js | 0 .../middlewares/clean-up-query-params.js | 0 .../{routers => api}/middlewares/cors.js | 0 .../middlewares/credentials.js | 0 .../middlewares/db-conn-setup.js | 0 .../middlewares/error-middleware.js | 0 .../middlewares/increment-map-view-count.js | 0 .../map => api}/middlewares/init-profiler.js | 0 .../middlewares/last-modified-header.js | 0 .../last-updated-time-layergroup.js | 0 .../map => api}/middlewares/layer-stats.js | 0 .../middlewares/layergroup-id-header.js | 0 .../middlewares/layergroup-metadata.js | 0 .../middlewares/layergroup-token.js | 0 .../{routers => api}/middlewares/logger.js | 0 .../{routers => api}/middlewares/lzma.js | 0 .../map => api}/middlewares/map-error.js | 0 .../map-store-map-config-provider.js | 2 +- .../middlewares/named-map-provider.js | 0 .../{routers => api}/middlewares/rate-limit.js | 0 .../middlewares/send-response.js | 0 .../middlewares/served-by-host-header.js | 0 .../{routers => api}/middlewares/stats.js | 0 .../middlewares/surrogate-key-header.js | 0 .../middlewares/syntax-error.js | 0 .../{routers => api}/middlewares/user.js | 0 .../middlewares/vector-error.js | 0 .../template/admin-template-controller.js} | 0 .../template/named-template-controller.js} | 18 +++++++++--------- .../template}/template-router.js | 6 +++--- .../template/tile-template-controller.js} | 2 +- lib/cartodb/{ => backends}/api/auth_api.js | 0 .../{ => backends}/api/filter_stats_api.js | 2 +- .../api/overviews_metadata_api.js | 2 +- .../{ => backends}/api/tables_extent_api.js | 0 .../{ => backends}/api/user_limits_api.js | 0 ...erver-info.js => server-info-controller.js} | 2 +- lib/cartodb/server.js | 4 ++-- test/acceptance/rate-limit.test.js | 4 ++-- .../integration/mapconfig_overviews_adapter.js | 4 ++-- test/integration/overviews-metadata-api.js | 2 +- test/unit/cartodb/error-middleware.test.js | 2 +- test/unit/cartodb/error_messages.test.js | 2 +- test/unit/cartodb/lzmaMiddleware.test.js | 2 +- test/unit/cartodb/prepare-context.test.js | 10 +++++----- 59 files changed, 69 insertions(+), 69 deletions(-) rename lib/cartodb/{routers => api}/api-router.js (96%) rename lib/cartodb/{routers/analyses.js => api/map/analyses-catalog-controller.js} (89%) rename lib/cartodb/{routers/layergroup/analysis.js => api/map/analysis-layergroup-controller.js} (100%) rename lib/cartodb/{routers/map/anonymous.js => api/map/anonymous-map-controller.js} (91%) rename lib/cartodb/{routers/layergroup/attributes.js => api/map/attributes-layergroup-controller.js} (96%) rename lib/cartodb/{routers/layergroup/dataview.js => api/map/dataview-layergroup-controller.js} (98%) rename lib/cartodb/{routers => api/map}/map-router.js (84%) rename lib/cartodb/{routers/layergroup/preview.js => api/map/preview-layergroup-controller.js} (98%) rename lib/cartodb/{routers/template/preview.js => api/map/preview-template-controller.js} (99%) rename lib/cartodb/{routers/layergroup/tile.js => api/map/tile-layergroup-controller.js} (98%) rename lib/cartodb/{routers/map => api}/middlewares/augment-layergroup-data.js (100%) rename lib/cartodb/{routers => api}/middlewares/authorize.js (100%) rename lib/cartodb/{routers => api}/middlewares/cache-channel-header.js (100%) rename lib/cartodb/{routers => api}/middlewares/cache-control-header.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/check-json-content-type.js (100%) rename lib/cartodb/{routers => api}/middlewares/clean-up-query-params.js (100%) rename lib/cartodb/{routers => api}/middlewares/cors.js (100%) rename lib/cartodb/{routers => api}/middlewares/credentials.js (100%) rename lib/cartodb/{routers => api}/middlewares/db-conn-setup.js (100%) rename lib/cartodb/{routers => api}/middlewares/error-middleware.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/increment-map-view-count.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/init-profiler.js (100%) rename lib/cartodb/{routers => api}/middlewares/last-modified-header.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/last-updated-time-layergroup.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/layer-stats.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/layergroup-id-header.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/layergroup-metadata.js (100%) rename lib/cartodb/{routers => api}/middlewares/layergroup-token.js (100%) rename lib/cartodb/{routers => api}/middlewares/logger.js (100%) rename lib/cartodb/{routers => api}/middlewares/lzma.js (100%) rename lib/cartodb/{routers/map => api}/middlewares/map-error.js (100%) rename lib/cartodb/{routers/layergroup => api}/middlewares/map-store-map-config-provider.js (91%) rename lib/cartodb/{routers/template => api}/middlewares/named-map-provider.js (100%) rename lib/cartodb/{routers => api}/middlewares/rate-limit.js (100%) rename lib/cartodb/{routers => api}/middlewares/send-response.js (100%) rename lib/cartodb/{routers => api}/middlewares/served-by-host-header.js (100%) rename lib/cartodb/{routers => api}/middlewares/stats.js (100%) rename lib/cartodb/{routers => api}/middlewares/surrogate-key-header.js (100%) rename lib/cartodb/{routers => api}/middlewares/syntax-error.js (100%) rename lib/cartodb/{routers => api}/middlewares/user.js (100%) rename lib/cartodb/{routers => api}/middlewares/vector-error.js (100%) rename lib/cartodb/{routers/template/admin.js => api/template/admin-template-controller.js} (100%) rename lib/cartodb/{routers/map/named.js => api/template/named-template-controller.js} (91%) rename lib/cartodb/{routers => api/template}/template-router.js (89%) rename lib/cartodb/{routers/template/tile.js => api/template/tile-template-controller.js} (97%) rename lib/cartodb/{ => backends}/api/auth_api.js (100%) rename lib/cartodb/{ => backends}/api/filter_stats_api.js (96%) rename lib/cartodb/{ => backends}/api/overviews_metadata_api.js (95%) rename lib/cartodb/{ => backends}/api/tables_extent_api.js (100%) rename lib/cartodb/{ => backends}/api/user_limits_api.js (100%) rename lib/cartodb/{routers/server-info.js => server-info-controller.js} (96%) diff --git a/lib/cartodb/routers/api-router.js b/lib/cartodb/api/api-router.js similarity index 96% rename from lib/cartodb/routers/api-router.js rename to lib/cartodb/api/api-router.js index ca46d6ae..4608d477 100644 --- a/lib/cartodb/routers/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -13,11 +13,11 @@ const TemplateMaps = require('../backends/template_maps.js'); const PgQueryRunner = require('../backends/pg_query_runner'); const StatsBackend = require('../backends/stats'); -const AuthApi = require('../api/auth_api'); -const UserLimitsApi = require('../api/user_limits_api'); -const OverviewsMetadataApi = require('../api/overviews_metadata_api'); -const FilterStatsApi = require('../api/filter_stats_api'); -const TablesExtentApi = require('../api/tables_extent_api'); +const AuthApi = require('../backends/api/auth_api'); +const UserLimitsApi = require('../backends/api/user_limits_api'); +const OverviewsMetadataApi = require('../backends/api/overviews_metadata_api'); +const FilterStatsApi = require('../backends/api/filter_stats_api'); +const TablesExtentApi = require('../backends/api/tables_extent_api'); const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); const SurrogateKeysCache = require('../cache/surrogate_keys_cache'); @@ -51,8 +51,8 @@ const sendResponse = require('./middlewares/send-response'); const syntaxError = require('./middlewares/syntax-error'); const errorMiddleware = require('./middlewares/error-middleware'); -const MapRouter = require('./map-router'); -const TemplateRouter = require('./template-router'); +const MapRouter = require('./map/map-router'); +const TemplateRouter = require('./template/template-router'); module.exports = class ApiRouter { constructor ({ serverOptions, environmentOptions }) { diff --git a/lib/cartodb/routers/analyses.js b/lib/cartodb/api/map/analyses-catalog-controller.js similarity index 89% rename from lib/cartodb/routers/analyses.js rename to lib/cartodb/api/map/analyses-catalog-controller.js index 145d6a0d..300345c2 100644 --- a/lib/cartodb/routers/analyses.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -1,12 +1,12 @@ const PSQL = require('cartodb-psql'); -const cleanUpQueryParams = require('./middlewares/clean-up-query-params'); -const credentials = require('./middlewares/credentials'); -const authorize = require('./middlewares/authorize'); -const dbConnSetup = require('./middlewares/db-conn-setup'); -const rateLimit = require('./middlewares/rate-limit'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const authorize = require('../middlewares/authorize'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const cacheControlHeader = require('./middlewares/cache-control-header'); -const dbParamsFromResLocals = require('../utils/database-params'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const dbParamsFromResLocals = require('../../utils/database-params'); function AnalysesController(pgConnection, authApi, userLimitsApi) { this.pgConnection = pgConnection; diff --git a/lib/cartodb/routers/layergroup/analysis.js b/lib/cartodb/api/map/analysis-layergroup-controller.js similarity index 100% rename from lib/cartodb/routers/layergroup/analysis.js rename to lib/cartodb/api/map/analysis-layergroup-controller.js diff --git a/lib/cartodb/routers/map/anonymous.js b/lib/cartodb/api/map/anonymous-map-controller.js similarity index 91% rename from lib/cartodb/routers/map/anonymous.js rename to lib/cartodb/api/map/anonymous-map-controller.js index 3e2a3582..4c65988b 100644 --- a/lib/cartodb/routers/map/anonymous.js +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -5,19 +5,19 @@ const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); const credentials = require('../middlewares/credentials'); const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); -const initProfiler = require('./middlewares/init-profiler'); -const checkJsonContentType = require('./middlewares/check-json-content-type'); -const incrementMapViewCount = require('./middlewares/increment-map-view-count'); -const augmentLayergroupData = require('./middlewares/augment-layergroup-data'); +const initProfiler = require('../middlewares/init-profiler'); +const checkJsonContentType = require('../middlewares/check-json-content-type'); +const incrementMapViewCount = require('../middlewares/increment-map-view-count'); +const augmentLayergroupData = require('../middlewares/augment-layergroup-data'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); const lastModifiedHeader = require('../middlewares/last-modified-header'); -const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); -const layerStats = require('./middlewares/layer-stats'); -const layergroupIdHeader = require('./middlewares/layergroup-id-header'); -const layergroupMetadata = require('./middlewares/layergroup-metadata'); -const mapError = require('./middlewares/map-error'); +const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup'); +const layerStats = require('../middlewares/layer-stats'); +const layergroupIdHeader = require('../middlewares/layergroup-id-header'); +const layergroupMetadata = require('../middlewares/layergroup-metadata'); +const mapError = require('../middlewares/map-error'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; diff --git a/lib/cartodb/routers/layergroup/attributes.js b/lib/cartodb/api/map/attributes-layergroup-controller.js similarity index 96% rename from lib/cartodb/routers/layergroup/attributes.js rename to lib/cartodb/api/map/attributes-layergroup-controller.js index 3666483a..52ba5f30 100644 --- a/lib/cartodb/routers/layergroup/attributes.js +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -5,7 +5,7 @@ const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); diff --git a/lib/cartodb/routers/layergroup/dataview.js b/lib/cartodb/api/map/dataview-layergroup-controller.js similarity index 98% rename from lib/cartodb/routers/layergroup/dataview.js rename to lib/cartodb/api/map/dataview-layergroup-controller.js index 559a51d2..219723bc 100644 --- a/lib/cartodb/routers/layergroup/dataview.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -5,7 +5,7 @@ const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); diff --git a/lib/cartodb/routers/map-router.js b/lib/cartodb/api/map/map-router.js similarity index 84% rename from lib/cartodb/routers/map-router.js rename to lib/cartodb/api/map/map-router.js index d02cd563..8303040f 100644 --- a/lib/cartodb/routers/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -1,13 +1,13 @@ const { Router: router } = require('express'); -const AnalysisLayergroupController = require('./layergroup/analysis'); -const AttributesLayergroupController = require('./layergroup/attributes'); -const DataviewLayergroupController = require('./layergroup/dataview'); -const PreviewLayergroupController = require('./layergroup/preview'); -const TileLayergroupController = require('./layergroup/tile'); -const AnonymousMapController = require('./map/anonymous'); -const PreviewTemplateController = require('./template/preview'); -const AnalysesController = require('./analyses'); +const AnalysisLayergroupController = require('./analysis-layergroup-controller'); +const AttributesLayergroupController = require('./attributes-layergroup-controller'); +const DataviewLayergroupController = require('./dataview-layergroup-controller'); +const PreviewLayergroupController = require('./preview-layergroup-controller'); +const TileLayergroupController = require('./tile-layergroup-controller'); +const AnonymousMapController = require('./anonymous-map-controller'); +const PreviewTemplateController = require('./preview-template-controller'); +const AnalysesCatalogController = require('./analyses-catalog-controller'); module.exports = class MapRouter { constructor ({ collaborators }) { @@ -108,7 +108,7 @@ module.exports = class MapRouter { userLimitsApi ); - this.analysesController = new AnalysesController( + this.analysesController = new AnalysesCatalogController( pgConnection, authApi, userLimitsApi diff --git a/lib/cartodb/routers/layergroup/preview.js b/lib/cartodb/api/map/preview-layergroup-controller.js similarity index 98% rename from lib/cartodb/routers/layergroup/preview.js rename to lib/cartodb/api/map/preview-layergroup-controller.js index d22efffd..1fbf7754 100644 --- a/lib/cartodb/routers/layergroup/preview.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -5,7 +5,7 @@ const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); diff --git a/lib/cartodb/routers/template/preview.js b/lib/cartodb/api/map/preview-template-controller.js similarity index 99% rename from lib/cartodb/routers/template/preview.js rename to lib/cartodb/api/map/preview-template-controller.js index 224579ad..ecce4b7f 100644 --- a/lib/cartodb/routers/template/preview.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -2,7 +2,7 @@ const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); const credentials = require('../middlewares/credentials'); const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); -const namedMapProvider = require('./middlewares/named-map-provider'); +const namedMapProvider = require('../middlewares/named-map-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); diff --git a/lib/cartodb/routers/layergroup/tile.js b/lib/cartodb/api/map/tile-layergroup-controller.js similarity index 98% rename from lib/cartodb/routers/layergroup/tile.js rename to lib/cartodb/api/map/tile-layergroup-controller.js index 0443ca4e..34abcbda 100644 --- a/lib/cartodb/routers/layergroup/tile.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -5,7 +5,7 @@ const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); diff --git a/lib/cartodb/routers/map/middlewares/augment-layergroup-data.js b/lib/cartodb/api/middlewares/augment-layergroup-data.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/augment-layergroup-data.js rename to lib/cartodb/api/middlewares/augment-layergroup-data.js diff --git a/lib/cartodb/routers/middlewares/authorize.js b/lib/cartodb/api/middlewares/authorize.js similarity index 100% rename from lib/cartodb/routers/middlewares/authorize.js rename to lib/cartodb/api/middlewares/authorize.js diff --git a/lib/cartodb/routers/middlewares/cache-channel-header.js b/lib/cartodb/api/middlewares/cache-channel-header.js similarity index 100% rename from lib/cartodb/routers/middlewares/cache-channel-header.js rename to lib/cartodb/api/middlewares/cache-channel-header.js diff --git a/lib/cartodb/routers/middlewares/cache-control-header.js b/lib/cartodb/api/middlewares/cache-control-header.js similarity index 100% rename from lib/cartodb/routers/middlewares/cache-control-header.js rename to lib/cartodb/api/middlewares/cache-control-header.js diff --git a/lib/cartodb/routers/map/middlewares/check-json-content-type.js b/lib/cartodb/api/middlewares/check-json-content-type.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/check-json-content-type.js rename to lib/cartodb/api/middlewares/check-json-content-type.js diff --git a/lib/cartodb/routers/middlewares/clean-up-query-params.js b/lib/cartodb/api/middlewares/clean-up-query-params.js similarity index 100% rename from lib/cartodb/routers/middlewares/clean-up-query-params.js rename to lib/cartodb/api/middlewares/clean-up-query-params.js diff --git a/lib/cartodb/routers/middlewares/cors.js b/lib/cartodb/api/middlewares/cors.js similarity index 100% rename from lib/cartodb/routers/middlewares/cors.js rename to lib/cartodb/api/middlewares/cors.js diff --git a/lib/cartodb/routers/middlewares/credentials.js b/lib/cartodb/api/middlewares/credentials.js similarity index 100% rename from lib/cartodb/routers/middlewares/credentials.js rename to lib/cartodb/api/middlewares/credentials.js diff --git a/lib/cartodb/routers/middlewares/db-conn-setup.js b/lib/cartodb/api/middlewares/db-conn-setup.js similarity index 100% rename from lib/cartodb/routers/middlewares/db-conn-setup.js rename to lib/cartodb/api/middlewares/db-conn-setup.js diff --git a/lib/cartodb/routers/middlewares/error-middleware.js b/lib/cartodb/api/middlewares/error-middleware.js similarity index 100% rename from lib/cartodb/routers/middlewares/error-middleware.js rename to lib/cartodb/api/middlewares/error-middleware.js diff --git a/lib/cartodb/routers/map/middlewares/increment-map-view-count.js b/lib/cartodb/api/middlewares/increment-map-view-count.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/increment-map-view-count.js rename to lib/cartodb/api/middlewares/increment-map-view-count.js diff --git a/lib/cartodb/routers/map/middlewares/init-profiler.js b/lib/cartodb/api/middlewares/init-profiler.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/init-profiler.js rename to lib/cartodb/api/middlewares/init-profiler.js diff --git a/lib/cartodb/routers/middlewares/last-modified-header.js b/lib/cartodb/api/middlewares/last-modified-header.js similarity index 100% rename from lib/cartodb/routers/middlewares/last-modified-header.js rename to lib/cartodb/api/middlewares/last-modified-header.js diff --git a/lib/cartodb/routers/map/middlewares/last-updated-time-layergroup.js b/lib/cartodb/api/middlewares/last-updated-time-layergroup.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/last-updated-time-layergroup.js rename to lib/cartodb/api/middlewares/last-updated-time-layergroup.js diff --git a/lib/cartodb/routers/map/middlewares/layer-stats.js b/lib/cartodb/api/middlewares/layer-stats.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/layer-stats.js rename to lib/cartodb/api/middlewares/layer-stats.js diff --git a/lib/cartodb/routers/map/middlewares/layergroup-id-header.js b/lib/cartodb/api/middlewares/layergroup-id-header.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/layergroup-id-header.js rename to lib/cartodb/api/middlewares/layergroup-id-header.js diff --git a/lib/cartodb/routers/map/middlewares/layergroup-metadata.js b/lib/cartodb/api/middlewares/layergroup-metadata.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/layergroup-metadata.js rename to lib/cartodb/api/middlewares/layergroup-metadata.js diff --git a/lib/cartodb/routers/middlewares/layergroup-token.js b/lib/cartodb/api/middlewares/layergroup-token.js similarity index 100% rename from lib/cartodb/routers/middlewares/layergroup-token.js rename to lib/cartodb/api/middlewares/layergroup-token.js diff --git a/lib/cartodb/routers/middlewares/logger.js b/lib/cartodb/api/middlewares/logger.js similarity index 100% rename from lib/cartodb/routers/middlewares/logger.js rename to lib/cartodb/api/middlewares/logger.js diff --git a/lib/cartodb/routers/middlewares/lzma.js b/lib/cartodb/api/middlewares/lzma.js similarity index 100% rename from lib/cartodb/routers/middlewares/lzma.js rename to lib/cartodb/api/middlewares/lzma.js diff --git a/lib/cartodb/routers/map/middlewares/map-error.js b/lib/cartodb/api/middlewares/map-error.js similarity index 100% rename from lib/cartodb/routers/map/middlewares/map-error.js rename to lib/cartodb/api/middlewares/map-error.js diff --git a/lib/cartodb/routers/layergroup/middlewares/map-store-map-config-provider.js b/lib/cartodb/api/middlewares/map-store-map-config-provider.js similarity index 91% rename from lib/cartodb/routers/layergroup/middlewares/map-store-map-config-provider.js rename to lib/cartodb/api/middlewares/map-store-map-config-provider.js index 7d64cb01..f84dd3fc 100644 --- a/lib/cartodb/routers/layergroup/middlewares/map-store-map-config-provider.js +++ b/lib/cartodb/api/middlewares/map-store-map-config-provider.js @@ -1,4 +1,4 @@ -const MapStoreMapConfigProvider = require('../../../models/mapconfig/provider/map-store-provider'); +const MapStoreMapConfigProvider = require('../../models/mapconfig/provider/map-store-provider'); module.exports = function createMapStoreMapConfigProvider ( mapStore, diff --git a/lib/cartodb/routers/template/middlewares/named-map-provider.js b/lib/cartodb/api/middlewares/named-map-provider.js similarity index 100% rename from lib/cartodb/routers/template/middlewares/named-map-provider.js rename to lib/cartodb/api/middlewares/named-map-provider.js diff --git a/lib/cartodb/routers/middlewares/rate-limit.js b/lib/cartodb/api/middlewares/rate-limit.js similarity index 100% rename from lib/cartodb/routers/middlewares/rate-limit.js rename to lib/cartodb/api/middlewares/rate-limit.js diff --git a/lib/cartodb/routers/middlewares/send-response.js b/lib/cartodb/api/middlewares/send-response.js similarity index 100% rename from lib/cartodb/routers/middlewares/send-response.js rename to lib/cartodb/api/middlewares/send-response.js diff --git a/lib/cartodb/routers/middlewares/served-by-host-header.js b/lib/cartodb/api/middlewares/served-by-host-header.js similarity index 100% rename from lib/cartodb/routers/middlewares/served-by-host-header.js rename to lib/cartodb/api/middlewares/served-by-host-header.js diff --git a/lib/cartodb/routers/middlewares/stats.js b/lib/cartodb/api/middlewares/stats.js similarity index 100% rename from lib/cartodb/routers/middlewares/stats.js rename to lib/cartodb/api/middlewares/stats.js diff --git a/lib/cartodb/routers/middlewares/surrogate-key-header.js b/lib/cartodb/api/middlewares/surrogate-key-header.js similarity index 100% rename from lib/cartodb/routers/middlewares/surrogate-key-header.js rename to lib/cartodb/api/middlewares/surrogate-key-header.js diff --git a/lib/cartodb/routers/middlewares/syntax-error.js b/lib/cartodb/api/middlewares/syntax-error.js similarity index 100% rename from lib/cartodb/routers/middlewares/syntax-error.js rename to lib/cartodb/api/middlewares/syntax-error.js diff --git a/lib/cartodb/routers/middlewares/user.js b/lib/cartodb/api/middlewares/user.js similarity index 100% rename from lib/cartodb/routers/middlewares/user.js rename to lib/cartodb/api/middlewares/user.js diff --git a/lib/cartodb/routers/middlewares/vector-error.js b/lib/cartodb/api/middlewares/vector-error.js similarity index 100% rename from lib/cartodb/routers/middlewares/vector-error.js rename to lib/cartodb/api/middlewares/vector-error.js diff --git a/lib/cartodb/routers/template/admin.js b/lib/cartodb/api/template/admin-template-controller.js similarity index 100% rename from lib/cartodb/routers/template/admin.js rename to lib/cartodb/api/template/admin-template-controller.js diff --git a/lib/cartodb/routers/map/named.js b/lib/cartodb/api/template/named-template-controller.js similarity index 91% rename from lib/cartodb/routers/map/named.js rename to lib/cartodb/api/template/named-template-controller.js index 47699e83..140523b1 100644 --- a/lib/cartodb/routers/map/named.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -2,19 +2,19 @@ const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); const credentials = require('../middlewares/credentials'); const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); -const initProfiler = require('./middlewares/init-profiler'); -const checkJsonContentType = require('./middlewares/check-json-content-type'); -const incrementMapViewCount = require('./middlewares/increment-map-view-count'); -const augmentLayergroupData = require('./middlewares/augment-layergroup-data'); +const initProfiler = require('../middlewares/init-profiler'); +const checkJsonContentType = require('../middlewares/check-json-content-type'); +const incrementMapViewCount = require('../middlewares/increment-map-view-count'); +const augmentLayergroupData = require('../middlewares/augment-layergroup-data'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); const lastModifiedHeader = require('../middlewares/last-modified-header'); -const lastUpdatedTimeLayergroup = require('./middlewares/last-updated-time-layergroup'); -const layerStats = require('./middlewares/layer-stats'); -const layergroupIdHeader = require('./middlewares/layergroup-id-header'); -const layergroupMetadata = require('./middlewares/layergroup-metadata'); -const mapError = require('./middlewares/map-error'); +const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup'); +const layerStats = require('../middlewares/layer-stats'); +const layergroupIdHeader = require('../middlewares/layergroup-id-header'); +const layergroupMetadata = require('../middlewares/layergroup-metadata'); +const mapError = require('../middlewares/map-error'); const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const rateLimit = require('../middlewares/rate-limit'); diff --git a/lib/cartodb/routers/template-router.js b/lib/cartodb/api/template/template-router.js similarity index 89% rename from lib/cartodb/routers/template-router.js rename to lib/cartodb/api/template/template-router.js index d549dafd..b4a70715 100644 --- a/lib/cartodb/routers/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -1,8 +1,8 @@ const { Router: router } = require('express'); -const NamedMapController = require('./map/named'); -const AdminTemplateController = require('./template/admin'); -const TileTemplateController = require('./template/tile'); +const NamedMapController = require('./named-template-controller'); +const AdminTemplateController = require('./admin-template-controller'); +const TileTemplateController = require('./tile-template-controller'); module.exports = class TemplateRouter { constructor ({ collaborators }) { diff --git a/lib/cartodb/routers/template/tile.js b/lib/cartodb/api/template/tile-template-controller.js similarity index 97% rename from lib/cartodb/routers/template/tile.js rename to lib/cartodb/api/template/tile-template-controller.js index 730f7e9d..11c0bc19 100644 --- a/lib/cartodb/routers/template/tile.js +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -2,7 +2,7 @@ const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); const credentials = require('../middlewares/credentials'); const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); -const namedMapProvider = require('./middlewares/named-map-provider'); +const namedMapProvider = require('../middlewares/named-map-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/backends/api/auth_api.js similarity index 100% rename from lib/cartodb/api/auth_api.js rename to lib/cartodb/backends/api/auth_api.js diff --git a/lib/cartodb/api/filter_stats_api.js b/lib/cartodb/backends/api/filter_stats_api.js similarity index 96% rename from lib/cartodb/api/filter_stats_api.js rename to lib/cartodb/backends/api/filter_stats_api.js index 31e90a25..72fd970f 100644 --- a/lib/cartodb/api/filter_stats_api.js +++ b/lib/cartodb/backends/api/filter_stats_api.js @@ -1,6 +1,6 @@ var _ = require('underscore'); var step = require('step'); -var AnalysisFilter = require('../models/filter/analysis'); +var AnalysisFilter = require('../../models/filter/analysis'); function FilterStatsApi(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; diff --git a/lib/cartodb/api/overviews_metadata_api.js b/lib/cartodb/backends/api/overviews_metadata_api.js similarity index 95% rename from lib/cartodb/api/overviews_metadata_api.js rename to lib/cartodb/backends/api/overviews_metadata_api.js index 55b4966f..dda51d7e 100644 --- a/lib/cartodb/api/overviews_metadata_api.js +++ b/lib/cartodb/backends/api/overviews_metadata_api.js @@ -1,4 +1,4 @@ -var SubstitutionTokens = require('../utils/substitution-tokens'); +var SubstitutionTokens = require('../../utils/substitution-tokens'); function OverviewsMetadataApi(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; diff --git a/lib/cartodb/api/tables_extent_api.js b/lib/cartodb/backends/api/tables_extent_api.js similarity index 100% rename from lib/cartodb/api/tables_extent_api.js rename to lib/cartodb/backends/api/tables_extent_api.js diff --git a/lib/cartodb/api/user_limits_api.js b/lib/cartodb/backends/api/user_limits_api.js similarity index 100% rename from lib/cartodb/api/user_limits_api.js rename to lib/cartodb/backends/api/user_limits_api.js diff --git a/lib/cartodb/routers/server-info.js b/lib/cartodb/server-info-controller.js similarity index 96% rename from lib/cartodb/routers/server-info.js rename to lib/cartodb/server-info-controller.js index 12f6432b..17e0649c 100644 --- a/lib/cartodb/routers/server-info.js +++ b/lib/cartodb/server-info-controller.js @@ -1,4 +1,4 @@ -var HealthCheck = require('../monitoring/health_check'); +var HealthCheck = require('./monitoring/health_check'); var WELCOME_MSG = "This is the CartoDB Maps API, " + "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html"; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 40f1195c..ea51f2db 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -5,8 +5,8 @@ const { mapnik } = windshaft; const jsonReplacer = require('./utils/json-replacer'); -const ApiRouter = require('./routers/api-router'); -const ServerInfoController = require('./routers/server-info'); +const ApiRouter = require('./api/api-router'); +const ServerInfoController = require('./server-info-controller'); const StatsClient = require('./stats/client'); diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js index 568db07b..5b7cdbc0 100644 --- a/test/acceptance/rate-limit.test.js +++ b/test/acceptance/rate-limit.test.js @@ -5,8 +5,8 @@ const redis = require('redis'); const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const TestClient = require('../support/test-client'); -const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api'); -const rateLimitMiddleware = require('../../lib/cartodb/routers/middlewares/rate-limit'); +const UserLimitsApi = require('../../lib/cartodb/backends/api/user_limits_api'); +const rateLimitMiddleware = require('../../lib/cartodb/api/middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; let userLimitsApi; diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index 87a87935..79196e3b 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -5,8 +5,8 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); -var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api'); +var OverviewsMetadataApi = require('../../lib/cartodb/backends/api/overviews_metadata_api'); +var FilterStatsApi = require('../../lib/cartodb/backends/api/filter_stats_api'); var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter'); var redisPool = new RedisPool(global.environment.redis); diff --git a/test/integration/overviews-metadata-api.js b/test/integration/overviews-metadata-api.js index 4d015870..e51c2129 100644 --- a/test/integration/overviews-metadata-api.js +++ b/test/integration/overviews-metadata-api.js @@ -7,7 +7,7 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); +var OverviewsMetadataApi = require('../../lib/cartodb/backends/api/overviews_metadata_api'); describe('OverviewsMetadataApi', function() { diff --git a/test/unit/cartodb/error-middleware.test.js b/test/unit/cartodb/error-middleware.test.js index 74cb8fa5..88b1ed6e 100644 --- a/test/unit/cartodb/error-middleware.test.js +++ b/test/unit/cartodb/error-middleware.test.js @@ -1,7 +1,7 @@ require('../../support/test_helper.js'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/routers/middlewares/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware'); describe('error-middleware', function() { diff --git a/test/unit/cartodb/error_messages.test.js b/test/unit/cartodb/error_messages.test.js index dfc489de..ecc6fe3c 100644 --- a/test/unit/cartodb/error_messages.test.js +++ b/test/unit/cartodb/error_messages.test.js @@ -2,7 +2,7 @@ require('../../support/test_helper'); var assert = require('assert'); -var errorMiddleware = require('../../../lib/cartodb/routers/middlewares/error-middleware'); +var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware'); describe('error messages clean up', function() { diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js index 06219a4d..9701d7ca 100644 --- a/test/unit/cartodb/lzmaMiddleware.test.js +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -1,7 +1,7 @@ var assert = require('assert'); var testHelper = require('../../support/test_helper'); -var lzmaMiddleware = require('../../../lib/cartodb/routers/middlewares/lzma'); +var lzmaMiddleware = require('../../../lib/cartodb/api/middlewares/lzma'); describe('lzma-middleware', function() { diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index 37265f23..e9f0a320 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -4,13 +4,13 @@ var _ = require('underscore'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../../lib/cartodb/backends/pg_connection'); -var AuthApi = require('../../../lib/cartodb/api/auth_api'); +var AuthApi = require('../../../lib/cartodb/backends/api/auth_api'); var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); -const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/routers/middlewares/clean-up-query-params'); -const authorizeMiddleware = require('../../../lib/cartodb/routers/middlewares/authorize'); -const dbConnSetupMiddleware = require('../../../lib/cartodb/routers/middlewares/db-conn-setup'); -const credentialsMiddleware = require('../../../lib/cartodb/routers/middlewares/credentials'); +const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/api/middlewares/clean-up-query-params'); +const authorizeMiddleware = require('../../../lib/cartodb/api/middlewares/authorize'); +const dbConnSetupMiddleware = require('../../../lib/cartodb/api/middlewares/db-conn-setup'); +const credentialsMiddleware = require('../../../lib/cartodb/api/middlewares/credentials'); var windshaft = require('windshaft'); From 8dec4814a9a025dd0c329d4430d55973aa5bb7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 9 Apr 2018 18:08:56 +0200 Subject: [PATCH 038/185] Rename AuthAppi by AuthBackend --- lib/cartodb/api/api-router.js | 6 +++--- .../api/map/analyses-catalog-controller.js | 6 +++--- .../api/map/analysis-layergroup-controller.js | 6 +++--- .../api/map/anonymous-map-controller.js | 8 ++++---- .../map/attributes-layergroup-controller.js | 6 +++--- .../api/map/dataview-layergroup-controller.js | 12 +++++------ lib/cartodb/api/map/map-router.js | 18 ++++++++--------- .../api/map/preview-layergroup-controller.js | 8 ++++---- .../api/map/preview-template-controller.js | 6 +++--- .../api/map/tile-layergroup-controller.js | 10 +++++----- lib/cartodb/api/middlewares/authorize.js | 4 ++-- .../api/template/admin-template-controller.js | 20 +++++++++---------- .../api/template/named-template-controller.js | 8 ++++---- lib/cartodb/api/template/template-router.js | 8 ++++---- .../api/template/tile-template-controller.js | 6 +++--- .../backends/{api/auth_api.js => auth.js} | 12 +++++------ test/unit/cartodb/prepare-context.test.js | 6 +++--- 17 files changed, 75 insertions(+), 75 deletions(-) rename lib/cartodb/backends/{api/auth_api.js => auth.js} (94%) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 4608d477..f5b40c9e 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -12,8 +12,8 @@ const DataviewBackend = require('../backends/dataview'); const TemplateMaps = require('../backends/template_maps.js'); const PgQueryRunner = require('../backends/pg_query_runner'); const StatsBackend = require('../backends/stats'); +const AuthBackend = require('../backends/auth'); -const AuthApi = require('../backends/api/auth_api'); const UserLimitsApi = require('../backends/api/user_limits_api'); const OverviewsMetadataApi = require('../backends/api/overviews_metadata_api'); const FilterStatsApi = require('../backends/api/filter_stats_api'); @@ -113,7 +113,7 @@ module.exports = class ApiRouter { rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled } }); - const authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); + const authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps); const layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); @@ -164,7 +164,7 @@ module.exports = class ApiRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache, templateMaps, mapBackend, diff --git a/lib/cartodb/api/map/analyses-catalog-controller.js b/lib/cartodb/api/map/analyses-catalog-controller.js index 300345c2..855666dd 100644 --- a/lib/cartodb/api/map/analyses-catalog-controller.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -8,9 +8,9 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const cacheControlHeader = require('../middlewares/cache-control-header'); const dbParamsFromResLocals = require('../../utils/database-params'); -function AnalysesController(pgConnection, authApi, userLimitsApi) { +function AnalysesController(pgConnection, authBackend, userLimitsApi) { this.pgConnection = pgConnection; - this.authApi = authApi; + this.authBackend = authBackend; this.userLimitsApi = userLimitsApi; } @@ -20,7 +20,7 @@ AnalysesController.prototype.register = function (mapRouter) { mapRouter.get( `/analyses/catalog`, credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), cleanUpQueryParams(), diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js index b2164140..e0cf3222 100644 --- a/lib/cartodb/api/map/analysis-layergroup-controller.js +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -14,7 +14,7 @@ module.exports = class AnalysisLayergroupController { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.analysisStatusBackend = analysisStatusBackend; @@ -22,7 +22,7 @@ module.exports = class AnalysisLayergroupController { this.mapStore = mapStore; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } @@ -31,7 +31,7 @@ module.exports = class AnalysisLayergroupController { `/:token/analysis/node/:nodeId`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), cleanUpQueryParams(), diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js index 4c65988b..a20b1407 100644 --- a/lib/cartodb/api/map/anonymous-map-controller.js +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -23,7 +23,7 @@ const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** - * @param {AuthApi} authApi + * @param {AuthBackend} authBackend * @param {PgConnection} pgConnection * @param {TemplateMaps} templateMaps * @param {MapBackend} mapBackend @@ -45,7 +45,7 @@ function AnonymousMapController ( layergroupAffectedTables, mapConfigAdapter, statsBackend, - authApi, + authBackend, layergroupMetadata ) { this.pgConnection = pgConnection; @@ -57,7 +57,7 @@ function AnonymousMapController ( this.layergroupAffectedTables = layergroupAffectedTables; this.mapConfigAdapter = mapConfigAdapter; this.statsBackend = statsBackend; - this.authApi = authApi; + this.authBackend = authBackend; this.layergroupMetadata = layergroupMetadata; } @@ -78,7 +78,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { return [ credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), cleanUpQueryParams(['aggregation']), diff --git a/lib/cartodb/api/map/attributes-layergroup-controller.js b/lib/cartodb/api/map/attributes-layergroup-controller.js index 52ba5f30..17af3df7 100644 --- a/lib/cartodb/api/map/attributes-layergroup-controller.js +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -18,7 +18,7 @@ module.exports = class AttributesLayergroupController { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.attributesBackend = attributesBackend; @@ -26,7 +26,7 @@ module.exports = class AttributesLayergroupController { this.mapStore = mapStore; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } @@ -35,7 +35,7 @@ module.exports = class AttributesLayergroupController { `/:token/:layer/attributes/:fid`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), cleanUpQueryParams(), diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js index 219723bc..84b4c905 100644 --- a/lib/cartodb/api/map/dataview-layergroup-controller.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -33,7 +33,7 @@ module.exports = class DataviewLayergroupController { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.dataviewBackend = dataviewBackend; @@ -41,7 +41,7 @@ module.exports = class DataviewLayergroupController { this.mapStore = mapStore; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } @@ -53,7 +53,7 @@ module.exports = class DataviewLayergroupController { `/:token/dataview/:dataviewName`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), @@ -74,7 +74,7 @@ module.exports = class DataviewLayergroupController { `/:token/:layer/widget/:dataviewName`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), @@ -95,7 +95,7 @@ module.exports = class DataviewLayergroupController { `/:token/dataview/:dataviewName/search`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), @@ -116,7 +116,7 @@ module.exports = class DataviewLayergroupController { `/:token/:layer/widget/:dataviewName/search`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index 8303040f..2e2a3f4a 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -21,7 +21,7 @@ module.exports = class MapRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache, templateMaps, mapBackend, @@ -39,7 +39,7 @@ module.exports = class MapRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ); @@ -49,7 +49,7 @@ module.exports = class MapRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ); @@ -59,7 +59,7 @@ module.exports = class MapRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ); @@ -69,7 +69,7 @@ module.exports = class MapRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ); @@ -79,7 +79,7 @@ module.exports = class MapRouter { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ); @@ -93,7 +93,7 @@ module.exports = class MapRouter { layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, - authApi, + authBackend, layergroupMetadata ); @@ -104,13 +104,13 @@ module.exports = class MapRouter { tablesExtentApi, metadataBackend, pgConnection, - authApi, + authBackend, userLimitsApi ); this.analysesController = new AnalysesCatalogController( pgConnection, - authApi, + authBackend, userLimitsApi ); } diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js index 1fbf7754..484b831c 100644 --- a/lib/cartodb/api/map/preview-layergroup-controller.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -18,7 +18,7 @@ module.exports = class PreviewLayergroupController { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.previewBackend = previewBackend; @@ -26,7 +26,7 @@ module.exports = class PreviewLayergroupController { this.mapStore = mapStore; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } @@ -37,7 +37,7 @@ module.exports = class PreviewLayergroupController { `/static/center/:token/:z/:lat/:lng/:width/:height.:format`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), @@ -59,7 +59,7 @@ module.exports = class PreviewLayergroupController { `/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index ecce4b7f..73a194e6 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -29,7 +29,7 @@ function PreviewTemplateController ( tablesExtentApi, metadataBackend, pgConnection, - authApi, + authBackend, userLimitsApi ) { this.namedMapProviderCache = namedMapProviderCache; @@ -38,7 +38,7 @@ function PreviewTemplateController ( this.tablesExtentApi = tablesExtentApi; this.metadataBackend = metadataBackend; this.pgConnection = pgConnection; - this.authApi = authApi; + this.authBackend = authBackend; this.userLimitsApi = userLimitsApi; } @@ -48,7 +48,7 @@ PreviewTemplateController.prototype.register = function (mapRouter) { mapRouter.get( `/static/named/:template_id/:width/:height.:format`, credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js index 34abcbda..d747d040 100644 --- a/lib/cartodb/api/map/tile-layergroup-controller.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -28,7 +28,7 @@ module.exports = class TileLayergroupController { mapStore, userLimitsApi, layergroupAffectedTablesCache, - authApi, + authBackend, surrogateKeysCache ) { this.tileBackend = tileBackend; @@ -36,7 +36,7 @@ module.exports = class TileLayergroupController { this.mapStore = mapStore; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - this.authApi = authApi; + this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; } @@ -50,7 +50,7 @@ module.exports = class TileLayergroupController { `/:token/:z${number()}/:x${number()}/:y${number()}@:scale_factor${number()}?x.:format`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), cleanUpQueryParams(), @@ -75,7 +75,7 @@ module.exports = class TileLayergroupController { `/:token/:z${number()}/:x${number()}/:y${number()}.:format`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), cleanUpQueryParams(), @@ -100,7 +100,7 @@ module.exports = class TileLayergroupController { `/:token${not('static')}/:layer/:z${number()}/:x${number()}/:y${number()}.(:format)`, layergroupToken(), credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), cleanUpQueryParams(), diff --git a/lib/cartodb/api/middlewares/authorize.js b/lib/cartodb/api/middlewares/authorize.js index a1323fa9..54cab1c4 100644 --- a/lib/cartodb/api/middlewares/authorize.js +++ b/lib/cartodb/api/middlewares/authorize.js @@ -1,6 +1,6 @@ -module.exports = function authorize (authApi) { +module.exports = function authorize (authBackend) { return function authorizeMiddleware (req, res, next) { - authApi.authorize(req, res, (err, authorized) => { + authBackend.authorize(req, res, (err, authorized) => { req.profiler.done('authorize'); if (err) { diff --git a/lib/cartodb/api/template/admin-template-controller.js b/lib/cartodb/api/template/admin-template-controller.js index b5d1b964..e79f6bd9 100644 --- a/lib/cartodb/api/template/admin-template-controller.js +++ b/lib/cartodb/api/template/admin-template-controller.js @@ -4,13 +4,13 @@ const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** - * @param {AuthApi} authApi + * @param {AuthBackend} authBackend * @param {PgConnection} pgConnection * @param {TemplateMaps} templateMaps * @constructor */ -function AdminTemplateController(authApi, templateMaps, userLimitsApi) { - this.authApi = authApi; +function AdminTemplateController(authBackend, templateMaps, userLimitsApi) { + this.authBackend = authBackend; this.templateMaps = templateMaps; this.userLimitsApi = userLimitsApi; } @@ -23,7 +23,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { templateRouter.post( `/`, credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), + authorizedByAPIKey({ authBackend: this.authBackend, action: 'create', label: 'POST TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), createTemplate({ templateMaps: this.templateMaps }) @@ -32,7 +32,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { templateRouter.put( `/:template_id`, credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), + authorizedByAPIKey({ authBackend: this.authBackend, action: 'update', label: 'PUT TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), updateTemplate({ templateMaps: this.templateMaps }) @@ -41,7 +41,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { templateRouter.get( `/:template_id`, credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), + authorizedByAPIKey({ authBackend: this.authBackend, action: 'get', label: 'GET TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), retrieveTemplate({ templateMaps: this.templateMaps }) ); @@ -49,7 +49,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { templateRouter.delete( `/:template_id`, credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), + authorizedByAPIKey({ authBackend: this.authBackend, action: 'delete', label: 'DELETE TEMPLATE' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), destroyTemplate({ templateMaps: this.templateMaps }) ); @@ -57,7 +57,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { templateRouter.get( `/`, credentials(), - authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), + authorizedByAPIKey({ authBackend: this.authBackend, action: 'list', label: 'GET TEMPLATE LIST' }), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), listTemplates({ templateMaps: this.templateMaps }) ); @@ -75,11 +75,11 @@ function checkContentType ({ action, label }) { }; } -function authorizedByAPIKey ({ authApi, action, label }) { +function authorizedByAPIKey ({ authBackend, action, label }) { return function authorizedByAPIKeyMiddleware (req, res, next) { const { user } = res.locals; - authApi.authorizedByAPIKey(user, res, (err, authenticated) => { + authBackend.authorizedByAPIKey(user, res, (err, authenticated) => { if (err) { return next(err); } diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js index 140523b1..f61d5dc7 100644 --- a/lib/cartodb/api/template/named-template-controller.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -21,7 +21,7 @@ const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; /** - * @param {AuthApi} authApi + * @param {AuthBackend} authBackend * @param {PgConnection} pgConnection * @param {TemplateMaps} templateMaps * @param {MapBackend} mapBackend @@ -43,7 +43,7 @@ function NamedMapController ( layergroupAffectedTables, mapConfigAdapter, statsBackend, - authApi, + authBackend, layergroupMetadata ) { this.pgConnection = pgConnection; @@ -55,7 +55,7 @@ function NamedMapController ( this.layergroupAffectedTables = layergroupAffectedTables; this.mapConfigAdapter = mapConfigAdapter; this.statsBackend = statsBackend; - this.authApi = authApi; + this.authBackend = authBackend; this.layergroupMetadata = layergroupMetadata; } @@ -82,7 +82,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () return [ credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), cleanUpQueryParams(['aggregation']), diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js index b4a70715..87f381d1 100644 --- a/lib/cartodb/api/template/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -16,7 +16,7 @@ module.exports = class TemplateRouter { layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, - authApi, + authBackend, layergroupMetadata, namedMapProviderCache, tileBackend, @@ -32,7 +32,7 @@ module.exports = class TemplateRouter { layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, - authApi, + authBackend, layergroupMetadata ); @@ -41,12 +41,12 @@ module.exports = class TemplateRouter { tileBackend, surrogateKeysCache, pgConnection, - authApi, + authBackend, userLimitsApi ); this.adminTemplateController = new AdminTemplateController( - authApi, + authBackend, templateMaps, userLimitsApi ); diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js index 11c0bc19..5e2cb72b 100644 --- a/lib/cartodb/api/template/tile-template-controller.js +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -16,14 +16,14 @@ function TileTemplateController ( tileBackend, surrogateKeysCache, pgConnection, - authApi, + authBackend, userLimitsApi ) { this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.surrogateKeysCache = surrogateKeysCache; this.pgConnection = pgConnection; - this.authApi = authApi; + this.authBackend = authBackend; this.userLimitsApi = userLimitsApi; } @@ -33,7 +33,7 @@ TileTemplateController.prototype.register = function (templateRouter) { templateRouter.get( `/:template_id/:layer/:z/:x/:y.(:format)`, credentials(), - authorize(this.authApi), + authorize(this.authBackend), dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), cleanUpQueryParams(), diff --git a/lib/cartodb/backends/api/auth_api.js b/lib/cartodb/backends/auth.js similarity index 94% rename from lib/cartodb/backends/api/auth_api.js rename to lib/cartodb/backends/auth.js index d27ea3e1..e51084e1 100644 --- a/lib/cartodb/backends/api/auth_api.js +++ b/lib/cartodb/backends/auth.js @@ -7,16 +7,16 @@ var _ = require('underscore'); // AUTH_FALLBACK * @param {MapStore} mapStore * @param {TemplateMaps} templateMaps * @constructor - * @type {AuthApi} + * @type {AuthBackend} */ -function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) { +function AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps) { this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.mapStore = mapStore; this.templateMaps = templateMaps; } -module.exports = AuthApi; +module.exports = AuthBackend; // Check if the user is authorized by a signer // @@ -25,7 +25,7 @@ module.exports = AuthApi; // null if the request is not signed by anyone // or will be a string cartodb username otherwise. // -AuthApi.prototype.authorizedBySigner = function(req, res, callback) { +AuthBackend.prototype.authorizedBySigner = function(req, res, callback) { if ( ! res.locals.token || ! res.locals.signer ) { return callback(null, false); // no signer requested } @@ -60,7 +60,7 @@ function isValidApiKey(apikey) { // @param callback function(err, authorized) // NOTE: authorized is expected to be 0 or 1 (integer) // -AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) { +AuthBackend.prototype.authorizedByAPIKey = function(user, res, callback) { const apikeyToken = res.locals.api_key; const basicAuthUsername = res.locals.basicAuthUsername; @@ -160,7 +160,7 @@ function usernameMatches (basicAuthUsername, requestUsername) { * @param res - standard res object. Contains the auth parameters in locals * @param callback function(err, allowed) is access allowed not? */ -AuthApi.prototype.authorize = function(req, res, callback) { +AuthBackend.prototype.authorize = function(req, res, callback) { var user = res.locals.user; this.authorizedByAPIKey(user, res, (err, isAuthorizedByApikey) => { diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index e9f0a320..02b63dc7 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -4,7 +4,7 @@ var _ = require('underscore'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../../lib/cartodb/backends/pg_connection'); -var AuthApi = require('../../../lib/cartodb/backends/api/auth_api'); +var AuthBackend = require('../../../lib/cartodb/backends/auth'); var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/api/middlewares/clean-up-query-params'); @@ -31,10 +31,10 @@ describe('prepare-context', function() { var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var templateMaps = new TemplateMaps(redisPool); - var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps); + var authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps); cleanUpQueryParams = cleanUpQueryParamsMiddleware(); - authorize = authorizeMiddleware(authApi); + authorize = authorizeMiddleware(authBackend); dbConnSetup = dbConnSetupMiddleware(pgConnection); setCredentials = credentialsMiddleware(); }); From e0d8dc0334605dc44e0323be51505181a494a0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 9 Apr 2018 18:56:01 +0200 Subject: [PATCH 039/185] Rename filter stats api by filter stats backend --- lib/cartodb/api/api-router.js | 6 +++--- .../backends/{api/filter_stats_api.js => filter-stats.js} | 8 ++++---- .../mapconfig/adapter/mapconfig-overviews-adapter.js | 6 +++--- test/integration/mapconfig_overviews_adapter.js | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) rename lib/cartodb/backends/{api/filter_stats_api.js => filter-stats.js} (86%) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index f5b40c9e..15469962 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -16,7 +16,7 @@ const AuthBackend = require('../backends/auth'); const UserLimitsApi = require('../backends/api/user_limits_api'); const OverviewsMetadataApi = require('../backends/api/overviews_metadata_api'); -const FilterStatsApi = require('../backends/api/filter_stats_api'); +const FilterStatsApi = require('../backends/filter-stats'); const TablesExtentApi = require('../backends/api/tables_extent_api'); const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); @@ -124,7 +124,7 @@ module.exports = class ApiRouter { const pgQueryRunner = new PgQueryRunner(pgConnection); const overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); - const filterStatsApi = new FilterStatsApi(pgQueryRunner); + const filterStatsBackend = new FilterStatsApi(pgQueryRunner); const tablesExtentApi = new TablesExtentApi(pgQueryRunner); const mapConfigAdapter = new MapConfigAdapter( @@ -134,7 +134,7 @@ module.exports = class ApiRouter { new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), new AggregationMapConfigAdapter(pgConnection), - new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), + new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsBackend), new TurboCartoAdapter() ); diff --git a/lib/cartodb/backends/api/filter_stats_api.js b/lib/cartodb/backends/filter-stats.js similarity index 86% rename from lib/cartodb/backends/api/filter_stats_api.js rename to lib/cartodb/backends/filter-stats.js index 72fd970f..ba427405 100644 --- a/lib/cartodb/backends/api/filter_stats_api.js +++ b/lib/cartodb/backends/filter-stats.js @@ -1,12 +1,12 @@ var _ = require('underscore'); var step = require('step'); -var AnalysisFilter = require('../../models/filter/analysis'); +var AnalysisFilter = require('../models/filter/analysis'); -function FilterStatsApi(pgQueryRunner) { +function FilterStatsBackends(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = FilterStatsApi; +module.exports = FilterStatsBackends; function getEstimatedRows(pgQueryRunner, username, query, callback) { pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) { @@ -23,7 +23,7 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) { }); } -FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { +FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { var stats = {}; var self = this; step( diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js index 72c29593..a3133da2 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js @@ -2,9 +2,9 @@ var step = require('step'); var queue = require('queue-async'); var _ = require('underscore'); -function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) { +function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsBackend) { this.overviewsMetadataApi = overviewsMetadataApi; - this.filterStatsApi = filterStatsApi; + this.filterStatsBackend = filterStatsBackend; } module.exports = MapConfigOverviewsAdapter; @@ -53,7 +53,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf function collectStatsData(err, filters, unfiltered_query) { var next_step = this; if ( filters ) { - self.filterStatsApi.getFilterStats( + self.filterStatsBackend.getFilterStats( user, unfiltered_query, filters, function(err, stats) { diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index 79196e3b..288c2560 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -6,7 +6,7 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); var OverviewsMetadataApi = require('../../lib/cartodb/backends/api/overviews_metadata_api'); -var FilterStatsApi = require('../../lib/cartodb/backends/api/filter_stats_api'); +var FilterStatsBackend = require('../../lib/cartodb/backends/filter-stats'); var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter'); var redisPool = new RedisPool(global.environment.redis); @@ -14,9 +14,9 @@ var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); -var filterStatsApi = new FilterStatsApi(pgQueryRunner); +var filterStatsBackend = new FilterStatsBackend(pgQueryRunner); -var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); +var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsBackend); describe('MapConfigOverviewsAdapter', function() { From ba36a4722814430ed17a1ce45be57e31ccfb2003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 9 Apr 2018 19:47:24 +0200 Subject: [PATCH 040/185] Rename overviews metadata api by overviews metadata backend --- lib/cartodb/api/api-router.js | 6 +++--- ...erviews_metadata_api.js => overviews-metadata.js} | 8 ++++---- .../mapconfig/adapter/mapconfig-overviews-adapter.js | 6 +++--- test/integration/mapconfig_overviews_adapter.js | 6 +++--- test/integration/overviews-metadata-api.js | 12 ++++++------ 5 files changed, 19 insertions(+), 19 deletions(-) rename lib/cartodb/backends/{api/overviews_metadata_api.js => overviews-metadata.js} (82%) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 15469962..3f0f7ac0 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -15,7 +15,7 @@ const StatsBackend = require('../backends/stats'); const AuthBackend = require('../backends/auth'); const UserLimitsApi = require('../backends/api/user_limits_api'); -const OverviewsMetadataApi = require('../backends/api/overviews_metadata_api'); +const OverviewsMetadataBackend = require('../backends/overviews-metadata'); const FilterStatsApi = require('../backends/filter-stats'); const TablesExtentApi = require('../backends/api/tables_extent_api'); @@ -122,7 +122,7 @@ module.exports = class ApiRouter { } const pgQueryRunner = new PgQueryRunner(pgConnection); - const overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + const overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); const filterStatsBackend = new FilterStatsApi(pgQueryRunner); const tablesExtentApi = new TablesExtentApi(pgQueryRunner); @@ -134,7 +134,7 @@ module.exports = class ApiRouter { new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), new AggregationMapConfigAdapter(pgConnection), - new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsBackend), + new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend), new TurboCartoAdapter() ); diff --git a/lib/cartodb/backends/api/overviews_metadata_api.js b/lib/cartodb/backends/overviews-metadata.js similarity index 82% rename from lib/cartodb/backends/api/overviews_metadata_api.js rename to lib/cartodb/backends/overviews-metadata.js index dda51d7e..31045534 100644 --- a/lib/cartodb/backends/api/overviews_metadata_api.js +++ b/lib/cartodb/backends/overviews-metadata.js @@ -1,10 +1,10 @@ -var SubstitutionTokens = require('../../utils/substitution-tokens'); +var SubstitutionTokens = require('../utils/substitution-tokens'); -function OverviewsMetadataApi(pgQueryRunner) { +function OverviewsMetadataBackend(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = OverviewsMetadataApi; +module.exports = OverviewsMetadataBackend; function prepareSql(sql) { return sql && SubstitutionTokens.replace(sql, { @@ -15,7 +15,7 @@ function prepareSql(sql) { }); } -OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) { +OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sql, callback) { // FIXME: Currently using internal function _cdb_schema_name // CDB_Overviews should provide the schema information directly. var query = 'SELECT *, _cdb_schema_name(base_table)' + diff --git a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js index a3133da2..a104c592 100644 --- a/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js @@ -2,8 +2,8 @@ var step = require('step'); var queue = require('queue-async'); var _ = require('underscore'); -function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsBackend) { - this.overviewsMetadataApi = overviewsMetadataApi; +function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) { + this.overviewsMetadataBackend = overviewsMetadataBackend; this.filterStatsBackend = filterStatsBackend; } @@ -25,7 +25,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) { return done(null, layer); } - self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){ + self.overviewsMetadataBackend.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){ if (err) { done(err, layer); } else { diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index 288c2560..d5414a7e 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -5,7 +5,7 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/backends/api/overviews_metadata_api'); +var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata'); var FilterStatsBackend = require('../../lib/cartodb/backends/filter-stats'); var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter'); @@ -13,10 +13,10 @@ var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); -var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); +var overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); var filterStatsBackend = new FilterStatsBackend(pgQueryRunner); -var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsBackend); +var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend); describe('MapConfigOverviewsAdapter', function() { diff --git a/test/integration/overviews-metadata-api.js b/test/integration/overviews-metadata-api.js index e51c2129..ce82a1d9 100644 --- a/test/integration/overviews-metadata-api.js +++ b/test/integration/overviews-metadata-api.js @@ -7,24 +7,24 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var OverviewsMetadataApi = require('../../lib/cartodb/backends/api/overviews_metadata_api'); +var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata'); -describe('OverviewsMetadataApi', function() { +describe('OverviewsMetadataBackend', function() { - var overviewsMetadataApi; + var overviewsMetadataBackend; before(function() { var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); - overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); }); it('should return an empty relation for tables that have no overviews', function(done) { var query = 'select * from test_table'; - overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) { assert.ok(!err, err); assert.deepEqual(result, {}); @@ -35,7 +35,7 @@ describe('OverviewsMetadataApi', function() { it('should return overviews metadata', function(done) { var query = 'select * from test_table_overviews'; - overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) { assert.ok(!err, err); assert.deepEqual(result, { From 8519d2724b188ba08ed210fc1b887ec38492ee7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 10 Apr 2018 09:40:09 +0200 Subject: [PATCH 041/185] Rename tables extent api by tables extent backend --- lib/cartodb/api/api-router.js | 6 +++--- lib/cartodb/api/map/map-router.js | 4 ++-- lib/cartodb/api/map/preview-template-controller.js | 10 +++++----- .../{api/tables_extent_api.js => tables-extent.js} | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) rename lib/cartodb/backends/{api/tables_extent_api.js => tables-extent.js} (90%) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 3f0f7ac0..5a08eb6d 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -17,7 +17,7 @@ const AuthBackend = require('../backends/auth'); const UserLimitsApi = require('../backends/api/user_limits_api'); const OverviewsMetadataBackend = require('../backends/overviews-metadata'); const FilterStatsApi = require('../backends/filter-stats'); -const TablesExtentApi = require('../backends/api/tables_extent_api'); +const TablesExtentBackend = require('../backends/tables-extent'); const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); const SurrogateKeysCache = require('../cache/surrogate_keys_cache'); @@ -125,7 +125,7 @@ module.exports = class ApiRouter { const overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner); const filterStatsBackend = new FilterStatsApi(pgQueryRunner); - const tablesExtentApi = new TablesExtentApi(pgQueryRunner); + const tablesExtentBackend = new TablesExtentBackend(pgQueryRunner); const mapConfigAdapter = new MapConfigAdapter( new MapConfigNamedLayersAdapter(templateMaps, pgConnection), @@ -173,7 +173,7 @@ module.exports = class ApiRouter { statsBackend, layergroupMetadata, namedMapProviderCache, - tablesExtentApi + tablesExtentBackend }; this.mapRouter = new MapRouter({ collaborators }); diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index 2e2a3f4a..78b87d1e 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -30,7 +30,7 @@ module.exports = class MapRouter { statsBackend, layergroupMetadata, namedMapProviderCache, - tablesExtentApi + tablesExtentBackend } = collaborators; this.analysisLayergroupController = new AnalysisLayergroupController( @@ -101,7 +101,7 @@ module.exports = class MapRouter { namedMapProviderCache, previewBackend, surrogateKeysCache, - tablesExtentApi, + tablesExtentBackend, metadataBackend, pgConnection, authBackend, diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index 73a194e6..acf58a91 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -26,7 +26,7 @@ function PreviewTemplateController ( namedMapProviderCache, previewBackend, surrogateKeysCache, - tablesExtentApi, + tablesExtentBackend, metadataBackend, pgConnection, authBackend, @@ -35,7 +35,7 @@ function PreviewTemplateController ( this.namedMapProviderCache = namedMapProviderCache; this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; - this.tablesExtentApi = tablesExtentApi; + this.tablesExtentBackend = tablesExtentBackend; this.metadataBackend = metadataBackend; this.pgConnection = pgConnection; this.authBackend = authBackend; @@ -61,7 +61,7 @@ PreviewTemplateController.prototype.register = function (mapRouter) { namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP' }), - getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), + getStaticImageOptions({ tablesExtentBackend: this.tablesExtentBackend }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), setContentTypeHeader(), incrementMapViews({ metadataBackend: this.metadataBackend }), @@ -138,7 +138,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) }; } -function getStaticImageOptions ({ tablesExtentApi }) { +function getStaticImageOptions ({ tablesExtentBackend }) { return function getStaticImageOptionsMiddleware(req, res, next) { const { user, mapConfigProvider, template } = res.locals; const { zoom, lon, lat, bbox } = req.query; @@ -164,7 +164,7 @@ function getStaticImageOptions ({ tablesExtentApi }) { return next(); } - tablesExtentApi.getBounds(user, tables, (err, bounds) => { + tablesExtentBackend.getBounds(user, tables, (err, bounds) => { if (err) { return next(); } diff --git a/lib/cartodb/backends/api/tables_extent_api.js b/lib/cartodb/backends/tables-extent.js similarity index 90% rename from lib/cartodb/backends/api/tables_extent_api.js rename to lib/cartodb/backends/tables-extent.js index 7b534a3f..5e78a6b1 100644 --- a/lib/cartodb/backends/api/tables_extent_api.js +++ b/lib/cartodb/backends/tables-extent.js @@ -1,8 +1,8 @@ -function TablesExtentApi(pgQueryRunner) { +function TablesExtentBackend(pgQueryRunner) { this.pgQueryRunner = pgQueryRunner; } -module.exports = TablesExtentApi; +module.exports = TablesExtentBackend; /** * Given a username and a list of tables it will return the estimated extent in SRID 4326 for all the tables based on @@ -13,7 +13,7 @@ module.exports = TablesExtentApi; * `table_name` format as valid input * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north` */ -TablesExtentApi.prototype.getBounds = function (username, tables, callback) { +TablesExtentBackend.prototype.getBounds = function (username, tables, callback) { var estimatedExtentSQLs = tables.map(function(table) { return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')"; }); From d1a4057a8dfbbf109a1cdbbc56d9c8e3ac90f5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 10 Apr 2018 10:16:07 +0200 Subject: [PATCH 042/185] Rename user limits api by user limits backend --- lib/cartodb/api/api-router.js | 8 ++++---- .../api/map/analyses-catalog-controller.js | 6 +++--- .../api/map/analysis-layergroup-controller.js | 6 +++--- .../api/map/anonymous-map-controller.js | 14 ++++++------- .../map/attributes-layergroup-controller.js | 8 ++++---- .../api/map/dataview-layergroup-controller.js | 20 +++++++++---------- lib/cartodb/api/map/map-router.js | 18 ++++++++--------- .../api/map/preview-layergroup-controller.js | 12 +++++------ .../api/map/preview-template-controller.js | 6 +++--- .../api/map/tile-layergroup-controller.js | 16 +++++++-------- .../map-store-map-config-provider.js | 4 ++-- lib/cartodb/api/middlewares/rate-limit.js | 4 ++-- .../api/template/admin-template-controller.js | 14 ++++++------- .../api/template/named-template-controller.js | 20 +++++++++---------- lib/cartodb/api/template/template-router.js | 8 ++++---- .../api/template/tile-template-controller.js | 6 +++--- .../user_limits_api.js => user-limits.js} | 14 ++++++------- lib/cartodb/cache/named_map_provider_cache.js | 6 +++--- .../provider/create-layergroup-provider.js | 8 ++++---- .../mapconfig/provider/map-store-provider.js | 8 ++++---- .../mapconfig/provider/named-map-provider.js | 6 +++--- test/acceptance/rate-limit.test.js | 4 ++-- 22 files changed, 108 insertions(+), 108 deletions(-) rename lib/cartodb/backends/{api/user_limits_api.js => user-limits.js} (81%) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 5a08eb6d..7108bbbf 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -14,7 +14,7 @@ const PgQueryRunner = require('../backends/pg_query_runner'); const StatsBackend = require('../backends/stats'); const AuthBackend = require('../backends/auth'); -const UserLimitsApi = require('../backends/api/user_limits_api'); +const UserLimitsBackend = require('../backends/user-limits'); const OverviewsMetadataBackend = require('../backends/overviews-metadata'); const FilterStatsApi = require('../backends/filter-stats'); const TablesExtentBackend = require('../backends/tables-extent'); @@ -106,7 +106,7 @@ module.exports = class ApiRouter { const dataviewBackend = new DataviewBackend(analysisBackend); const statsBackend = new StatsBackend(); - const userLimitsApi = new UserLimitsApi(metadataBackend, { + const userLimitsBackend = new UserLimitsBackend(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, render: serverOptions.renderer.mapnik.limits.render || 0, @@ -145,7 +145,7 @@ module.exports = class ApiRouter { templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, layergroupAffectedTablesCache ); @@ -162,7 +162,7 @@ module.exports = class ApiRouter { tileBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache, diff --git a/lib/cartodb/api/map/analyses-catalog-controller.js b/lib/cartodb/api/map/analyses-catalog-controller.js index 855666dd..91defc21 100644 --- a/lib/cartodb/api/map/analyses-catalog-controller.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -8,10 +8,10 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const cacheControlHeader = require('../middlewares/cache-control-header'); const dbParamsFromResLocals = require('../../utils/database-params'); -function AnalysesController(pgConnection, authBackend, userLimitsApi) { +function AnalysesController(pgConnection, authBackend, userLimitsBackend) { this.pgConnection = pgConnection; this.authBackend = authBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; } module.exports = AnalysesController; @@ -22,7 +22,7 @@ AnalysesController.prototype.register = function (mapRouter) { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), cleanUpQueryParams(), createPGClient(), getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js index e0cf3222..5d89803b 100644 --- a/lib/cartodb/api/map/analysis-layergroup-controller.js +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -12,7 +12,7 @@ module.exports = class AnalysisLayergroupController { analysisStatusBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -20,7 +20,7 @@ module.exports = class AnalysisLayergroupController { this.analysisStatusBackend = analysisStatusBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; @@ -33,7 +33,7 @@ module.exports = class AnalysisLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), cleanUpQueryParams(), analysisNodeStatus(this.analysisStatusBackend) ); diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js index a20b1407..964ffacd 100644 --- a/lib/cartodb/api/map/anonymous-map-controller.js +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -29,7 +29,7 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {MapBackend} mapBackend * @param metadataBackend * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi + * @param {UserLimitsBackend} userLimitsBackend * @param {LayergroupAffectedTables} layergroupAffectedTables * @param {MapConfigAdapter} mapConfigAdapter * @param {StatsBackend} statsBackend @@ -41,7 +41,7 @@ function AnonymousMapController ( mapBackend, metadataBackend, surrogateKeysCache, - userLimitsApi, + userLimitsBackend, layergroupAffectedTables, mapConfigAdapter, statsBackend, @@ -53,7 +53,7 @@ function AnonymousMapController ( this.mapBackend = mapBackend; this.metadataBackend = metadataBackend; this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTables = layergroupAffectedTables; this.mapConfigAdapter = mapConfigAdapter; this.statsBackend = statsBackend; @@ -80,7 +80,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), cleanUpQueryParams(['aggregation']), initProfiler(isTemplateInstantiation), checkJsonContentType(), @@ -88,7 +88,7 @@ AnonymousMapController.prototype.composeCreateMapMiddleware = function () { prepareAdapterMapConfig(this.mapConfigAdapter), createLayergroup ( this.mapBackend, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTables ), @@ -166,7 +166,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) { }; } -function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { +function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) { return function createLayergroupMiddleware (req, res, next) { const requestMapConfig = req.body; @@ -185,7 +185,7 @@ function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTabl const mapConfigProvider = new CreateLayergroupMapConfigProvider( mapConfig, user, - userLimitsApi, + userLimitsBackend, pgConnection, affectedTablesCache, params diff --git a/lib/cartodb/api/map/attributes-layergroup-controller.js b/lib/cartodb/api/map/attributes-layergroup-controller.js index 17af3df7..ef9614fc 100644 --- a/lib/cartodb/api/map/attributes-layergroup-controller.js +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -16,7 +16,7 @@ module.exports = class AttributesLayergroupController { attributesBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -24,7 +24,7 @@ module.exports = class AttributesLayergroupController { this.attributesBackend = attributesBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; @@ -37,11 +37,11 @@ module.exports = class AttributesLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), cleanUpQueryParams(), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js index 84b4c905..0b6f8cc9 100644 --- a/lib/cartodb/api/map/dataview-layergroup-controller.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -31,7 +31,7 @@ module.exports = class DataviewLayergroupController { dataviewBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -39,7 +39,7 @@ module.exports = class DataviewLayergroupController { this.dataviewBackend = dataviewBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; @@ -55,11 +55,11 @@ module.exports = class DataviewLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), @@ -76,11 +76,11 @@ module.exports = class DataviewLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), @@ -97,11 +97,11 @@ module.exports = class DataviewLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), @@ -118,11 +118,11 @@ module.exports = class DataviewLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index 78b87d1e..a6e91c29 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -19,7 +19,7 @@ module.exports = class MapRouter { tileBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache, @@ -37,7 +37,7 @@ module.exports = class MapRouter { analysisStatusBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -47,7 +47,7 @@ module.exports = class MapRouter { attributesBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -57,7 +57,7 @@ module.exports = class MapRouter { dataviewBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -67,7 +67,7 @@ module.exports = class MapRouter { previewBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -77,7 +77,7 @@ module.exports = class MapRouter { tileBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -89,7 +89,7 @@ module.exports = class MapRouter { mapBackend, metadataBackend, surrogateKeysCache, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, @@ -105,13 +105,13 @@ module.exports = class MapRouter { metadataBackend, pgConnection, authBackend, - userLimitsApi + userLimitsBackend ); this.analysesController = new AnalysesCatalogController( pgConnection, authBackend, - userLimitsApi + userLimitsBackend ); } diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js index 484b831c..dcacc4f5 100644 --- a/lib/cartodb/api/map/preview-layergroup-controller.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -16,7 +16,7 @@ module.exports = class PreviewLayergroupController { previewBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -24,7 +24,7 @@ module.exports = class PreviewLayergroupController { this.previewBackend = previewBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; @@ -39,11 +39,11 @@ module.exports = class PreviewLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache, forcedFormat @@ -61,11 +61,11 @@ module.exports = class PreviewLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache, forcedFormat diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index acf58a91..91a3f5b6 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -30,7 +30,7 @@ function PreviewTemplateController ( metadataBackend, pgConnection, authBackend, - userLimitsApi + userLimitsBackend ) { this.namedMapProviderCache = namedMapProviderCache; this.previewBackend = previewBackend; @@ -39,7 +39,7 @@ function PreviewTemplateController ( this.metadataBackend = metadataBackend; this.pgConnection = pgConnection; this.authBackend = authBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; } module.exports = PreviewTemplateController; @@ -50,7 +50,7 @@ PreviewTemplateController.prototype.register = function (mapRouter) { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), namedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js index d747d040..40d85b7c 100644 --- a/lib/cartodb/api/map/tile-layergroup-controller.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -26,7 +26,7 @@ module.exports = class TileLayergroupController { tileBackend, pgConnection, mapStore, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, authBackend, surrogateKeysCache @@ -34,7 +34,7 @@ module.exports = class TileLayergroupController { this.tileBackend = tileBackend; this.pgConnection = pgConnection; this.mapStore = mapStore; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.authBackend = authBackend; this.surrogateKeysCache = surrogateKeysCache; @@ -52,11 +52,11 @@ module.exports = class TileLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), cleanUpQueryParams(), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), @@ -77,11 +77,11 @@ module.exports = class TileLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), cleanUpQueryParams(), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), @@ -102,11 +102,11 @@ module.exports = class TileLayergroupController { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), cleanUpQueryParams(), createMapStoreMapConfigProvider( this.mapStore, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTablesCache ), diff --git a/lib/cartodb/api/middlewares/map-store-map-config-provider.js b/lib/cartodb/api/middlewares/map-store-map-config-provider.js index f84dd3fc..58cdb79e 100644 --- a/lib/cartodb/api/middlewares/map-store-map-config-provider.js +++ b/lib/cartodb/api/middlewares/map-store-map-config-provider.js @@ -2,7 +2,7 @@ const MapStoreMapConfigProvider = require('../../models/mapconfig/provider/map-s module.exports = function createMapStoreMapConfigProvider ( mapStore, - userLimitsApi, + userLimitsBackend, pgConnection, affectedTablesCache, forcedFormat = null @@ -26,7 +26,7 @@ module.exports = function createMapStoreMapConfigProvider ( res.locals.mapConfigProvider = new MapStoreMapConfigProvider( mapStore, user, - userLimitsApi, + userLimitsBackend, pgConnection, affectedTablesCache, params diff --git a/lib/cartodb/api/middlewares/rate-limit.js b/lib/cartodb/api/middlewares/rate-limit.js index b3c6d7fe..55062d76 100644 --- a/lib/cartodb/api/middlewares/rate-limit.js +++ b/lib/cartodb/api/middlewares/rate-limit.js @@ -19,13 +19,13 @@ const RATE_LIMIT_ENDPOINTS_GROUPS = { NAMED_TILES: 'named_tiles' }; -function rateLimit(userLimitsApi, endpointGroup = null) { +function rateLimit(userLimitsBackend, endpointGroup = null) { if (!isRateLimitEnabled(endpointGroup)) { return function rateLimitDisabledMiddleware(req, res, next) { next(); }; } return function rateLimitMiddleware(req, res, next) { - userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { + userLimitsBackend.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) { if (err) { return next(err); } diff --git a/lib/cartodb/api/template/admin-template-controller.js b/lib/cartodb/api/template/admin-template-controller.js index e79f6bd9..b4d1b65f 100644 --- a/lib/cartodb/api/template/admin-template-controller.js +++ b/lib/cartodb/api/template/admin-template-controller.js @@ -9,10 +9,10 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {TemplateMaps} templateMaps * @constructor */ -function AdminTemplateController(authBackend, templateMaps, userLimitsApi) { +function AdminTemplateController(authBackend, templateMaps, userLimitsBackend) { this.authBackend = authBackend; this.templateMaps = templateMaps; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; } module.exports = AdminTemplateController; @@ -24,7 +24,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { `/`, credentials(), authorizedByAPIKey({ authBackend: this.authBackend, action: 'create', label: 'POST TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), createTemplate({ templateMaps: this.templateMaps }) ); @@ -33,7 +33,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { `/:template_id`, credentials(), authorizedByAPIKey({ authBackend: this.authBackend, action: 'update', label: 'PUT TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), updateTemplate({ templateMaps: this.templateMaps }) ); @@ -42,7 +42,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { `/:template_id`, credentials(), authorizedByAPIKey({ authBackend: this.authBackend, action: 'get', label: 'GET TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), retrieveTemplate({ templateMaps: this.templateMaps }) ); @@ -50,7 +50,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { `/:template_id`, credentials(), authorizedByAPIKey({ authBackend: this.authBackend, action: 'delete', label: 'DELETE TEMPLATE' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), destroyTemplate({ templateMaps: this.templateMaps }) ); @@ -58,7 +58,7 @@ AdminTemplateController.prototype.register = function (templateRouter) { `/`, credentials(), authorizedByAPIKey({ authBackend: this.authBackend, action: 'list', label: 'GET TEMPLATE LIST' }), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), listTemplates({ templateMaps: this.templateMaps }) ); }; diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js index f61d5dc7..eed39b34 100644 --- a/lib/cartodb/api/template/named-template-controller.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -27,7 +27,7 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {MapBackend} mapBackend * @param metadataBackend * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi + * @param {UserLimitsBackend} userLimitsBackend * @param {LayergroupAffectedTables} layergroupAffectedTables * @param {MapConfigAdapter} mapConfigAdapter * @param {StatsBackend} statsBackend @@ -39,7 +39,7 @@ function NamedMapController ( mapBackend, metadataBackend, surrogateKeysCache, - userLimitsApi, + userLimitsBackend, layergroupAffectedTables, mapConfigAdapter, statsBackend, @@ -51,7 +51,7 @@ function NamedMapController ( this.mapBackend = mapBackend; this.metadataBackend = metadataBackend; this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.layergroupAffectedTables = layergroupAffectedTables; this.mapConfigAdapter = mapConfigAdapter; this.statsBackend = statsBackend; @@ -84,7 +84,7 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), cleanUpQueryParams(['aggregation']), initProfiler(isTemplateInstantiation), checkJsonContentType(), @@ -93,13 +93,13 @@ NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () this.templateMaps, this.pgConnection, this.metadataBackend, - this.userLimitsApi, + this.userLimitsBackend, this.mapConfigAdapter, this.layergroupAffectedTables ), instantiateLayergroup( this.mapBackend, - this.userLimitsApi, + this.userLimitsBackend, this.pgConnection, this.layergroupAffectedTables ), @@ -145,7 +145,7 @@ function getTemplate ( templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, affectedTablesCache ) { @@ -161,7 +161,7 @@ function getTemplate ( templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, affectedTablesCache, user, @@ -186,13 +186,13 @@ function getTemplate ( }; } -function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { +function instantiateLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) { return function instantiateLayergroupMiddleware (req, res, next) { const { user, mapConfig, rendererParams } = res.locals; const mapConfigProvider = new CreateLayergroupMapConfigProvider( mapConfig, user, - userLimitsApi, + userLimitsBackend, pgConnection, affectedTablesCache, rendererParams diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js index 87f381d1..643e83c3 100644 --- a/lib/cartodb/api/template/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -12,7 +12,7 @@ module.exports = class TemplateRouter { mapBackend, metadataBackend, surrogateKeysCache, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, @@ -28,7 +28,7 @@ module.exports = class TemplateRouter { mapBackend, metadataBackend, surrogateKeysCache, - userLimitsApi, + userLimitsBackend, layergroupAffectedTablesCache, mapConfigAdapter, statsBackend, @@ -42,13 +42,13 @@ module.exports = class TemplateRouter { surrogateKeysCache, pgConnection, authBackend, - userLimitsApi + userLimitsBackend ); this.adminTemplateController = new AdminTemplateController( authBackend, templateMaps, - userLimitsApi + userLimitsBackend ); } diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js index 5e2cb72b..c99a0aea 100644 --- a/lib/cartodb/api/template/tile-template-controller.js +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -17,14 +17,14 @@ function TileTemplateController ( surrogateKeysCache, pgConnection, authBackend, - userLimitsApi + userLimitsBackend ) { this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.surrogateKeysCache = surrogateKeysCache; this.pgConnection = pgConnection; this.authBackend = authBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; } module.exports = TileTemplateController; @@ -35,7 +35,7 @@ TileTemplateController.prototype.register = function (templateRouter) { credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), cleanUpQueryParams(), namedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, diff --git a/lib/cartodb/backends/api/user_limits_api.js b/lib/cartodb/backends/user-limits.js similarity index 81% rename from lib/cartodb/backends/api/user_limits_api.js rename to lib/cartodb/backends/user-limits.js index caf9f151..519f1cd1 100644 --- a/lib/cartodb/backends/api/user_limits_api.js +++ b/lib/cartodb/backends/user-limits.js @@ -5,9 +5,9 @@ var step = require('step'); * @param metadataBackend * @param options * @constructor - * @type {UserLimitsApi} + * @type {UserLimitsBackend} */ -function UserLimitsApi(metadataBackend, options) { +function UserLimitsBackend(metadataBackend, options) { this.metadataBackend = metadataBackend; this.options = options || {}; this.options.limits = this.options.limits || {}; @@ -15,9 +15,9 @@ function UserLimitsApi(metadataBackend, options) { this.preprareRateLimit(); } -module.exports = UserLimitsApi; +module.exports = UserLimitsBackend; -UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) { +UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callback) { var self = this; var limits = { @@ -40,7 +40,7 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) }); }; -UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { +UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) { var self = this; step( @@ -80,12 +80,12 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call ); }; -UserLimitsApi.prototype.preprareRateLimit = function () { +UserLimitsBackend.prototype.preprareRateLimit = function () { if (this.options.limits.rateLimitsEnabled) { this.metadataBackend.loadRateLimitsScript(); } }; -UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) { +UserLimitsBackend.prototype.getRateLimit = function (user, endpointGroup, callback) { this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback); }; diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index e0850822..b64decb1 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -10,14 +10,14 @@ function NamedMapProviderCache( templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, affectedTablesCache ) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.mapConfigAdapter = mapConfigAdapter; this.affectedTablesCache = affectedTablesCache; @@ -36,7 +36,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.templateMaps, this.pgConnection, this.metadataBackend, - this.userLimitsApi, + this.userLimitsBackend, this.mapConfigAdapter, this.affectedTablesCache, user, diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index b3b3f191..fe92bb47 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -7,16 +7,16 @@ const QueryTables = require('cartodb-query-tables'); /** * @param {MapConfig} mapConfig * @param {String} user - * @param {UserLimitsApi} userLimitsApi + * @param {UserLimitsBackend} userLimitsBackend * @param {Object} params * @constructor * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) { +function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsBackend, pgConnection, affectedTablesCache, params) { this.mapConfig = mapConfig; this.user = user; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.pgConnection = pgConnection; this.affectedTablesCache = affectedTablesCache; this.params = params; @@ -36,7 +36,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); + self.userLimitsBackend.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index f89a79ce..d57c8d7b 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -7,15 +7,15 @@ const QueryTables = require('cartodb-query-tables'); /** * @param {MapStore} mapStore * @param {String} user - * @param {UserLimitsApi} userLimitsApi + * @param {UserLimitsBackend} userLimitsBackend * @param {Object} params * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsBackend, pgConnection, affectedTablesCache, params) { this.mapStore = mapStore; this.user = user; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.pgConnection = pgConnection; this.affectedTablesCache = affectedTablesCache; this.token = params.token; @@ -38,7 +38,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { step( function prepareContextLimits() { - self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); + self.userLimitsBackend.getRenderLimits(self.user, self.params.api_key, this); }, function handleRenderLimits(err, renderLimits) { assert.ifError(err); diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 50612065..5e931679 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -15,7 +15,7 @@ function NamedMapMapConfigProvider( templateMaps, pgConnection, metadataBackend, - userLimitsApi, + userLimitsBackend, mapConfigAdapter, affectedTablesCache, owner, @@ -27,7 +27,7 @@ function NamedMapMapConfigProvider( this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; - this.userLimitsApi = userLimitsApi; + this.userLimitsBackend = userLimitsBackend; this.mapConfigAdapter = mapConfigAdapter; this.owner = owner; @@ -125,7 +125,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { function prepareContextLimits(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; - self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this); + self.userLimitsBackend.getRenderLimits(self.owner, self.params.api_key, this); }, function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js index 5b7cdbc0..925d5028 100644 --- a/test/acceptance/rate-limit.test.js +++ b/test/acceptance/rate-limit.test.js @@ -5,7 +5,7 @@ const redis = require('redis'); const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const TestClient = require('../support/test-client'); -const UserLimitsApi = require('../../lib/cartodb/backends/api/user_limits_api'); +const UserLimitsBackend = require('../../lib/cartodb/backends/user-limits'); const rateLimitMiddleware = require('../../lib/cartodb/api/middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; @@ -218,7 +218,7 @@ describe('rate limit middleware', function () { const redisPool = new RedisPool(global.environment.redis); const metadataBackend = cartodbRedis({ pool: redisPool }); - userLimitsApi = new UserLimitsApi(metadataBackend, { + userLimitsApi = new UserLimitsBackend(metadataBackend, { limits: { rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled } From 1bbd84b37a514581d7a7dfa2b817e4250b71ee83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 10 Apr 2018 15:56:52 +0200 Subject: [PATCH 043/185] Please jshint --- .../mapconfig/provider/create-layergroup-provider.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index fe92bb47..7e48a9bb 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -13,7 +13,14 @@ const QueryTables = require('cartodb-query-tables'); * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsBackend, pgConnection, affectedTablesCache, params) { +function CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsBackend, + pgConnection, + affectedTablesCache, + params +) { this.mapConfig = mapConfig; this.user = user; this.userLimitsBackend = userLimitsBackend; From c0943a7c580d9b355f7cec4b4b67ca162f03549c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 10 Apr 2018 20:26:36 +0200 Subject: [PATCH 044/185] Use config to define base path for express routers --- lib/cartodb/api/api-router.js | 11 +++-------- lib/cartodb/api/map/map-router.js | 9 ++++----- lib/cartodb/api/template/template-router.js | 9 ++++----- lib/cartodb/server_options.js | 13 +++++++++++++ .../ported/support/ported_server_options.js | 14 ++++++++++++++ 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 7108bbbf..d75d23a5 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -176,8 +176,8 @@ module.exports = class ApiRouter { tablesExtentBackend }; - this.mapRouter = new MapRouter({ collaborators }); - this.templateRouter = new TemplateRouter({ collaborators }); + this.mapRouter = new MapRouter({ collaborators, serverOptions }); + this.templateRouter = new TemplateRouter({ collaborators, serverOptions }); } register (app) { @@ -206,12 +206,7 @@ module.exports = class ApiRouter { apiRouter.use(syntaxError()); apiRouter.use(errorMiddleware()); - const paths = [ - '/api/v1', - '/user/:user/api/v1', - '/tiles', // Deprecated - '/database/:dbname' // Deprecated: used in ported test from windshaft - ]; + const paths = this.serverOptions.api_base_paths; app.use(`(?:${paths.join('|')})`, apiRouter); } diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index a6e91c29..e8a3c2fb 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -10,7 +10,9 @@ const PreviewTemplateController = require('./preview-template-controller'); const AnalysesCatalogController = require('./analyses-catalog-controller'); module.exports = class MapRouter { - constructor ({ collaborators }) { + constructor ({ collaborators, serverOptions }) { + this.serverOptions = serverOptions; + const { analysisStatusBackend, attributesBackend, @@ -127,10 +129,7 @@ module.exports = class MapRouter { this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); - const paths = [ - '/map', - '/layergroup' - ]; + const paths = this.serverOptions.map_base_paths; apiRouter.use(`(?:${paths.join('|')})`, mapRouter); } diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js index 643e83c3..9b759f0a 100644 --- a/lib/cartodb/api/template/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -5,7 +5,9 @@ const AdminTemplateController = require('./admin-template-controller'); const TileTemplateController = require('./tile-template-controller'); module.exports = class TemplateRouter { - constructor ({ collaborators }) { + constructor ({ collaborators, serverOptions }) { + this.serverOptions = serverOptions; + const { pgConnection, templateMaps, @@ -59,10 +61,7 @@ module.exports = class TemplateRouter { this.tileTemplateController.register(templateRouter); this.adminTemplateController.register(templateRouter); - const paths = [ - '/map/named', - '/template' - ]; + const paths = this.serverOptions.template_base_paths; apiRouter.use(`(?:${paths.join('|')})`, templateRouter); } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index adc03003..c73a6bca 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -67,6 +67,19 @@ module.exports = { // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)', + api_base_paths: global.environment.api_base_paths || [ + '/api/v1', + '/user/:user/api/v1' + ], + + map_base_paths: global.environment.map_base_paths || [ + '/map' + ], + + template_base_paths: global.environment.template_base_paths || [ + '/map/named' + ], + grainstore: { map: { // TODO: allow to specify in configuration diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 1ce262af..931fa584 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -11,6 +11,20 @@ module.exports = _.extend({}, serverOptions, { base_url: '/database/:dbname/table/:table', // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_mapconfig: '/database/:dbname/layergroup', + + api_base_paths: [ + '/tiles', + '/database/:dbname' + ], + + map_base_paths: [ + '/layergroup' + ], + + template_base_paths: [ + '/template' + ], + grainstore: { datasource: { geometry_field: 'the_geom', From e06d3200c31f4e9e15b444980653d4c450f7f31e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 11 Apr 2018 12:33:07 +0200 Subject: [PATCH 045/185] Use a better API to define public routes --- lib/cartodb/api/api-router.js | 2 +- lib/cartodb/api/map/map-router.js | 2 +- lib/cartodb/api/template/template-router.js | 2 +- lib/cartodb/server_options.js | 30 +++++++++++-------- .../ported/support/ported_server_options.js | 30 +++++++++++-------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index d75d23a5..b122a343 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -206,7 +206,7 @@ module.exports = class ApiRouter { apiRouter.use(syntaxError()); apiRouter.use(errorMiddleware()); - const paths = this.serverOptions.api_base_paths; + const paths = this.serverOptions.routes.api.paths; app.use(`(?:${paths.join('|')})`, apiRouter); } diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index e8a3c2fb..ae681168 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -129,7 +129,7 @@ module.exports = class MapRouter { this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); - const paths = this.serverOptions.map_base_paths; + const paths = this.serverOptions.routes.api.map.paths; apiRouter.use(`(?:${paths.join('|')})`, mapRouter); } diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js index 9b759f0a..cfb883e7 100644 --- a/lib/cartodb/api/template/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -61,7 +61,7 @@ module.exports = class TemplateRouter { this.tileTemplateController.register(templateRouter); this.adminTemplateController.register(templateRouter); - const paths = this.serverOptions.template_base_paths; + const paths = this.serverOptions.routes.api.template.paths; apiRouter.use(`(?:${paths.join('|')})`, templateRouter); } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index c73a6bca..d05e5e1d 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -67,18 +67,24 @@ module.exports = { // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)', - api_base_paths: global.environment.api_base_paths || [ - '/api/v1', - '/user/:user/api/v1' - ], - - map_base_paths: global.environment.map_base_paths || [ - '/map' - ], - - template_base_paths: global.environment.template_base_paths || [ - '/map/named' - ], + routes: global.environment.routes || { + api: { + paths: [ + '/api/v1', + '/user/:user/api/v1' + ], + map: { + paths: [ + '/map' + ] + }, + template: { + paths: [ + '/map/named' + ] + } + } + }, grainstore: { map: { diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 931fa584..d850b1c9 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -12,18 +12,24 @@ module.exports = _.extend({}, serverOptions, { // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_mapconfig: '/database/:dbname/layergroup', - api_base_paths: [ - '/tiles', - '/database/:dbname' - ], - - map_base_paths: [ - '/layergroup' - ], - - template_base_paths: [ - '/template' - ], + routes: { + api: { + paths: [ + '/tiles', + '/database/:dbname' + ], + map: { + paths: [ + '/layergroup' + ] + }, + template: { + paths: [ + '/template' + ] + } + } + }, grainstore: { datasource: { From 9a52edacb2a7ac56659af060bb490e34e6956c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 11 Apr 2018 16:00:14 +0200 Subject: [PATCH 046/185] Add routes configuration to config example files --- config/environments/development.js.example | 34 ++++++++++++++++++++++ config/environments/production.js.example | 34 ++++++++++++++++++++++ config/environments/staging.js.example | 34 ++++++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 12de0912..95711dad 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -13,6 +13,8 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.localhost' + // DEPRECATED: use routes property instead + // --------------------------------------- // Base URLs for the APIs // // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API @@ -26,6 +28,38 @@ var config = { // "tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Base URLs for the APIs + // + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + api: { + // "/tiles" is for compatibility with versions up to 1.6.x + paths: [ + '/api/v1', + '/user/:user/api/v1', + '/tiles' // for compatibility with versions up to 1.6.x + ], + // Base url for the Detached Maps API + // "maps" is the the new API, + // "tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/map', + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/map/named', + '/template' + ] + } + } + } + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be diff --git a/config/environments/production.js.example b/config/environments/production.js.example index beb9f15c..7b552507 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -13,6 +13,8 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' + // DEPRECATED: use routes property instead + // --------------------------------------- // Base URLs for the APIs // // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API @@ -26,6 +28,38 @@ var config = { // "tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Base URLs for the APIs + // + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + api: { + // "/tiles" is for compatibility with versions up to 1.6.x + paths: [ + '/api/v1', + '/user/:user/api/v1', + '/tiles' // for compatibility with versions up to 1.6.x + ], + // Base url for the Detached Maps API + // "maps" is the the new API, + // "tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/map', + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/map/named', + '/template' + ] + } + } + } + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 40d3bb8b..29c15781 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -13,6 +13,8 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' + // DEPRECATED: use routes property instead + // --------------------------------------- // Base URLs for the APIs // // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API @@ -26,6 +28,38 @@ var config = { // "/tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Base URLs for the APIs + // + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + api: { + // "/tiles" is for compatibility with versions up to 1.6.x + paths: [ + '/api/v1', + '/user/:user/api/v1', + '/tiles' // for compatibility with versions up to 1.6.x + ], + // Base url for the Detached Maps API + // "maps" is the the new API, + // "tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/map', + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/map/named', + '/template' + ] + } + } + } + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be From 730b29c9cc5b5fbf5521f50ba72779a6271217f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 11 Apr 2018 18:51:44 +0200 Subject: [PATCH 047/185] Remove defined dependecies but never used --- .../api/map/analysis-layergroup-controller.js | 13 +------------ lib/cartodb/api/map/map-router.js | 5 +---- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js index 5d89803b..0e46dc81 100644 --- a/lib/cartodb/api/map/analysis-layergroup-controller.js +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -8,22 +8,11 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const dbParamsFromResLocals = require('../../utils/database-params'); module.exports = class AnalysisLayergroupController { - constructor ( - analysisStatusBackend, - pgConnection, - mapStore, - userLimitsBackend, - layergroupAffectedTablesCache, - authBackend, - surrogateKeysCache - ) { + constructor (analysisStatusBackend, pgConnection, userLimitsBackend, authBackend) { this.analysisStatusBackend = analysisStatusBackend; this.pgConnection = pgConnection; - this.mapStore = mapStore; this.userLimitsBackend = userLimitsBackend; - this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; this.authBackend = authBackend; - this.surrogateKeysCache = surrogateKeysCache; } register (mapRouter) { diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index ae681168..a3c2bbaa 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -38,11 +38,8 @@ module.exports = class MapRouter { this.analysisLayergroupController = new AnalysisLayergroupController( analysisStatusBackend, pgConnection, - mapStore, userLimitsBackend, - layergroupAffectedTablesCache, - authBackend, - surrogateKeysCache + authBackend ); this.attributesLayergroupController = new AttributesLayergroupController( From 56495522b8a23d96db22789dd654ec6d032ee3d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 18 Apr 2018 18:52:09 +0200 Subject: [PATCH 048/185] Use array of path to avoid collisions and extract scale-factor param properly --- .../api/map/tile-layergroup-controller.js | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js index c79929b7..2c049019 100644 --- a/lib/cartodb/api/map/tile-layergroup-controller.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -42,13 +42,11 @@ module.exports = class TileLayergroupController { } register (mapRouter) { - // REGEXP matches with positive and negative numnbers - const number = () => `(-?\\d+)`; // REGEXP doesn't match with `val` const not = (val) => `(?!${val})([^\/]+?)`; mapRouter.get( - `/:token/:z${number()}/:x${number()}/:y${number()}@:scale_factor${number()}?x.:format`, + [ `/:token/:z/:x/:y@:scale_factor?x.:format`, `/:token/:z/:x/:y.:format` ], layergroupToken(), coordinates(), credentials(), @@ -74,33 +72,7 @@ module.exports = class TileLayergroupController { ); mapRouter.get( - `/:token/:z${number()}/:x${number()}/:y${number()}.:format`, - layergroupToken(), - coordinates(), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - cleanUpQueryParams(), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getTile(this.tileBackend, 'map_tile'), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - incrementSuccessMetrics(global.statsClient), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - mapRouter.get( - `/:token${not('static')}/:layer/:z${number()}/:x${number()}/:y${number()}.(:format)`, + `/:token${not('static')}/:layer/:z/:x/:y.(:format)`, layergroupToken(), coordinates(), credentials(), From 9bd9503e9bb22fd2b65ecdc2d032b913ab546b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 18 Apr 2018 19:07:38 +0200 Subject: [PATCH 049/185] Merge paths to perform the same middleware stack --- .../api/map/tile-layergroup-controller.js | 39 ++++--------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js index 2c049019..f882d13c 100644 --- a/lib/cartodb/api/map/tile-layergroup-controller.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -45,8 +45,11 @@ module.exports = class TileLayergroupController { // REGEXP doesn't match with `val` const not = (val) => `(?!${val})([^\/]+?)`; - mapRouter.get( - [ `/:token/:z/:x/:y@:scale_factor?x.:format`, `/:token/:z/:x/:y.:format` ], + mapRouter.get([ + `/:token/:z/:x/:y@:scale_factor?x.:format`, + `/:token/:z/:x/:y.:format`, + `/:token${not('static')}/:layer/:z/:x/:y.(:format)` + ], layergroupToken(), coordinates(), credentials(), @@ -60,33 +63,7 @@ module.exports = class TileLayergroupController { this.pgConnection, this.layergroupAffectedTablesCache ), - getTile(this.tileBackend, 'map_tile'), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - incrementSuccessMetrics(global.statsClient), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - mapRouter.get( - `/:token${not('static')}/:layer/:z/:x/:y.(:format)`, - layergroupToken(), - coordinates(), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - cleanUpQueryParams(), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getTile(this.tileBackend, 'maplayer_tile'), + getTile(this.tileBackend), cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), @@ -108,9 +85,9 @@ function getStatusCode(tile, format){ return tile.length === 0 && format === 'mvt' ? 204 : 200; } -function getTile (tileBackend, profileLabel = 'tile') { +function getTile (tileBackend) { return function getTileMiddleware (req, res, next) { - req.profiler.start(`windshaft.${profileLabel}`); + req.profiler.start(`windshaft.${req.params.layer ? 'maplayer_tile' : 'map_tile'}`); const { mapConfigProvider } = res.locals; const { token } = res.locals; From 3c3731252dfffb2a197f6b18ad89a49c5859ab67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 May 2018 18:24:41 +0200 Subject: [PATCH 050/185] Respect default values with object.assign --- lib/cartodb/controllers/factory.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/controllers/factory.js b/lib/cartodb/controllers/factory.js index ce341fbd..27ad35bf 100644 --- a/lib/cartodb/controllers/factory.js +++ b/lib/cartodb/controllers/factory.js @@ -59,11 +59,11 @@ const ServerInfoController = require('./server-info'); module.exports = class ControllersFactory { constructor ({ serverOptions, environmentOptions }) { - const redisOptions = Object.assign({}, environmentOptions.redis, { + const redisOptions = Object.assign({ name: 'windshaft-server', unwatchOnRelease: false, noReadyCheck: true - }); + }, environmentOptions.redis); const redisPool = new RedisPool(redisOptions); @@ -84,10 +84,11 @@ module.exports = class ControllersFactory { const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); - const rendererCacheOpts = Object.assign({}, serverOptions.renderCache || {}, { + const rendererCacheOpts = Object.assign({ ttl: 60000, // 60 seconds TTL by default statsInterval: 60000 // reports stats every milliseconds defined here - }); + }, serverOptions.renderCache || {}); + const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); rendererStatsReporter.start(); From 7561635b24e6dcc61c0dbcc31b897b187edf0dfc Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 7 May 2018 19:03:19 +0200 Subject: [PATCH 051/185] WIP:add layer metadata --- .../layer-stats/mapnik-layer-stats.js | 170 +++++++++++++++++- lib/cartodb/utils/query-utils.js | 81 +++++++++ 2 files changed, 242 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index c060f964..d5806960 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,5 @@ var queryUtils = require('../../utils/query-utils'); +const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); function MapnikLayerStats () { this._types = { @@ -11,18 +12,169 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; +function queryPromise(dbConnection, query, callback) { + return new Promise(function(resolve, reject) { + dbConnection.query(query, function (err, res) { + err = callback(err, res); + if (err) { + reject(err); + } + else { + resolve(); + } + }); + + }); +} + MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); - // This query would gather stats for postgresql table if not exists - dbConnection.query(queryRowCountSql, function (err, res) { - if (err) { - return callback(null, {estimatedFeatureCount: -1}); - } else { - // We decided that the relation is 1 row == 1 feature - return callback(null, {estimatedFeatureCount: res.rows[0].rows}); + let query = layer.options.sql; + let rawQuery = layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql; + let metaOptions = layer.options.metadata || {}; + + let stats = {}; + + // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere + // we would set stats.estimatedFeatureCount and stats.geometryType (if metaOptions.geometryType) from it. + + // We'll add promises for queries to be executed to the next two lists; + // the queries in statQueries2 will be executed after all of statQueries are completed, + // so any results from them can be used. + // Query promises will store results in the shared stats object. + let statQueries = [], statQueries2 = []; + + if (stats.estimatedFeatureCount === undefined) { + statQueries.push( + queryPromise(dbConnection, queryUtils.getQueryRowEstimation(query), function(err, res) { + if (err) { + // at least for debugging we should err + stats.estimatedFeatureCount = -1; + return null; + } else { + // We decided that the relation is 1 row == 1 feature + stats.estimatedFeatureCount = res.rows[0].rows; + return null; + } + }) + ); + } + + if (metaOptions.featureCount) { + // TODO: if metaOptions.columnStats we can combine this with column stats query + statQueries.push( + queryPromise( + queryUtils.getQueryActualRowCount(rawQuery), + function(err, res) { + if (err) { + stats.featureCount = -1; + } else { + stats.featureCount = res.rows[0].rows; + } + return err; + } + ) + ); + } + + if (metaOptions.sample) { + const numRows = stats.featureCount === undefined ? stats.estimatedFeatureCount : stats.featureCount; + const sampleProb = Math.min(metaOptions.sample / numRows, 1); + statQueries2.push( + queryPromise( + queryUtils.getQuerySample(rawQuery, sampleProb), + function(err, res) { + if (err) { + stats.sample = []; + } else { + stats.sample = res.rows; + } + return err; + } + ) + ); + } + + if (metaOptions.geometryType && stats.geometryType === undefined) { + const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); + statQueries.push( + queryPromise(queryUtils.getQueryGeometryType(rawQuery, geometryColumn), function(err, res) { + if (!err) { + stats.geometryType = res.geom_type; + } + return err; + }) + ); + } + + function columnAggregations(field) { + if (field.type === 'number') { + return ['min', 'max', 'avg', 'sum']; } - }); + if (field.type === 'date') { // TODO other types too? + return ['min', 'max']; + } + } + + if (metaOptions.columns || metaOptions.columnStats) { + statQueries.push( + // TODO: note we have getLayerColumns in aggregation mapconfig. + // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters + queryPromise(queryUtils.getQueryLimited(rawQuery, 0), function(err, res) { + if (!err) { + stats.columns = res.fields; + if (metaOptions.columnStats) { + let aggr = []; + Object.keys(stats.columns).forEach(name => { + aggr = aggr.concat(columnAggregations(stats.columns[name]) + .map(fn => `${fn}(${name}) AS ${name}_${fn}`)); + if (stats.columns[name].type === 'string') { + statQueries2.push( + queryPromise(topQuery(rawQuery, name, N), function(err, res){ + if (!err) { + const topN = metaOptions.columnStats.topCategories || 1024; + // TODO: metaOptions.columnStats.maxCategories => use PG stats to dismiss columns with more distinct values + statQueries2.push( + queryPromise( + queryUtils.getQueryTopCategories(rawQuery, topN), + function(err, res) { + if (!err) { + stats.columns[name].categories = res.rows; + } + return err; + } + ) + ); + + } + return err; + }) + ); + } + }) + statQueries2.push( + queryPromise(`SELECT ${aggr.join(',')} FROM (${rawQuery})`, function(err, res){ + if (!err) { + Object.keys(stats.columns).forEach(name => { + columnAggregations(stats.columns[name]).forEach(fn => { + stats.columns[name][fn] = res.rows[0][`${name}_${fn}`] + }); + }); + } + return err; + }) + ); + } + } + return err; + }) + ); + + } + + Promise.all(statQueries).then( () => { + Promise.all(statQueries2).then( () => callback(null, stats) ).catch( err => callback(err) ); + }).catch( err => callback(err) ); }; module.exports = MapnikLayerStats; diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index c1b56276..e4bd67d3 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -71,3 +71,84 @@ module.exports.countNaNs = function countNaNs(ctx) { `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` }`; }; + +module.exports.getQueryTopCategories = function(query, column, topN, includeNulls=false) { + const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`; + return ` + SELECT ${column} AS category, COUNT(*) AS frequency + FROM (${query}) AS __cdb_query + ${where} + GROUP BY ${column} ORDER BY 2 DESC + LIMIT ${topN} + `; +} + +module.exports.getQueryActualRowCount = function (query) { + return 'select COUNT(*) AS rows FROM (${query}) AS __cdb_query'; +}; + + +module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { + const table = simpleQueryTable(query); + if (table) { + return getTableSample(table, sampleProb, randomSeed); + } + return ` + WITH __cdb_rndseed AS ( + SELECT setseed(${randomSeed}) + ) + SELECT * + FROM (${query}) AS __cdb_query + WHERE random() < $ + `; + q = `WITH _rndseed as (SELECT setseed(0.5)) + SELECT * FROM (${this._source._query}) as _cdb_query_wrapper WHERE random() < ${sampleProb};`; +}; + +module.exports.getTableSample = function(table, sampleProb, randomSeed) { + sampleProb *= 100; + randomSeed *= Math.pow(2, 31) -1; + return ` + SELECT * FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) + `; +} + +function simpleQueryTable(sql) { + const basicQuery = + /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; + const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i'); + // queries for named maps are wrapped like this: + var wrappedQuery = new RegExp( + "^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" + + basicQuery.source + + "\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$", + 'i' + ); + let match = sql.match(unwrappedQuery); + if (!match) { + match = sql.match(wrappedQuery); + } + if (match) { + schema = match[2]; + table = match[3]; + return schema ? `${schema}.${table}` : table; + } + return false; +} + +module.exports.getQueryGeometryType = function(query, geometryColumn) { + return ` + SELECT ST_GeometryType(${geometryColumn}) AS geom_type + FROM (${query}) AS __cdb_query + WHERE ${geometryColumn} IS NOT NULL + LIMIT 1 + `; +}; + +module.exports.getQueryLimited = function(query, limit=0) { + return ` + SELECT * + FROM (${query}) AS __cdb_query + LIMIT ${limit} + `; +}; From ebab879acaf622ab3c3ddf5df83761d4e5ec3cbf Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 11:07:47 +0200 Subject: [PATCH 052/185] Fix bugs & typos --- lib/cartodb/utils/query-utils.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index e4bd67d3..238a905d 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -81,10 +81,10 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull GROUP BY ${column} ORDER BY 2 DESC LIMIT ${topN} `; -} +}; module.exports.getQueryActualRowCount = function (query) { - return 'select COUNT(*) AS rows FROM (${query}) AS __cdb_query'; + return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; }; @@ -101,11 +101,9 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { FROM (${query}) AS __cdb_query WHERE random() < $ `; - q = `WITH _rndseed as (SELECT setseed(0.5)) - SELECT * FROM (${this._source._query}) as _cdb_query_wrapper WHERE random() < ${sampleProb};`; }; -module.exports.getTableSample = function(table, sampleProb, randomSeed) { +function getTableSample(table, sampleProb, randomSeed) { sampleProb *= 100; randomSeed *= Math.pow(2, 31) -1; return ` @@ -129,8 +127,8 @@ function simpleQueryTable(sql) { match = sql.match(wrappedQuery); } if (match) { - schema = match[2]; - table = match[3]; + const schema = match[2]; + const table = match[3]; return schema ? `${schema}.${table}` : table; } return false; From c647f852d64a3cdb3c2e8f14b80d3bc1c62b9b70 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 11:09:09 +0200 Subject: [PATCH 053/185] Refactor metadata queries execution Also fixed bug where sampling query generation needed results of count queries --- .../layer-stats/mapnik-layer-stats.js | 277 +++++++++--------- lib/cartodb/utils/phased-execution.js | 91 ++++++ 2 files changed, 231 insertions(+), 137 deletions(-) create mode 100644 lib/cartodb/utils/phased-execution.js diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index d5806960..047229af 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,5 @@ var queryUtils = require('../../utils/query-utils'); +const PhasedExecution = require('../../utils/phased-execution'); const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); function MapnikLayerStats () { @@ -27,154 +28,156 @@ function queryPromise(dbConnection, query, callback) { }); } +function columnAggregations(field) { + if (field.type === 'number') { + return ['min', 'max', 'avg', 'sum']; + } + if (field.type === 'date') { // TODO other types too? + return ['min', 'max']; + } +} + +function firstPhaseQueries(queries, ctx) { + if (queries.results.estimatedFeatureCount === undefined) { + queries.task( + queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.query), function(err, res) { + if (err) { + // at least for debugging we should err + queries.results.estimatedFeatureCount = -1; + return null; + } else { + // We decided that the relation is 1 row == 1 feature + queries.results.estimatedFeatureCount = res.rows[0].rows; + return null; + } + }) + ); + } + + if (ctx.metaOptions.featureCount) { + // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query + queries.task( + queryPromise( + queryUtils.getQueryActualRowCount(ctx.rawQuery), + function(err, res) { + if (err) { + queries.results.featureCount = -1; + } else { + queries.results.featureCount = res.rows[0].rows; + } + return err; + } + ) + ); + } + + if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { + const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); + queries.task( + queryPromise(queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn), function(err, res) { + if (!err) { + queries.results.geometryType = res.geom_type; + } + return err; + }) + ); + } + + if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { + queries.task( + // TODO: note we have getLayerColumns in aggregation mapconfig. + // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters + queryPromise(queryUtils.getQueryLimited(ctx.rawQuery, 0), function(err, res) { + if (!err) { + queries.results.columns = res.fields; + } + return err; + }) + ); + } +} + +function secondPhaseQueries(queries, ctx) { + if (ctx.metaOptions.sample) { + const numRows = queries.results.featureCount === undefined ? + queries.results.estimatedFeatureCount : + queries.results.featureCount; + const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); + queries.task( + queryPromise( + queryUtils.getQuerySample(ctx.rawQuery, sampleProb), + function(err, res) { + if (err) { + queries.results.sample = []; + } else { + queries.results.sample = res.rows; + } + return err; + } + ) + ); + } + + if (ctx.metaOptions.columnStats) { + let aggr = []; + Object.keys(queries.results.columns).forEach(name => { + aggr = aggr.concat(columnAggregations(queries.results.columns[name]) + .map(fn => `${fn}(${name}) AS ${name}_${fn}`)); + if (queries.results.columns[name].type === 'string') { + const topN = ctx.metaOptions.columnStats.topCategories || 1024; + // TODO: ctx.metaOptions.columnStats.maxCategories + // => use PG stats to dismiss columns with more distinct values + queries.task( + queryPromise(queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN), function(err, res){ + if (!err) { + queries.results.columns[name].categories = res.rows; + } + return err; + }) + ); + } + }); + queries.task( + queryPromise(`SELECT ${aggr.join(',')} FROM (${ctx.rawQuery})`, function(err, res){ + if (!err) { + Object.keys(queries.results.columns).forEach(name => { + columnAggregations(queries.results.columns[name]).forEach(fn => { + queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + }); + } + return err; + }) + ); + } + +} + MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - let query = layer.options.sql; - let rawQuery = layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql; - let metaOptions = layer.options.metadata || {}; + let context = { + dbConnection, + query: layer.options.sql, + rawQuery: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql, + metaOptions: layer.options.metadata || {} + }; - let stats = {}; + let queries = new PhasedExecution(); // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere - // we would set stats.estimatedFeatureCount and stats.geometryType (if metaOptions.geometryType) from it. + // we would set queries.results.estimatedFeatureCount and queries.results.geometryType + // (if metaOptions.geometryType) from it. // We'll add promises for queries to be executed to the next two lists; // the queries in statQueries2 will be executed after all of statQueries are completed, // so any results from them can be used. // Query promises will store results in the shared stats object. - let statQueries = [], statQueries2 = []; - if (stats.estimatedFeatureCount === undefined) { - statQueries.push( - queryPromise(dbConnection, queryUtils.getQueryRowEstimation(query), function(err, res) { - if (err) { - // at least for debugging we should err - stats.estimatedFeatureCount = -1; - return null; - } else { - // We decided that the relation is 1 row == 1 feature - stats.estimatedFeatureCount = res.rows[0].rows; - return null; - } - }) - ); - } - - if (metaOptions.featureCount) { - // TODO: if metaOptions.columnStats we can combine this with column stats query - statQueries.push( - queryPromise( - queryUtils.getQueryActualRowCount(rawQuery), - function(err, res) { - if (err) { - stats.featureCount = -1; - } else { - stats.featureCount = res.rows[0].rows; - } - return err; - } - ) - ); - } - - if (metaOptions.sample) { - const numRows = stats.featureCount === undefined ? stats.estimatedFeatureCount : stats.featureCount; - const sampleProb = Math.min(metaOptions.sample / numRows, 1); - statQueries2.push( - queryPromise( - queryUtils.getQuerySample(rawQuery, sampleProb), - function(err, res) { - if (err) { - stats.sample = []; - } else { - stats.sample = res.rows; - } - return err; - } - ) - ); - } - - if (metaOptions.geometryType && stats.geometryType === undefined) { - const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); - statQueries.push( - queryPromise(queryUtils.getQueryGeometryType(rawQuery, geometryColumn), function(err, res) { - if (!err) { - stats.geometryType = res.geom_type; - } - return err; - }) - ); - } - - function columnAggregations(field) { - if (field.type === 'number') { - return ['min', 'max', 'avg', 'sum']; - } - if (field.type === 'date') { // TODO other types too? - return ['min', 'max']; - } - } - - if (metaOptions.columns || metaOptions.columnStats) { - statQueries.push( - // TODO: note we have getLayerColumns in aggregation mapconfig. - // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters - queryPromise(queryUtils.getQueryLimited(rawQuery, 0), function(err, res) { - if (!err) { - stats.columns = res.fields; - if (metaOptions.columnStats) { - let aggr = []; - Object.keys(stats.columns).forEach(name => { - aggr = aggr.concat(columnAggregations(stats.columns[name]) - .map(fn => `${fn}(${name}) AS ${name}_${fn}`)); - if (stats.columns[name].type === 'string') { - statQueries2.push( - queryPromise(topQuery(rawQuery, name, N), function(err, res){ - if (!err) { - const topN = metaOptions.columnStats.topCategories || 1024; - // TODO: metaOptions.columnStats.maxCategories => use PG stats to dismiss columns with more distinct values - statQueries2.push( - queryPromise( - queryUtils.getQueryTopCategories(rawQuery, topN), - function(err, res) { - if (!err) { - stats.columns[name].categories = res.rows; - } - return err; - } - ) - ); - - } - return err; - }) - ); - } - }) - statQueries2.push( - queryPromise(`SELECT ${aggr.join(',')} FROM (${rawQuery})`, function(err, res){ - if (!err) { - Object.keys(stats.columns).forEach(name => { - columnAggregations(stats.columns[name]).forEach(fn => { - stats.columns[name][fn] = res.rows[0][`${name}_${fn}`] - }); - }); - } - return err; - }) - ); - } - } - return err; - }) - ); - - } - - Promise.all(statQueries).then( () => { - Promise.all(statQueries2).then( () => callback(null, stats) ).catch( err => callback(err) ); - }).catch( err => callback(err) ); + // Queries will be executed in two phases, with results from the first phase needed + // to define the queries of the second phase + queries.phase(() => firstPhaseQueries(queries, context)); + queries.phase(() => secondPhaseQueries(queries, context)); + queries.run(callback); }; module.exports = MapnikLayerStats; diff --git a/lib/cartodb/utils/phased-execution.js b/lib/cartodb/utils/phased-execution.js new file mode 100644 index 00000000..ec1de9b1 --- /dev/null +++ b/lib/cartodb/utils/phased-execution.js @@ -0,0 +1,91 @@ +/** + * PhasedExecution handles the execution of async tasks (via Promises) + * which have dependencies between them in a simplified manner. + * Instead of using the complete task dependency graph, tasks + * are organized into execution phases. So that tasks from a latter + * phase will be initialized after tasks from previous phases have + * finished. + * + * All tasks place their results in a shared object to make them + * available to tasks of latter phases. + * + * Each phase is defined by a function that defines its tasks. + * + * Example: + * + * let p = new PhasedExecution(); + * // Define first phase with tasks 1 & 2 + * p.phase(() => { + * p.results.phase1 = 1 + * p.task(new Promise((resolve) => { + * setTimeout( () => { + * console.log('At 1:', p.results); + * p.results.task1 = 100; + * resolve(); + * }, 400); + * })); + * p.task(new Promise((resolve) => { + * setTimeout( () => { + * console.log('At 2:', p.results); + * p.results.task2 = 200; + * resolve(); + * }, 100); + * })); + * }); + * // Define second phase with tasks 3 & 4 + * p.phase(() => { + * p.results.phase2 = 2 + * p.task(new Promise((resolve) => { + * setTimeout( () => { + * console.log('At 3:', p.results); + * p.results.task3 = 300; + * resolve(); + * }, 50); + * })); + * p.task(new Promise((resolve) => { + * setTimeout( () => { + * console.log('At 4:', p.results); + * p.results.task4 = 400; + * resolve(); + * }, 100); + * })); + * }); + * // Define third phase with task 5 + * p.phase(() => { + * p.results.phase3 = 3 + * p.task(new Promise((resolve) => { + * setTimeout( () => { + * console.log('At 5:', p.results); + * p.results.task5 = 500; + * resolve(); + * }, 50); + * })); + * }); + * // Execute all tasks + * p.run(() => { + * console.log("RESULTS:", p.results); + * }); + */ +module.exports = class PhasedExecution { + constructor() { + this.results = {}; + this.phases = []; + } + phase(phasegenerator) { + this.phases.push(phasegenerator); + } + task(promise) { + this.tasks.push(promise); + } + run(callback) { + this.tasks = []; + let phase = this.phases.shift(); + if (phase) { + phase(this); + return Promise.all(this.tasks) + .then(this.run()) + .then(() => { if (callback) { callback(this.results); } }); + } + } +}; +// TODO: error handling From 636cd8cd509124e38a97f350cb51f7ecd49770bc Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 12:56:06 +0200 Subject: [PATCH 054/185] Fix:phase execution phase (not only its tasks) must be executed after the tasks of previous phases --- lib/cartodb/utils/phased-execution.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/utils/phased-execution.js b/lib/cartodb/utils/phased-execution.js index ec1de9b1..68fc7933 100644 --- a/lib/cartodb/utils/phased-execution.js +++ b/lib/cartodb/utils/phased-execution.js @@ -83,7 +83,7 @@ module.exports = class PhasedExecution { if (phase) { phase(this); return Promise.all(this.tasks) - .then(this.run()) + .then(() => this.run()) .then(() => { if (callback) { callback(this.results); } }); } } From b96be69a5c68df5af0753b4a6a6f4e5b8c28bcc0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 13:08:01 +0200 Subject: [PATCH 055/185] Clarify example --- lib/cartodb/utils/phased-execution.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/utils/phased-execution.js b/lib/cartodb/utils/phased-execution.js index 68fc7933..c9ac445d 100644 --- a/lib/cartodb/utils/phased-execution.js +++ b/lib/cartodb/utils/phased-execution.js @@ -16,17 +16,18 @@ * let p = new PhasedExecution(); * // Define first phase with tasks 1 & 2 * p.phase(() => { + * console.log('At phase I', p.results);* * p.results.phase1 = 1 * p.task(new Promise((resolve) => { * setTimeout( () => { - * console.log('At 1:', p.results); + * console.log('At task 1:', p.results); * p.results.task1 = 100; * resolve(); * }, 400); * })); * p.task(new Promise((resolve) => { * setTimeout( () => { - * console.log('At 2:', p.results); + * console.log('At task 2:', p.results); * p.results.task2 = 200; * resolve(); * }, 100); @@ -34,17 +35,18 @@ * }); * // Define second phase with tasks 3 & 4 * p.phase(() => { + * console.log('At phase II', p.results); * p.results.phase2 = 2 * p.task(new Promise((resolve) => { * setTimeout( () => { - * console.log('At 3:', p.results); + * console.log('At task 3:', p.results); * p.results.task3 = 300; * resolve(); * }, 50); * })); * p.task(new Promise((resolve) => { * setTimeout( () => { - * console.log('At 4:', p.results); + * console.log('At task 4:', p.results); * p.results.task4 = 400; * resolve(); * }, 100); @@ -52,18 +54,21 @@ * }); * // Define third phase with task 5 * p.phase(() => { + * console.log('At phase III', p.results); * p.results.phase3 = 3 * p.task(new Promise((resolve) => { * setTimeout( () => { - * console.log('At 5:', p.results); + * console.log('At task 5:', p.results); * p.results.task5 = 500; * resolve(); * }, 50); * })); * }); * // Execute all tasks - * p.run(() => { - * console.log("RESULTS:", p.results); + * p.run((results) => { + * console.log("RESULTS:", results); + * }).catch((err) => { + * console.log("ERROR:", error); * }); */ module.exports = class PhasedExecution { @@ -88,4 +93,3 @@ module.exports = class PhasedExecution { } } }; -// TODO: error handling From 7d68a2967fd38e4c5ecfbf9c0a7407763937f268 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 13:08:26 +0200 Subject: [PATCH 056/185] Fix: callback expected errors in first argument --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 047229af..a86f49be 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -177,7 +177,7 @@ function (layer, dbConnection, callback) { // to define the queries of the second phase queries.phase(() => firstPhaseQueries(queries, context)); queries.phase(() => secondPhaseQueries(queries, context)); - queries.run(callback); + queries.run(results => callback(null, results)).catch(error => callback(error)); }; module.exports = MapnikLayerStats; From 2cf0b9d097b7dfe26f7d52747ccf484770fa68ed Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 8 May 2018 14:04:19 +0200 Subject: [PATCH 057/185] Update Windshaft to 4.7.2 --- NEWS.md | 1 + package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 74bbe860..bd8fab57 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ Released 2018-mm-dd New features: - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 +- Upgrades Windshaft to [4.7.2](https://github.com/CartoDB/Windshaft/blob/4.7.2/NEWS.md#version-472). Includes improvements in Mapnik and sets MVT's simplify_distance option to 0. Bug Fixes: diff --git a/package.json b/package.json index 1387632c..697d8fe4 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.7.1", + "windshaft": "4.7.2", "yargs": "~5.0.0" }, "devDependencies": { From fa647a915ce66c1122eff3e7daf18864b7654d98 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 8 May 2018 14:08:12 +0200 Subject: [PATCH 058/185] Update yarn.lock with the Windshaft 4.7.2 --- yarn.lock | 154 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 36 deletions(-) diff --git a/yarn.lock b/yarn.lock index 357b8175..444d50da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,20 +2,20 @@ # yarn lockfile v1 -"@carto/mapnik@3.6.2-carto.8": - version "3.6.2-carto.8" - resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.8.tgz#70448689d9b14d644bebd079f5714871c458a46d" +"@carto/mapnik@3.6.2-carto.9": + version "3.6.2-carto.9" + resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.9.tgz#a56c976c944d67f2accc7ade1283a898006328b7" dependencies: mapnik-vector-tile cartodb/mapnik-vector-tile#v1.6.1-carto.1 nan "2.10.0" - node-pre-gyp "0.7.0" + node-pre-gyp "0.10.0" protozero "1.5.1" -"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb7": - version "2.5.1-cdb7" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/ec881cb9ac52113f895f23857430e2d434bb99a6" +"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb8": + version "2.5.1-cdb8" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/135a07f7dc6c2c5fe676d6cd989b060b88c5cafe" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.9" "@mapbox/sphericalmercator" "~1.0.1" mapnik-pool "~0.1.3" @@ -23,11 +23,11 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -"abaculus@github:cartodb/abaculus#2.0.3-cdb8": - version "2.0.3-cdb8" - resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/31c03f2442943d4c47740fa154cda753b5cccd8a" +"abaculus@github:cartodb/abaculus#2.0.3-cdb9": + version "2.0.3-cdb9" + resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/37813206b6423d44d0c073eaffa4d608beb910f5" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.9" d3-queue "^2.0.2" sphericalmercator "1.0.x" @@ -338,6 +338,10 @@ chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +chownr@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" + chroma-js@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" @@ -455,7 +459,7 @@ debug@2.6.0: dependencies: ms "0.7.2" -debug@2.6.9, debug@^2.2.0: +debug@2.6.9, debug@^2.1.2, debug@^2.2.0: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -748,6 +752,12 @@ fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" +fs-minipass@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d" + dependencies: + minipass "^2.2.1" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1005,6 +1015,18 @@ iconv-lite@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" +iconv-lite@^0.4.4: + version "0.4.23" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" + dependencies: + safer-buffer ">= 2.1.2 < 3" + +ignore-walk@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.1.tgz#a83e62e7d272ac0e3b551aaa82831a19b69f82f8" + dependencies: + minimatch "^3.0.4" + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -1363,6 +1385,19 @@ minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" +minipass@^2.2.1, minipass@^2.2.4: + version "2.3.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.0.tgz#2e11b1c46df7fe7f1afbe9a490280add21ffe384" + dependencies: + safe-buffer "^5.1.1" + yallist "^3.0.0" + +minizlib@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.0.tgz#11e13658ce46bc3a70a267aac58359d1e0c29ceb" + dependencies: + minipass "^2.2.1" + mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" @@ -1429,6 +1464,14 @@ ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" +needle@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/needle/-/needle-2.2.1.tgz#b5e325bd3aae8c2678902fa296f729455d1d3a7d" + dependencies: + debug "^2.1.2" + iconv-lite "^0.4.4" + sax "^1.2.4" + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -1444,20 +1487,20 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.7.0.tgz#55aeffbaed93b50d0a4657d469198cd80ac9df36" +node-pre-gyp@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.10.0.tgz#6e4ef5bb5c5203c6552448828c852c40111aac46" dependencies: detect-libc "^1.0.2" mkdirp "^0.5.1" + needle "^2.2.0" nopt "^4.0.1" + npm-packlist "^1.1.6" npmlog "^4.0.2" rc "^1.1.7" - request "2.83.0" rimraf "^2.6.1" semver "^5.3.0" - tar "^2.2.1" - tar-pack "^3.4.0" + tar "^4" node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: version "0.6.39" @@ -1501,6 +1544,17 @@ normalize-package-data@^2.3.2: semver "2 || 3 || 4 || 5" validate-npm-package-license "^3.0.1" +npm-bundled@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308" + +npm-packlist@^1.1.6: + version "1.1.10" + resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.10.tgz#1039db9e985727e464df066f4cf0ab6ef85c398a" + dependencies: + ignore-walk "^3.0.1" + npm-bundled "^1.0.1" + npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" @@ -1904,9 +1958,9 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.83.0, request@2.x, request@^2.55.0: - version "2.83.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" +request@2.85.0: + version "2.85.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1931,9 +1985,9 @@ request@2.83.0, request@2.x, request@^2.55.0: tunnel-agent "^0.6.0" uuid "^3.1.0" -request@2.85.0: - version "2.85.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" +request@2.x, request@^2.55.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1992,10 +2046,22 @@ safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, s version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" +safe-buffer@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" +"safer-buffer@>= 2.1.2 < 3": + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + +sax@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + "semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -2247,6 +2313,18 @@ tar@^2.2.1: fstream "^1.0.2" inherits "2" +tar@^4: + version "4.4.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.2.tgz#60685211ba46b38847b1ae7ee1a24d744a2cd462" + dependencies: + chownr "^1.0.1" + fs-minipass "^1.2.5" + minipass "^2.2.4" + minizlib "^1.1.0" + mkdirp "^0.5.0" + safe-buffer "^5.1.2" + yallist "^3.0.2" + through2@~0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/through2/-/through2-0.2.3.tgz#eb3284da4ea311b6cc8ace3653748a52abf25a3f" @@ -2258,11 +2336,11 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb12": - version "0.6.18-cdb12" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/e3c0d80e604ca4a5dfad648ee6f6fb355d415a88" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb13": + version "0.6.18-cdb13" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/0b800a4263b205b1e889c0e6d713ec288fa20c0e" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.9" generic-pool "~2.4.0" mime "~1.6.0" sphericalmercator "~1.0.4" @@ -2419,13 +2497,13 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.7.1: - version "4.7.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.1.tgz#2b92753b2f6e97b239e15e1576ec312cc7dfeb13" +windshaft@4.7.2: + version "4.7.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.2.tgz#280b208910f38417e75a82d6ec321522346d3d43" dependencies: - "@carto/mapnik" "3.6.2-carto.8" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb7 - abaculus cartodb/abaculus#2.0.3-cdb8 + "@carto/mapnik" "3.6.2-carto.9" + "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb8 + abaculus cartodb/abaculus#2.0.3-cdb9 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 cartodb-psql "^0.10.1" @@ -2439,7 +2517,7 @@ windshaft@4.7.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb12 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb13 torque.js "~2.11.0" underscore "~1.6.0" @@ -2480,6 +2558,10 @@ y18n@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" +yallist@^3.0.0, yallist@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" + yargs-parser@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" From 97d1b4fafad202e83c5d1bd0b4fcda8b89af55c5 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 8 May 2018 18:57:32 +0200 Subject: [PATCH 059/185] Revert "Update Windshaft to 4.7.2" This reverts commit 2cf0b9d097b7dfe26f7d52747ccf484770fa68ed. --- NEWS.md | 1 - package.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index bd8fab57..74bbe860 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,7 +5,6 @@ Released 2018-mm-dd New features: - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 -- Upgrades Windshaft to [4.7.2](https://github.com/CartoDB/Windshaft/blob/4.7.2/NEWS.md#version-472). Includes improvements in Mapnik and sets MVT's simplify_distance option to 0. Bug Fixes: diff --git a/package.json b/package.json index 697d8fe4..1387632c 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.7.2", + "windshaft": "4.7.1", "yargs": "~5.0.0" }, "devDependencies": { From 03bcf573c3f5ae7a832c3ec404efd561d7af5995 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 8 May 2018 18:58:04 +0200 Subject: [PATCH 060/185] Revert "Update yarn.lock with the Windshaft 4.7.2" This reverts commit fa647a915ce66c1122eff3e7daf18864b7654d98. --- yarn.lock | 154 +++++++++++++----------------------------------------- 1 file changed, 36 insertions(+), 118 deletions(-) diff --git a/yarn.lock b/yarn.lock index 444d50da..357b8175 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,20 +2,20 @@ # yarn lockfile v1 -"@carto/mapnik@3.6.2-carto.9": - version "3.6.2-carto.9" - resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.9.tgz#a56c976c944d67f2accc7ade1283a898006328b7" +"@carto/mapnik@3.6.2-carto.8": + version "3.6.2-carto.8" + resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.8.tgz#70448689d9b14d644bebd079f5714871c458a46d" dependencies: mapnik-vector-tile cartodb/mapnik-vector-tile#v1.6.1-carto.1 nan "2.10.0" - node-pre-gyp "0.10.0" + node-pre-gyp "0.7.0" protozero "1.5.1" -"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb8": - version "2.5.1-cdb8" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/135a07f7dc6c2c5fe676d6cd989b060b88c5cafe" +"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb7": + version "2.5.1-cdb7" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/ec881cb9ac52113f895f23857430e2d434bb99a6" dependencies: - "@carto/mapnik" "3.6.2-carto.9" + "@carto/mapnik" "3.6.2-carto.8" "@mapbox/sphericalmercator" "~1.0.1" mapnik-pool "~0.1.3" @@ -23,11 +23,11 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -"abaculus@github:cartodb/abaculus#2.0.3-cdb9": - version "2.0.3-cdb9" - resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/37813206b6423d44d0c073eaffa4d608beb910f5" +"abaculus@github:cartodb/abaculus#2.0.3-cdb8": + version "2.0.3-cdb8" + resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/31c03f2442943d4c47740fa154cda753b5cccd8a" dependencies: - "@carto/mapnik" "3.6.2-carto.9" + "@carto/mapnik" "3.6.2-carto.8" d3-queue "^2.0.2" sphericalmercator "1.0.x" @@ -338,10 +338,6 @@ chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" -chownr@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" - chroma-js@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" @@ -459,7 +455,7 @@ debug@2.6.0: dependencies: ms "0.7.2" -debug@2.6.9, debug@^2.1.2, debug@^2.2.0: +debug@2.6.9, debug@^2.2.0: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -752,12 +748,6 @@ fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" -fs-minipass@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d" - dependencies: - minipass "^2.2.1" - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1015,18 +1005,6 @@ iconv-lite@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" -iconv-lite@^0.4.4: - version "0.4.23" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" - dependencies: - safer-buffer ">= 2.1.2 < 3" - -ignore-walk@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.1.tgz#a83e62e7d272ac0e3b551aaa82831a19b69f82f8" - dependencies: - minimatch "^3.0.4" - inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -1385,19 +1363,6 @@ minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" -minipass@^2.2.1, minipass@^2.2.4: - version "2.3.0" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.0.tgz#2e11b1c46df7fe7f1afbe9a490280add21ffe384" - dependencies: - safe-buffer "^5.1.1" - yallist "^3.0.0" - -minizlib@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.0.tgz#11e13658ce46bc3a70a267aac58359d1e0c29ceb" - dependencies: - minipass "^2.2.1" - mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" @@ -1464,14 +1429,6 @@ ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" -needle@^2.2.0: - version "2.2.1" - resolved "https://registry.yarnpkg.com/needle/-/needle-2.2.1.tgz#b5e325bd3aae8c2678902fa296f729455d1d3a7d" - dependencies: - debug "^2.1.2" - iconv-lite "^0.4.4" - sax "^1.2.4" - negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -1487,20 +1444,20 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@0.10.0: - version "0.10.0" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.10.0.tgz#6e4ef5bb5c5203c6552448828c852c40111aac46" +node-pre-gyp@0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.7.0.tgz#55aeffbaed93b50d0a4657d469198cd80ac9df36" dependencies: detect-libc "^1.0.2" mkdirp "^0.5.1" - needle "^2.2.0" nopt "^4.0.1" - npm-packlist "^1.1.6" npmlog "^4.0.2" rc "^1.1.7" + request "2.83.0" rimraf "^2.6.1" semver "^5.3.0" - tar "^4" + tar "^2.2.1" + tar-pack "^3.4.0" node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: version "0.6.39" @@ -1544,17 +1501,6 @@ normalize-package-data@^2.3.2: semver "2 || 3 || 4 || 5" validate-npm-package-license "^3.0.1" -npm-bundled@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308" - -npm-packlist@^1.1.6: - version "1.1.10" - resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.10.tgz#1039db9e985727e464df066f4cf0ab6ef85c398a" - dependencies: - ignore-walk "^3.0.1" - npm-bundled "^1.0.1" - npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" @@ -1958,9 +1904,9 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.85.0: - version "2.85.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" +request@2.83.0, request@2.x, request@^2.55.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1985,9 +1931,9 @@ request@2.85.0: tunnel-agent "^0.6.0" uuid "^3.1.0" -request@2.x, request@^2.55.0: - version "2.83.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" +request@2.85.0: + version "2.85.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -2046,22 +1992,10 @@ safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, s version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" -safe-buffer@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"safer-buffer@>= 2.1.2 < 3": - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - -sax@^1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" - "semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -2313,18 +2247,6 @@ tar@^2.2.1: fstream "^1.0.2" inherits "2" -tar@^4: - version "4.4.2" - resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.2.tgz#60685211ba46b38847b1ae7ee1a24d744a2cd462" - dependencies: - chownr "^1.0.1" - fs-minipass "^1.2.5" - minipass "^2.2.4" - minizlib "^1.1.0" - mkdirp "^0.5.0" - safe-buffer "^5.1.2" - yallist "^3.0.2" - through2@~0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/through2/-/through2-0.2.3.tgz#eb3284da4ea311b6cc8ace3653748a52abf25a3f" @@ -2336,11 +2258,11 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb13": - version "0.6.18-cdb13" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/0b800a4263b205b1e889c0e6d713ec288fa20c0e" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb12": + version "0.6.18-cdb12" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/e3c0d80e604ca4a5dfad648ee6f6fb355d415a88" dependencies: - "@carto/mapnik" "3.6.2-carto.9" + "@carto/mapnik" "3.6.2-carto.8" generic-pool "~2.4.0" mime "~1.6.0" sphericalmercator "~1.0.4" @@ -2497,13 +2419,13 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.7.2: - version "4.7.2" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.2.tgz#280b208910f38417e75a82d6ec321522346d3d43" +windshaft@4.7.1: + version "4.7.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.1.tgz#2b92753b2f6e97b239e15e1576ec312cc7dfeb13" dependencies: - "@carto/mapnik" "3.6.2-carto.9" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb8 - abaculus cartodb/abaculus#2.0.3-cdb9 + "@carto/mapnik" "3.6.2-carto.8" + "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb7 + abaculus cartodb/abaculus#2.0.3-cdb8 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 cartodb-psql "^0.10.1" @@ -2517,7 +2439,7 @@ windshaft@4.7.2: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb13 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb12 torque.js "~2.11.0" underscore "~1.6.0" @@ -2558,10 +2480,6 @@ y18n@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" -yallist@^3.0.0, yallist@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" - yargs-parser@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" From 9c9cfd015d5b906e6ff160cead7cb3b79b2523b2 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 20:06:14 +0200 Subject: [PATCH 061/185] Add test for optional layer metadata --- .../stats/mapnik_stats_layergroup.js | 170 +++++++++++++++++- 1 file changed, 169 insertions(+), 1 deletion(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index cdac7bb1..549f10c9 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -49,7 +49,7 @@ describe('Create mapnik layergroup', function() { sql: [ 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', ' from test_table t1, test_table_2 t2', - ' where t1.cartodb_id = t2.cartodb_id;' + ' where t1.cartodb_id = t2.cartodb_id' ].join(''), cartocss_version: cartocssVersion, cartocss: cartocss @@ -276,4 +276,172 @@ describe('Create mapnik layergroup', function() { testClient.drain(done); }); }); + + function layerWithMetadata(layer, metadata) { + return Object.assign(layer, { + options: Object.assign(layer.options, { metadata }) + }); + } + + it('should provide columns as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columns: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { type: 'number' }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { type: 'string' } + }; + assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns); + testClient.drain(done); + }); + }); + + it('should provide column stats as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { + type: 'number', + avg: 3, + max: 5, + min: 1, + sum: 15 + }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { + type: 'string', + categories: [ + { + category: "Calle de la Palma 72, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle de Pérez Galdós 9, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle Divino Pastor 12, Madrid, Spain", + frequency: 1 + }, + { + category: "Manuel Fernández y González 8, Madrid, Spain", + frequency: 1 + }, + { + category: "Plaza Conde de Toreno 2, Madrid, Spain", + frequency: 1 + } + ] + } + }; + assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns); + testClient.drain(done); + }); + }); + + it('should provide row count as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + testClient.drain(done); + }); + }); + + it('should provide geometry type as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + testClient.drain(done); + }); + }); + + it('should provide a sample as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + sample: 3 + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'address', 'the_geom', 'the_geom_webmercator' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + it('should only provide requested optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true, + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined); + assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined); + testClient.drain(done); + }); + }); }); From 741bcd1a8092cb40179f4a63c63a25e00da39ddb Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 20:07:20 +0200 Subject: [PATCH 062/185] Metadata fixes --- .../layer-stats/mapnik-layer-stats.js | 122 +++++++++++++----- lib/cartodb/utils/query-utils.js | 14 +- 2 files changed, 97 insertions(+), 39 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index a86f49be..1f7fce76 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -35,6 +35,7 @@ function columnAggregations(field) { if (field.type === 'date') { // TODO other types too? return ['min', 'max']; } + return []; } function firstPhaseQueries(queries, ctx) { @@ -58,8 +59,9 @@ function firstPhaseQueries(queries, ctx) { // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query queries.task( queryPromise( + ctx.dbConnection, queryUtils.getQueryActualRowCount(ctx.rawQuery), - function(err, res) { + (err, res) => { if (err) { queries.results.featureCount = -1; } else { @@ -74,12 +76,16 @@ function firstPhaseQueries(queries, ctx) { if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); queries.task( - queryPromise(queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn), function(err, res) { - if (!err) { - queries.results.geometryType = res.geom_type; + queryPromise( + ctx.dbConnection, + queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn), + (err, res) => { + if (!err) { + queries.results.geometryType = res.rows[0].geom_type; + } + return err; } - return err; - }) + ) ); } @@ -87,12 +93,16 @@ function firstPhaseQueries(queries, ctx) { queries.task( // TODO: note we have getLayerColumns in aggregation mapconfig. // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters - queryPromise(queryUtils.getQueryLimited(ctx.rawQuery, 0), function(err, res) { - if (!err) { - queries.results.columns = res.fields; + queryPromise( + ctx.dbConnection, + queryUtils.getQueryLimited(ctx.rawQuery, 0), + (err, res) => { + if (!err) { + queries.results.columns = formatResultFields(ctx.dbConnection, res.fields); + } + return err; } - return err; - }) + ) ); } } @@ -105,8 +115,9 @@ function secondPhaseQueries(queries, ctx) { const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); queries.task( queryPromise( + ctx.dbConnection, queryUtils.getQuerySample(ctx.rawQuery, sampleProb), - function(err, res) { + (err, res) => { if (err) { queries.results.sample = []; } else { @@ -121,38 +132,90 @@ function secondPhaseQueries(queries, ctx) { if (ctx.metaOptions.columnStats) { let aggr = []; Object.keys(queries.results.columns).forEach(name => { - aggr = aggr.concat(columnAggregations(queries.results.columns[name]) - .map(fn => `${fn}(${name}) AS ${name}_${fn}`)); + aggr = aggr.concat( + columnAggregations(queries.results.columns[name]) + .map(fn => `${fn}(${name}) AS ${name}_${fn}`) + ); if (queries.results.columns[name].type === 'string') { const topN = ctx.metaOptions.columnStats.topCategories || 1024; // TODO: ctx.metaOptions.columnStats.maxCategories // => use PG stats to dismiss columns with more distinct values queries.task( - queryPromise(queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN), function(err, res){ - if (!err) { - queries.results.columns[name].categories = res.rows; + queryPromise( + ctx.dbConnection, + queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN), + (err, res) => { + if (!err) { + queries.results.columns[name].categories = res.rows; + } + return err; } - return err; - }) + ) ); } }); queries.task( - queryPromise(`SELECT ${aggr.join(',')} FROM (${ctx.rawQuery})`, function(err, res){ - if (!err) { - Object.keys(queries.results.columns).forEach(name => { - columnAggregations(queries.results.columns[name]).forEach(fn => { - queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`]; + queryPromise( + ctx.dbConnection, + `SELECT ${aggr.join(',')} FROM (${ctx.rawQuery}) AS __cdb_query`, + (err, res) => { + if (!err) { + Object.keys(queries.results.columns).forEach(name => { + columnAggregations(queries.results.columns[name]).forEach(fn => { + queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`]; + }); }); - }); + } + return err; } - return err; - }) + ) ); } } +// This is adapted from SQL API: +function fieldType(cname) { + let tname; + switch (true) { + case /bool/.test(cname): + tname = 'boolean'; + break; + case /int|float|numeric/.test(cname): + tname = 'number'; + break; + case /text|char|unknown/.test(cname): + tname = 'string'; + break; + case /date|time/.test(cname): + tname = 'date'; + break; + default: + tname = cname; + } + if ( tname && cname.match(/^_/) ) { + tname += '[]'; + } + return tname; +} + +function formatResultFields(dbConnection, flds) { + flds = flds || []; + var nfields = {}; + for (var i=0; i firstPhaseQueries(queries, context)); diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index 238a905d..c7cb131d 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -21,10 +21,15 @@ module.exports.extractTableNames = function extractTableNames(query) { ].join(''); }; +module.exports.getQueryActualRowCount = function (query) { + return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; +}; + function getQueryRowEstimation(query) { return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows'; } -module.exports.getQueryRowCount = getQueryRowEstimation; + +module.exports.getQueryRowEstimation = getQueryRowEstimation; module.exports.getAggregationMetadata = ctx => ` WITH @@ -83,11 +88,6 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull `; }; -module.exports.getQueryActualRowCount = function (query) { - return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; -}; - - module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { const table = simpleQueryTable(query); if (table) { @@ -99,7 +99,7 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { ) SELECT * FROM (${query}) AS __cdb_query - WHERE random() < $ + WHERE random() < ${sampleProb} `; }; From eea7bed2f37d06dad5393e6b123bf8110fd28ac3 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 May 2018 20:41:42 +0200 Subject: [PATCH 063/185] Slightly more elegant results of queries --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 5 ++++- lib/cartodb/utils/phased-execution.js | 9 ++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 1f7fce76..b590f979 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -235,7 +235,10 @@ function (layer, dbConnection, callback) { // to define the queries of the second phase queries.phase(() => firstPhaseQueries(queries, context)); queries.phase(() => secondPhaseQueries(queries, context)); - queries.run(results => callback(null, results)).catch(error => callback(error)); + queries.run() + .then(results => callback(null, results)) + .catch(error => callback(error)); + }; module.exports = MapnikLayerStats; diff --git a/lib/cartodb/utils/phased-execution.js b/lib/cartodb/utils/phased-execution.js index c9ac445d..46dfbb39 100644 --- a/lib/cartodb/utils/phased-execution.js +++ b/lib/cartodb/utils/phased-execution.js @@ -65,7 +65,7 @@ * })); * }); * // Execute all tasks - * p.run((results) => { + * p.run().then((results) => { * console.log("RESULTS:", results); * }).catch((err) => { * console.log("ERROR:", error); @@ -82,14 +82,13 @@ module.exports = class PhasedExecution { task(promise) { this.tasks.push(promise); } - run(callback) { + run() { this.tasks = []; let phase = this.phases.shift(); if (phase) { phase(this); - return Promise.all(this.tasks) - .then(() => this.run()) - .then(() => { if (callback) { callback(this.results); } }); + return Promise.all(this.tasks).then(() => this.run()); } + return this.results; } }; From d8ef8cb12fd66673d973709c8d3ee23f1b5952d0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 May 2018 11:08:02 +0200 Subject: [PATCH 064/185] Debug travis test failures --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index b590f979..ffef2423 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -236,7 +236,7 @@ function (layer, dbConnection, callback) { queries.phase(() => firstPhaseQueries(queries, context)); queries.phase(() => secondPhaseQueries(queries, context)); queries.run() - .then(results => callback(null, results)) + .then(() => callback(null, queries.results)) .catch(error => callback(error)); }; From 944ce80c1efeb92de759665657fd1d62b64def7a Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 May 2018 11:42:53 +0200 Subject: [PATCH 065/185] Revert debugging change --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index ffef2423..b590f979 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -236,7 +236,7 @@ function (layer, dbConnection, callback) { queries.phase(() => firstPhaseQueries(queries, context)); queries.phase(() => secondPhaseQueries(queries, context)); queries.run() - .then(() => callback(null, queries.results)) + .then(results => callback(null, results)) .catch(error => callback(error)); }; From d706d0eb222ca2d6d6027a27714750199666fab0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 May 2018 11:44:49 +0200 Subject: [PATCH 066/185] More travis debugging through commits --- test/acceptance/stats/mapnik_stats_layergroup.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 549f10c9..3068c969 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -296,7 +296,6 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); const expectedColumns = { cartodb_id: { type: 'number' }, the_geom: { type: 'geometry' }, @@ -321,7 +320,6 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); const expectedColumns = { cartodb_id: { type: 'number', @@ -376,7 +374,6 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); testClient.drain(done); }); From fff5b3d85a4847506fb6f8a458ac8925ddedcfa3 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 May 2018 11:59:24 +0200 Subject: [PATCH 067/185] Revert debugging changes --- test/acceptance/stats/mapnik_stats_layergroup.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 3068c969..549f10c9 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -296,6 +296,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); const expectedColumns = { cartodb_id: { type: 'number' }, the_geom: { type: 'geometry' }, @@ -320,6 +321,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); const expectedColumns = { cartodb_id: { type: 'number', @@ -374,6 +376,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); testClient.drain(done); }); From ee7bd5fb8a171b789c4d701e6d219e6d092a5763 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 May 2018 12:42:42 +0200 Subject: [PATCH 068/185] Fix tests --- test/acceptance/stats/mapnik_stats_layergroup.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 549f10c9..f5030783 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -273,6 +273,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); assert.equal(layergroup.metadata.layers[2].type, 'http'); + global.environment.enabledFeatures.layerStats = true; testClient.drain(done); }); }); From 9feea665505eaeda50feea9246a1458286e06ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 May 2018 14:59:21 +0200 Subject: [PATCH 069/185] Use routes configuration to create and mount routes and controllers --- config/environments/development.js.example | 28 +++++++++--- config/environments/production.js.example | 28 +++++++++--- config/environments/staging.js.example | 28 +++++++++--- config/environments/test.js.example | 48 +++++++++++++++++++++ lib/cartodb/api/api-router.js | 44 ++++++++++--------- lib/cartodb/api/map/map-router.js | 10 ++--- lib/cartodb/api/template/template-router.js | 10 ++--- lib/cartodb/server_options.js | 33 ++++++++++++-- 8 files changed, 171 insertions(+), 58 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index e11b2a28..f9c58abb 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -32,28 +32,42 @@ var config = { // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API ,routes: { - api: { - // "/tiles" is for compatibility with versions up to 1.6.x + v1: { paths: [ '/api/v1', '/user/:user/api/v1', - '/tiles' // for compatibility with versions up to 1.6.x ], // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x + // "/api/v1/map" is the new API, map: { paths: [ '/map', - '/layergroup' ] }, // Base url for the Templated Maps API // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API // "/tiles/template" is for compatibility with versions up to 1.6.x template: { paths: [ - '/map/named', '/template' ] } diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 24eb18fd..89c5e5bd 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -32,28 +32,42 @@ var config = { // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API ,routes: { - api: { - // "/tiles" is for compatibility with versions up to 1.6.x + v1: { paths: [ '/api/v1', '/user/:user/api/v1', - '/tiles' // for compatibility with versions up to 1.6.x ], // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x + // "/api/v1/map" is the new API, map: { paths: [ '/map', - '/layergroup' ] }, // Base url for the Templated Maps API // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API // "/tiles/template" is for compatibility with versions up to 1.6.x template: { paths: [ - '/map/named', '/template' ] } diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 28c66185..76353f9b 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -32,28 +32,42 @@ var config = { // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API ,routes: { - api: { - // "/tiles" is for compatibility with versions up to 1.6.x + v1: { paths: [ '/api/v1', '/user/:user/api/v1', - '/tiles' // for compatibility with versions up to 1.6.x ], // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x + // "/api/v1/map" is the new API, map: { paths: [ '/map', - '/layergroup' ] }, // Base url for the Templated Maps API // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API // "/tiles/template" is for compatibility with versions up to 1.6.x template: { paths: [ - '/map/named', '/template' ] } diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 4d2c2d27..1f4e34a6 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -13,6 +13,8 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '(.*)' + // DEPRECATED: use routes property instead + // --------------------------------------- // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API @@ -26,6 +28,52 @@ var config = { // "tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Base URLs for the APIs + // + // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API + ,routes: { + v1: { + paths: [ + '/api/v1', + '/user/:user/api/v1', + ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, + map: { + paths: [ + '/map', + ] + }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, + template: { + paths: [ + '/map/named' + ] + } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } + } + } + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index ecffe562..5165007d 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -177,8 +177,8 @@ module.exports = class ApiRouter { tablesExtentBackend }; - this.mapRouter = new MapRouter({ collaborators, serverOptions }); - this.templateRouter = new TemplateRouter({ collaborators, serverOptions }); + this.mapRouter = new MapRouter({ collaborators }); + this.templateRouter = new TemplateRouter({ collaborators }); } register (app) { @@ -187,29 +187,33 @@ module.exports = class ApiRouter { app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache; } - const apiRouter = router(); + Object.keys(this.serverOptions.routes).forEach(apiVersion => { + const routes = this.serverOptions.routes[apiVersion]; - apiRouter.use(logger(this.serverOptions)); - apiRouter.use(bodyParser.json()); - apiRouter.use(servedByHostHeader()); - apiRouter.use(stats({ - enabled: this.serverOptions.useProfiler, - statsClient: global.statsClient - })); - apiRouter.use(lzmaMiddleware()); - apiRouter.use(cors()); - apiRouter.use(user()); + const apiRouter = router(); - this.templateRouter.register(apiRouter); - this.mapRouter.register(apiRouter); + apiRouter.use(logger(this.serverOptions)); + apiRouter.use(bodyParser.json()); + apiRouter.use(servedByHostHeader()); + apiRouter.use(stats({ + enabled: this.serverOptions.useProfiler, + statsClient: global.statsClient + })); + apiRouter.use(lzmaMiddleware()); + apiRouter.use(cors()); + apiRouter.use(user()); - apiRouter.use(sendResponse()); - apiRouter.use(syntaxError()); - apiRouter.use(errorMiddleware()); + this.templateRouter.register(apiRouter, routes.template.paths); + this.mapRouter.register(apiRouter, routes.map.paths); - const paths = this.serverOptions.routes.api.paths; + apiRouter.use(sendResponse()); + apiRouter.use(syntaxError()); + apiRouter.use(errorMiddleware()); - app.use(`(?:${paths.join('|')})`, apiRouter); + const apiPaths = routes.paths; + + apiPaths.forEach(path => app.use(path, apiRouter)); + }); } }; diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index a3c2bbaa..03f19937 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -10,9 +10,7 @@ const PreviewTemplateController = require('./preview-template-controller'); const AnalysesCatalogController = require('./analyses-catalog-controller'); module.exports = class MapRouter { - constructor ({ collaborators, serverOptions }) { - this.serverOptions = serverOptions; - + constructor ({ collaborators }) { const { analysisStatusBackend, attributesBackend, @@ -114,7 +112,7 @@ module.exports = class MapRouter { ); } - register (apiRouter) { + register (apiRouter, mapPaths) { const mapRouter = router(); this.analysisLayergroupController.register(mapRouter); @@ -126,8 +124,6 @@ module.exports = class MapRouter { this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); - const paths = this.serverOptions.routes.api.map.paths; - - apiRouter.use(`(?:${paths.join('|')})`, mapRouter); + mapPaths.forEach(path => apiRouter.use(path, mapRouter)); } }; diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js index cfb883e7..9ddcf9fc 100644 --- a/lib/cartodb/api/template/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -5,9 +5,7 @@ const AdminTemplateController = require('./admin-template-controller'); const TileTemplateController = require('./tile-template-controller'); module.exports = class TemplateRouter { - constructor ({ collaborators, serverOptions }) { - this.serverOptions = serverOptions; - + constructor ({ collaborators }) { const { pgConnection, templateMaps, @@ -54,15 +52,13 @@ module.exports = class TemplateRouter { ); } - register (apiRouter) { + register (apiRouter, templatePaths) { const templateRouter = router(); this.namedMapController.register(templateRouter); this.tileTemplateController.register(templateRouter); this.adminTemplateController.register(templateRouter); - const paths = this.serverOptions.routes.api.template.paths; - - apiRouter.use(`(?:${paths.join('|')})`, templateRouter); + templatePaths.forEach(path => apiRouter.use(path, templateRouter)); } }; diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index d05e5e1d..c3d0fdb8 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -67,22 +67,49 @@ module.exports = { // FIXME: Remove it. This is no longer needed, paths are defined in routers base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)', + // Base URLs for the APIs + // + // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API routes: global.environment.routes || { - api: { + v1: { paths: [ '/api/v1', - '/user/:user/api/v1' + '/user/:user/api/v1', ], + // Base url for the Detached Maps API + // "/api/v1/map" is the new API, map: { paths: [ - '/map' + '/map', ] }, + // Base url for the Templated Maps API + // "/api/v1/map/named" is the new API, template: { paths: [ '/map/named' ] } + }, + // For compatibility with versions up to 1.6.x + v0: { + paths: [ + '/tiles' + ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x + map: { + paths: [ + '/layergroup' + ] + }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x + template: { + paths: [ + '/template' + ] + } } }, From c51e2542870055965016ea7cfe2e60fb055dd83f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 May 2018 15:00:18 +0200 Subject: [PATCH 070/185] Set 404 as defautl status code and set the proper status code fir the response at very same time that the response body --- lib/cartodb/api/api-router.js | 2 ++ lib/cartodb/api/map/analyses-catalog-controller.js | 1 + lib/cartodb/api/map/analysis-layergroup-controller.js | 1 + lib/cartodb/api/map/anonymous-map-controller.js | 1 + lib/cartodb/api/map/attributes-layergroup-controller.js | 1 + lib/cartodb/api/map/dataview-layergroup-controller.js | 2 ++ lib/cartodb/api/map/preview-layergroup-controller.js | 2 ++ lib/cartodb/api/map/preview-template-controller.js | 2 ++ lib/cartodb/api/middlewares/initialize-status-code.js | 6 ++++++ lib/cartodb/api/middlewares/send-response.js | 2 +- lib/cartodb/api/template/admin-template-controller.js | 4 ++++ lib/cartodb/api/template/named-template-controller.js | 1 + lib/cartodb/api/template/tile-template-controller.js | 1 + 13 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 lib/cartodb/api/middlewares/initialize-status-code.js diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 5165007d..9d53285b 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -40,6 +40,7 @@ const ResourceLocator = require('../models/resource-locator'); const LayergroupMetadata = require('../utils/layergroup-metadata'); const RendererStatsReporter = require('../stats/reporter/renderer'); +const initializeStatusCode = require('./middlewares/initialize-status-code'); const logger = require('./middlewares/logger'); const bodyParser = require('body-parser'); const servedByHostHeader = require('./middlewares/served-by-host-header'); @@ -193,6 +194,7 @@ module.exports = class ApiRouter { const apiRouter = router(); apiRouter.use(logger(this.serverOptions)); + apiRouter.use(initializeStatusCode()); apiRouter.use(bodyParser.json()); apiRouter.use(servedByHostHeader()); apiRouter.use(stats({ diff --git a/lib/cartodb/api/map/analyses-catalog-controller.js b/lib/cartodb/api/map/analyses-catalog-controller.js index 91defc21..760c0e80 100644 --- a/lib/cartodb/api/map/analyses-catalog-controller.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -99,6 +99,7 @@ function prepareResponse () { return -1; }); + res.statusCode = 200; res.body = { catalog: analysisCatalog }; next(); diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js index 0e46dc81..96f46d70 100644 --- a/lib/cartodb/api/map/analysis-layergroup-controller.js +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -48,6 +48,7 @@ function analysisNodeStatus (analysisStatusBackend) { 'Last-Modified': new Date().toUTCString() }); + res.statusCode = 200; res.body = nodeStatus; next(); diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js index 964ffacd..21049437 100644 --- a/lib/cartodb/api/map/anonymous-map-controller.js +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -202,6 +202,7 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected return next(err); } + res.statusCode = 200; res.body = layergroup; res.locals.mapConfigProvider = mapConfigProvider; diff --git a/lib/cartodb/api/map/attributes-layergroup-controller.js b/lib/cartodb/api/map/attributes-layergroup-controller.js index ef9614fc..632cca9f 100644 --- a/lib/cartodb/api/map/attributes-layergroup-controller.js +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -77,6 +77,7 @@ function getFeatureAttributes (attributesBackend) { return next(err); } + res.statusCode = 200; res.body = tile; next(); diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js index 0b6f8cc9..710c5cd4 100644 --- a/lib/cartodb/api/map/dataview-layergroup-controller.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -151,6 +151,7 @@ function getDataview (dataviewBackend) { return next(err); } + res.statusCode = 200; res.body = dataview; next(); @@ -174,6 +175,7 @@ function dataviewSearch (dataviewBackend) { return next(err); } + res.statusCode = 200; res.body = searchResult; next(); diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js index 71dfdffd..29db8729 100644 --- a/lib/cartodb/api/map/preview-layergroup-controller.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -109,6 +109,7 @@ function getPreviewImageByCenter (previewBackend) { res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + res.statusCode = 200; res.body = image; next(); @@ -144,6 +145,7 @@ function getPreviewImageByBoundingBox (previewBackend) { res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + res.statusCode = 200; res.body = image; next(); diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index e19942d2..9f5257a3 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -268,6 +268,7 @@ function getImage({ previewBackend, label }) { res.set(headers); } + res.statusCode = 200; res.body = image; next(); @@ -287,6 +288,7 @@ function getImage({ previewBackend, label }) { res.set(headers); } + res.statusCode = 200; res.body = image; next(); diff --git a/lib/cartodb/api/middlewares/initialize-status-code.js b/lib/cartodb/api/middlewares/initialize-status-code.js new file mode 100644 index 00000000..9e6da2c7 --- /dev/null +++ b/lib/cartodb/api/middlewares/initialize-status-code.js @@ -0,0 +1,6 @@ +module.exports = function initializeStatusCode () { + return function initializeStatusCodeMiddleware (req, res, next) { + res.statusCode = 404; + next(); + }; +}; diff --git a/lib/cartodb/api/middlewares/send-response.js b/lib/cartodb/api/middlewares/send-response.js index 469cf0a7..ec13c5bf 100644 --- a/lib/cartodb/api/middlewares/send-response.js +++ b/lib/cartodb/api/middlewares/send-response.js @@ -2,7 +2,7 @@ module.exports = function sendResponse () { return function sendResponseMiddleware (req, res) { req.profiler.done('res'); - res.status(res.statusCode || 200); + res.status(res.statusCode); if (Buffer.isBuffer(res.body)) { return res.send(res.body); diff --git a/lib/cartodb/api/template/admin-template-controller.js b/lib/cartodb/api/template/admin-template-controller.js index 4cceda69..b45ebca7 100644 --- a/lib/cartodb/api/template/admin-template-controller.js +++ b/lib/cartodb/api/template/admin-template-controller.js @@ -115,6 +115,7 @@ function createTemplate ({ templateMaps }) { return next(err); } + res.statusCode = 200; res.body = { template_id: templateId }; next(); @@ -133,6 +134,7 @@ function updateTemplate ({ templateMaps }) { return next(err); } + res.statusCode = 200; res.body = { template_id: templateId }; next(); @@ -161,6 +163,7 @@ function retrieveTemplate ({ templateMaps }) { // so we remove it before returning to the user delete template.auth_id; + res.statusCode = 200; res.body = { template }; next(); @@ -199,6 +202,7 @@ function listTemplates ({ templateMaps }) { return next(err); } + res.statusCode = 200; res.body = { template_ids: templateIds }; next(); diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js index eed39b34..86613c5e 100644 --- a/lib/cartodb/api/template/named-template-controller.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -204,6 +204,7 @@ function instantiateLayergroup (mapBackend, userLimitsBackend, pgConnection, aff return next(err); } + res.statusCode = 200; res.body = layergroup; const { mapConfigProvider } = res.locals; diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js index 5cb2f4e4..de256ccb 100644 --- a/lib/cartodb/api/template/tile-template-controller.js +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -75,6 +75,7 @@ function getTile ({ tileBackend, label }) { res.set(headers); } + res.statusCode = 200; res.body = tile; next(); From 976ee35a35592bec278cf922c17d671e45161f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 May 2018 15:24:33 +0200 Subject: [PATCH 071/185] Use new routes configuration schema in ported test --- test/acceptance/ported/support/ported_server_options.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index d850b1c9..243875a0 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -13,16 +13,20 @@ module.exports = _.extend({}, serverOptions, { base_url_mapconfig: '/database/:dbname/layergroup', routes: { - api: { + v0: { paths: [ '/tiles', '/database/:dbname' ], + // Base url for the Detached Maps API + // "/tiles/layergroup" is for compatibility with versions up to 1.6.x map: { paths: [ '/layergroup' ] }, + // Base url for the Templated Maps API + // "/tiles/template" is for compatibility with versions up to 1.6.x template: { paths: [ '/template' @@ -31,6 +35,7 @@ module.exports = _.extend({}, serverOptions, { } }, + grainstore: { datasource: { geometry_field: 'the_geom', From f7745928abaa01d2ac5500541e1f696ece40556d Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 May 2018 15:42:41 +0200 Subject: [PATCH 072/185] Fix tests eliminate dependency on the order of PostgreSQL results --- .../stats/mapnik_stats_layergroup.js | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index f5030783..508878cf 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -319,6 +319,32 @@ describe('Create mapnik layergroup', function() { ] }); + // metadata categories are ordered only partially by descending frequency; + // this orders them completely to avoid ambiguities when comparing + function withSortedCategories(columns) { + function catOrder(a, b) { + if (a.frequency !== b.frequency) { + return b.frequency - a.frequency; + } + if (a.category < b.category) { + return -1; + } + if (a.category > b.category) { + return +1; + } + return 0; + } + let sorted = {}; + Object.keys(columns).forEach(name => { + let data = columns[name]; + if (data.hasOwnProperty('categories')) { + data = Object.assign(data, { categories: data.categories.sort(catOrder)}); + } + sorted[name] = data; + }); + return sorted; + } + testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); @@ -359,7 +385,10 @@ describe('Create mapnik layergroup', function() { ] } }; - assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns); + assert.deepEqual( + withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns), + withSortedCategories(expectedColumns) + ); testClient.drain(done); }); }); From 0741881959a5835bb9a7f89a7035d91fbf542f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 9 May 2018 16:24:38 +0200 Subject: [PATCH 073/185] Do not initialize status code to 404 when method is OPTIONS --- lib/cartodb/api/middlewares/initialize-status-code.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/api/middlewares/initialize-status-code.js b/lib/cartodb/api/middlewares/initialize-status-code.js index 9e6da2c7..ed6b1805 100644 --- a/lib/cartodb/api/middlewares/initialize-status-code.js +++ b/lib/cartodb/api/middlewares/initialize-status-code.js @@ -1,6 +1,9 @@ module.exports = function initializeStatusCode () { return function initializeStatusCodeMiddleware (req, res, next) { - res.statusCode = 404; + if (req.method !== 'OPTIONS') { + res.statusCode = 404; + } + next(); }; }; From 913b29070ff0ac9626d1e1951619f2a08c1cd8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 10 May 2018 15:59:38 +0200 Subject: [PATCH 074/185] tests raising pg_typeof error --- test/acceptance/dataviews/aggregation.js | 2 +- test/acceptance/dataviews/error-cases.js | 225 ++++++++++++++++------- 2 files changed, 164 insertions(+), 63 deletions(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 204fcac7..9330b36f 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -491,4 +491,4 @@ describe('Count aggregation', function () { this.testClient.drain(done); }); }); -}); \ No newline at end of file +}); diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index 3e6cc2a8..933211b4 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -3,8 +3,8 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); -describe('histogram-dataview', function() { - +describe('dataview error cases', function() { + afterEach(function(done) { if (this.testClient) { this.testClient.drain(done); @@ -13,75 +13,176 @@ describe('histogram-dataview', function() { } }); - var ERROR_RESPONSE = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - function createMapConfig(dataviews) { - return { - version: '1.5.0', - layers: [ - { - "type": "cartodb", - "options": { - "source": { - "id": "HEAD" - }, - "cartocss": "#points { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0" - } - } - ], - dataviews: dataviews, - analyses: [ - { - "id": "HEAD", - "type": "source", - "params": { - "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x" - } - } - ] + describe('generic errors', function() { + var ERROR_RESPONSE = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } }; - } - - it('should fail when invalid dataviews object is provided, string case', function(done) { - var mapConfig = createMapConfig("wadus-string"); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { - assert.ok(!err, err); - - assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); - - done(); + + function createMapConfig(dataviews) { + return { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + dataviews: dataviews, + analyses: [ + { + "id": "HEAD", + "type": "source", + "params": { + "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x" + } + } + ] + }; + } + + it('should fail when invalid dataviews object is provided, string case', function(done) { + var mapConfig = createMapConfig("wadus-string"); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { + assert.ok(!err, err); + + assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); + + done(); + }); + }); + + it('should fail when invalid dataviews object is provided, array case', function(done) { + var mapConfig = createMapConfig([]); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { + assert.ok(!err, err); + + assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); + + done(); + }); + }); + + it('should work with empty but valid objects', function(done) { + var mapConfig = createMapConfig({}); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err, err); + + assert.ok(layergroup); + assert.ok(layergroup.layergroupid); + + done(); + }); }); }); - it('should fail when invalid dataviews object is provided, array case', function(done) { - var mapConfig = createMapConfig([]); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { - assert.ok(!err, err); + describe('pg_typeof', function() { - assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); - - done(); + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } }); - }); - it('should work with empty but valid objects', function(done) { - var mapConfig = createMapConfig({}); - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err, err); + function createMapConfig(query) { + query = query || 'select * from populated_places_simple_reduced'; - assert.ok(layergroup); - assert.ok(layergroup.layergroupid); + return { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: query, + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', + cartocss_version: '2.0.1', + source: { id: "a0" } + } + } + ], + analyses: [{ + id: "a0", + type: "source", + params: { + query: query + } + }], + dataviews: { + aggregation_count_dataview: { + type: "aggregation", + source: { id: "a0" }, + options: { + column: "adm0name", + aggregation: "count", + aggregationColumn: "adm0name" + } + }, - done(); + formula_count_dataview: { + type: 'formula', + source: { id: "a0" }, + options: { + column: 'adm0_a3', + operation: 'count' + } + }, + } + }; + } + + it('should work without filters', function(done) { + this.testClient = new TestClient(createMapConfig()); + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { + if (err) { + return done(err); + } + + done(); + }); }); + + it('should work with filters', function(done) { + var params = { + filters: { + dataviews: {aggregation_count_dataview: {accept: ['Canada']}} + } + }; + + this.testClient = new TestClient(createMapConfig()); + this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) { + if (err) { + return done(err); + } + + done(); + }); + }); + + it('should return an error if query row equals to 0', function(done) { + const query = 'select * from populated_places_simple_reduced limit 0'; + this.testClient = new TestClient(createMapConfig(query)); + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { + if (err) { + return done(err); + } + + done(); + }); + }); + + }); }); From ed837fbf227e9b2bdce2d2dbced02e24c0c59254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 10 May 2018 17:25:02 +0200 Subject: [PATCH 075/185] improving tests raising the pg_typeof error --- test/acceptance/dataviews/error-cases.js | 42 +++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index 933211b4..eb85dab8 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -146,10 +146,7 @@ describe('dataview error cases', function() { it('should work without filters', function(done) { this.testClient = new TestClient(createMapConfig()); this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { - if (err) { - return done(err); - } - + assert.ifError(err); done(); }); }); @@ -163,22 +160,43 @@ describe('dataview error cases', function() { this.testClient = new TestClient(createMapConfig()); this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) { - if (err) { - return done(err); - } - + assert.ifError(err); done(); }); }); + it('should return an error if the column used by dataview does not exist', function(done) { + const query = 'select cartodb_id, the_geom, the_geom_webmercator from populated_places_simple_reduced'; + + const expectedResponse = { + status: 404, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + const expectedResponseBody = { + errors:['column "adm0name" does not exist'], + errors_with_context:[{ + type:'unknown', + message:'column "adm0name" does not exist' + }] + }; + + this.testClient = new TestClient(createMapConfig(query)); + this.testClient.getDataview('aggregation_count_dataview', { response: expectedResponse }, function(err, result) { + assert.ifError(err); + assert.deepEqual(result, expectedResponseBody); + done(); + }); + }); + it('should return an error if query row equals to 0', function(done) { const query = 'select * from populated_places_simple_reduced limit 0'; + this.testClient = new TestClient(createMapConfig(query)); this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { - if (err) { - return done(err); - } - + assert.ifError(err); done(); }); }); From 37a2e89c81f0817ddfffe9c27434578f8db409c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 10 May 2018 18:34:18 +0200 Subject: [PATCH 076/185] testing expected result with pg_typeof undefined --- test/acceptance/dataviews/error-cases.js | 43 ++++++++++++++---------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index eb85dab8..e360d51f 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -129,23 +129,14 @@ describe('dataview error cases', function() { aggregation: "count", aggregationColumn: "adm0name" } - }, - - formula_count_dataview: { - type: 'formula', - source: { id: "a0" }, - options: { - column: 'adm0_a3', - operation: 'count' - } - }, + } } }; } it('should work without filters', function(done) { this.testClient = new TestClient(createMapConfig()); - this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { + this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err) { assert.ifError(err); done(); }); @@ -159,7 +150,7 @@ describe('dataview error cases', function() { }; this.testClient = new TestClient(createMapConfig()); - this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) { + this.testClient.getDataview('aggregation_count_dataview', params, function(err) { assert.ifError(err); done(); }); @@ -168,10 +159,12 @@ describe('dataview error cases', function() { it('should return an error if the column used by dataview does not exist', function(done) { const query = 'select cartodb_id, the_geom, the_geom_webmercator from populated_places_simple_reduced'; - const expectedResponse = { - status: 404, - headers: { - 'Content-Type': 'application/json; charset=utf-8' + const params = { + response: { + status: 404, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } } }; @@ -184,19 +177,33 @@ describe('dataview error cases', function() { }; this.testClient = new TestClient(createMapConfig(query)); - this.testClient.getDataview('aggregation_count_dataview', { response: expectedResponse }, function(err, result) { + this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) { assert.ifError(err); assert.deepEqual(result, expectedResponseBody); done(); }); }); - it('should return an error if query row equals to 0', function(done) { + it('should not fail if query does not return rows', function(done) { const query = 'select * from populated_places_simple_reduced limit 0'; + const expectedResponseBody = { + aggregation: 'count', + count: 0, + nulls: 0, + nans: 0, + infinities: 0, + min: 0, + max: 0, + categoriesCount: 0, + categories: [], + type: 'aggregation' + }; + this.testClient = new TestClient(createMapConfig(query)); this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) { assert.ifError(err); + assert.deepEqual(result, expectedResponseBody); done(); }); }); From 99516f5a75eb78dc349f7fd2c44ab5515ce184bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 10 May 2018 18:35:52 +0200 Subject: [PATCH 077/185] fix tests and jshint happy --- test/acceptance/dataviews/error-cases.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index e360d51f..78acba5d 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -5,15 +5,16 @@ var TestClient = require('../../support/test-client'); describe('dataview error cases', function() { - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - describe('generic errors', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + var ERROR_RESPONSE = { status: 400, headers: { @@ -208,6 +209,5 @@ describe('dataview error cases', function() { }); }); - }); }); From eba68c56efedb5ac9b7479590fde8577b5d2a205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 10 May 2018 18:36:08 +0200 Subject: [PATCH 078/185] handling pg_typeof undefined --- lib/cartodb/models/dataview/base.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 8c1ada5d..80e021b1 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -52,6 +52,11 @@ module.exports = class BaseDataview { if (err) { return callback(err); } + + if (!result || !result.rows || !result.rows.length) { + return callback(new Error('The column type could not be determined')); + } + const pgType = result.rows[0].pg_typeof; callback(null, getPGTypeName(pgType)); }, readOnlyTransaction); From cae4dd81c9b370e70b42521e752196681bd84d02 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 10 May 2018 19:12:47 +0200 Subject: [PATCH 079/185] WIP: fix problems for aggregations & metadata --- .../layer-stats/mapnik-layer-stats.js | 32 +++++++++++++------ .../adapter/aggregation-mapconfig-adapter.js | 14 ++++++-- .../stats/mapnik_stats_layergroup.js | 2 ++ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index b590f979..ac62ac2b 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -9,6 +9,8 @@ function MapnikLayerStats () { }; } +// TODO: estimatedFeatureCount is post-aggregation; the rest is pre; distinguish and complement + MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; @@ -41,7 +43,7 @@ function columnAggregations(field) { function firstPhaseQueries(queries, ctx) { if (queries.results.estimatedFeatureCount === undefined) { queries.task( - queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.query), function(err, res) { + queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.aggrQuery), function(err, res) { if (err) { // at least for debugging we should err queries.results.estimatedFeatureCount = -1; @@ -56,11 +58,13 @@ function firstPhaseQueries(queries, ctx) { } if (ctx.metaOptions.featureCount) { + // TODO: pre/aggr + // TODO: for pre, use ctx.aggrMeta.pre_aggregation_count // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQueryActualRowCount(ctx.rawQuery), + queryUtils.getQueryActualRowCount(ctx.preQuery), (err, res) => { if (err) { queries.results.featureCount = -1; @@ -74,11 +78,13 @@ function firstPhaseQueries(queries, ctx) { } if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { + // TODO: pre/aggr + // TODO: for pre, use ctx.aggrMeta.geometry_type const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn), + queryUtils.getQueryGeometryType(ctx.preQuery, geometryColumn), (err, res) => { if (!err) { queries.results.geometryType = res.rows[0].geom_type; @@ -90,12 +96,15 @@ function firstPhaseQueries(queries, ctx) { } if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { + // TODO: pre/aggr + // TODO: for aggr, use layer.options.columns (will need to pass in ctx) + // note: post-aggregation columns are in layer.options.columns when aggregation is present queries.task( // TODO: note we have getLayerColumns in aggregation mapconfig. // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters queryPromise( ctx.dbConnection, - queryUtils.getQueryLimited(ctx.rawQuery, 0), + queryUtils.getQueryLimited(ctx.preQuery, 0), (err, res) => { if (!err) { queries.results.columns = formatResultFields(ctx.dbConnection, res.fields); @@ -116,7 +125,7 @@ function secondPhaseQueries(queries, ctx) { queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQuerySample(ctx.rawQuery, sampleProb), + queryUtils.getQuerySample(ctx.preQuery, sampleProb), (err, res) => { if (err) { queries.results.sample = []; @@ -130,6 +139,7 @@ function secondPhaseQueries(queries, ctx) { } if (ctx.metaOptions.columnStats) { + // TODO: pre/aggr let aggr = []; Object.keys(queries.results.columns).forEach(name => { aggr = aggr.concat( @@ -143,7 +153,7 @@ function secondPhaseQueries(queries, ctx) { queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN), + queryUtils.getQueryTopCategories(ctx.preQuery, name, topN), (err, res) => { if (!err) { queries.results.columns[name].categories = res.rows; @@ -157,7 +167,7 @@ function secondPhaseQueries(queries, ctx) { queries.task( queryPromise( ctx.dbConnection, - `SELECT ${aggr.join(',')} FROM (${ctx.rawQuery}) AS __cdb_query`, + `SELECT ${aggr.join(',')} FROM (${ctx.preQuery}) AS __cdb_query`, (err, res) => { if (!err) { Object.keys(queries.results.columns).forEach(name => { @@ -218,10 +228,14 @@ function formatResultFields(dbConnection, flds) { MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { + let aggrQuery = layer.options.sql_raw || layer.options.sql; + let preQuery = layer.options.aggregation_metadata ? layer.options.aggregation_metadata.pre_aggregation_sql : aggrQuery; + let context = { dbConnection, - query: layer.options.sql, - rawQuery: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql, + preQuery, + aggrQuery, + aggrMeta: layer.options.aggregation_metadata, metaOptions: layer.options.metadata || {} }; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 67c5eac4..8639b85b 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -73,7 +73,7 @@ module.exports = class AggregationMapConfigAdapter { if (adapted) { requestMapConfig.layers[index] = layer; } - const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); + const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); // <<- context.aggregation.layers.push(aggregatedFormats); }); @@ -84,7 +84,7 @@ module.exports = class AggregationMapConfigAdapter { _adaptLayer (connection, mapConfig, layer, index) { return new Promise((resolve, reject) => { - this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => { + this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt, aggrMeta) => { if (err) { return reject(err); } @@ -93,6 +93,7 @@ module.exports = class AggregationMapConfigAdapter { return resolve({ layer, index, adapted: shouldAdapt }); } + const sqlQuery = layer.options.sql; const sqlQueryWrap = layer.options.sql_wrap; let aggregationSql = mapConfig.getAggregatedQuery(index); @@ -110,6 +111,12 @@ module.exports = class AggregationMapConfigAdapter { layer.options.columns = columns; + layer.options.aggregation_metadata = { + pre_aggregation_sql: sqlQueryWrap || sqlQuery, + geometry_type: aggrMeta.type, + pre_aggregation_count: aggrMeta.count + }; + return resolve({ layer, index, adapted: shouldAdapt }); }); }); @@ -154,11 +161,12 @@ module.exports = class AggregationMapConfigAdapter { return callback(null, false); } - callback(null, true); + callback(null, true, result); }); } _getAggregationMetadata (mapConfig, layer, adapted) { + // also: pre-aggr query, columns, ... if (!adapted) { return { png: false, mvt: false }; } diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 508878cf..d8a9a06b 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -474,4 +474,6 @@ describe('Create mapnik layergroup', function() { testClient.drain(done); }); }); + + // TODO: add tests for metadata with aggregation }); From 72d8a26ede0b5fcd0f35252b81623b78a31f38f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 12:14:27 +0200 Subject: [PATCH 080/185] Use ES6 class --- .../api/map/anonymous-map-controller.js | 158 +++++++++--------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js index 21049437..b16e2c00 100644 --- a/lib/cartodb/api/map/anonymous-map-controller.js +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -22,88 +22,88 @@ const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provid const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -/** - * @param {AuthBackend} authBackend - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @param {MapBackend} mapBackend - * @param metadataBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsBackend} userLimitsBackend - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {MapConfigAdapter} mapConfigAdapter - * @param {StatsBackend} statsBackend - * @constructor - */ -function AnonymousMapController ( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsBackend, - layergroupAffectedTables, - mapConfigAdapter, - statsBackend, - authBackend, - layergroupMetadata -) { - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; - this.mapBackend = mapBackend; - this.metadataBackend = metadataBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsBackend = userLimitsBackend; - this.layergroupAffectedTables = layergroupAffectedTables; - this.mapConfigAdapter = mapConfigAdapter; - this.statsBackend = statsBackend; - this.authBackend = authBackend; - this.layergroupMetadata = layergroupMetadata; -} +module.exports = class AnonymousMapController { + /** + * @param {AuthBackend} authBackend + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsBackend} userLimitsBackend + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @constructor + */ + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTables = layergroupAffectedTables; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authBackend = authBackend; + this.layergroupMetadata = layergroupMetadata; + } -module.exports = AnonymousMapController; + register (mapRouter) { + mapRouter.options('/'); + mapRouter.get('/', this.composeCreateMapMiddleware()); + mapRouter.post('/', this.composeCreateMapMiddleware()); + } -AnonymousMapController.prototype.register = function (mapRouter) { - mapRouter.options('/'); - mapRouter.get('/', this.composeCreateMapMiddleware()); - mapRouter.post('/', this.composeCreateMapMiddleware()); -}; + composeCreateMapMiddleware () { + const isTemplateInstantiation = false; + const useTemplateHash = false; + const includeQuery = true; + const label = 'ANONYMOUS LAYERGROUP'; + const addContext = true; -AnonymousMapController.prototype.composeCreateMapMiddleware = function () { - const isTemplateInstantiation = false; - const useTemplateHash = false; - const includeQuery = true; - const label = 'ANONYMOUS LAYERGROUP'; - const addContext = true; - - return [ - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), - cleanUpQueryParams(['aggregation']), - initProfiler(isTemplateInstantiation), - checkJsonContentType(), - checkCreateLayergroup(), - prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup ( - this.mapBackend, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTables - ), - incrementMapViewCount(this.metadataBackend), - augmentLayergroupData(), - cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader({ now: true }), - lastUpdatedTimeLayergroup(), - layerStats(this.pgConnection, this.statsBackend), - layergroupIdHeader(this.templateMaps, useTemplateHash), - layergroupMetadata(this.layergroupMetadata, includeQuery), - mapError({ label, addContext }) - ]; + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkCreateLayergroup(), + prepareAdapterMapConfig(this.mapConfigAdapter), + createLayergroup ( + this.mapBackend, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), + lastUpdatedTimeLayergroup(), + layerStats(this.pgConnection, this.statsBackend), + layergroupIdHeader(this.templateMaps, useTemplateHash), + layergroupMetadata(this.layergroupMetadata, includeQuery), + mapError({ label, addContext }) + ]; + } }; function checkCreateLayergroup () { From d86a8392652ed8138347b53ce69e20229fb61d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 12:22:50 +0200 Subject: [PATCH 081/185] Blank line --- lib/cartodb/api/map/analysis-layergroup-controller.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js index 96f46d70..8a22dd09 100644 --- a/lib/cartodb/api/map/analysis-layergroup-controller.js +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -26,7 +26,6 @@ module.exports = class AnalysisLayergroupController { cleanUpQueryParams(), analysisNodeStatus(this.analysisStatusBackend) ); - } }; From 07dec2e64138760a12ccd0a1b80d99858230a4d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 12:38:52 +0200 Subject: [PATCH 082/185] Create .middlwares() to return a set of middlewares to be mounted --- lib/cartodb/api/map/analysis-layergroup-controller.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/map/analysis-layergroup-controller.js b/lib/cartodb/api/map/analysis-layergroup-controller.js index 8a22dd09..41acf819 100644 --- a/lib/cartodb/api/map/analysis-layergroup-controller.js +++ b/lib/cartodb/api/map/analysis-layergroup-controller.js @@ -16,8 +16,11 @@ module.exports = class AnalysisLayergroupController { } register (mapRouter) { - mapRouter.get( - `/:token/analysis/node/:nodeId`, + mapRouter.get('/:token/analysis/node/:nodeId', this.middlewares()); + } + + middlewares () { + return [ layergroupToken(), credentials(), authorize(this.authBackend), @@ -25,7 +28,7 @@ module.exports = class AnalysisLayergroupController { rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), cleanUpQueryParams(), analysisNodeStatus(this.analysisStatusBackend) - ); + ]; } }; From 1bc017eac99a6bb5f60014e171ac1a16547882d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 12:42:17 +0200 Subject: [PATCH 083/185] Rename function --- lib/cartodb/api/map/anonymous-map-controller.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/map/anonymous-map-controller.js b/lib/cartodb/api/map/anonymous-map-controller.js index b16e2c00..6bbacd37 100644 --- a/lib/cartodb/api/map/anonymous-map-controller.js +++ b/lib/cartodb/api/map/anonymous-map-controller.js @@ -64,11 +64,11 @@ module.exports = class AnonymousMapController { register (mapRouter) { mapRouter.options('/'); - mapRouter.get('/', this.composeCreateMapMiddleware()); - mapRouter.post('/', this.composeCreateMapMiddleware()); + mapRouter.get('/', this.middlewares()); + mapRouter.post('/', this.middlewares()); } - composeCreateMapMiddleware () { + middlewares () { const isTemplateInstantiation = false; const useTemplateHash = false; const includeQuery = true; From a77dd9a11fe6879147b7c5ac1f4dda45877867e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 13:20:05 +0200 Subject: [PATCH 084/185] Create .middlewares() method to be mounted in dataview controller --- .../api/map/dataview-layergroup-controller.js | 78 +++---------------- 1 file changed, 11 insertions(+), 67 deletions(-) diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js index 710c5cd4..18392b01 100644 --- a/lib/cartodb/api/map/dataview-layergroup-controller.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -5,6 +5,7 @@ const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const { DATAVIEW: DATAVIEW_GROUP, DATAVIEW_SEARCH: DATAVIEW_SEARCH_GROUP } = RATE_LIMIT_ENDPOINTS_GROUPS; const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); @@ -49,34 +50,19 @@ module.exports = class DataviewLayergroupController { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - mapRouter.get( - `/:token/dataview/:dataviewName`, - layergroupToken(), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - getDataview(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader() - ); + mapRouter.get('/:token/dataview/:dataviewName', this.middlewares('get', DATAVIEW_GROUP)); + mapRouter.get('/:token/:layer/widget/:dataviewName', this.middlewares('get', DATAVIEW_GROUP)); + mapRouter.get('/:token/dataview/:dataviewName/search', this.middlewares('search', DATAVIEW_SEARCH_GROUP)); + mapRouter.get('/:token/:layer/widget/:dataviewName/search', this.middlewares('search', DATAVIEW_SEARCH_GROUP)); + } - mapRouter.get( - `/:token/:layer/widget/:dataviewName`, + middlewares ({ action, rateLimitGroup }) { + return [ layergroupToken(), credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + rateLimit(this.userLimitsBackend, rateLimitGroup), cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), createMapStoreMapConfigProvider( this.mapStore, @@ -84,54 +70,12 @@ module.exports = class DataviewLayergroupController { this.pgConnection, this.layergroupAffectedTablesCache ), - getDataview(this.dataviewBackend), + action === 'search' ? dataviewSearch(this.dataviewBackend) : getDataview(this.dataviewBackend), cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader() - ); - - mapRouter.get( - `/:token/dataview/:dataviewName/search`, - layergroupToken(), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - dataviewSearch(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader() - ); - - mapRouter.get( - `/:token/:layer/widget/:dataviewName/search`, - layergroupToken(), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTablesCache - ), - dataviewSearch(this.dataviewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader() - ); + ]; } }; From 69505974fea40eb09c3538db95e29be6f9484ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 13:29:50 +0200 Subject: [PATCH 085/185] Pass object instead of argument list --- .../api/map/dataview-layergroup-controller.js | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js index 18392b01..1fad5e77 100644 --- a/lib/cartodb/api/map/dataview-layergroup-controller.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -50,10 +50,25 @@ module.exports = class DataviewLayergroupController { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - mapRouter.get('/:token/dataview/:dataviewName', this.middlewares('get', DATAVIEW_GROUP)); - mapRouter.get('/:token/:layer/widget/:dataviewName', this.middlewares('get', DATAVIEW_GROUP)); - mapRouter.get('/:token/dataview/:dataviewName/search', this.middlewares('search', DATAVIEW_SEARCH_GROUP)); - mapRouter.get('/:token/:layer/widget/:dataviewName/search', this.middlewares('search', DATAVIEW_SEARCH_GROUP)); + mapRouter.get('/:token/dataview/:dataviewName', this.middlewares({ + action: 'get', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW + })); + + mapRouter.get('/:token/:layer/widget/:dataviewName', this.middlewares({ + action: 'get', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW + })); + + mapRouter.get('/:token/dataview/:dataviewName/search', this.middlewares({ + action: 'search', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH + })); + + mapRouter.get('/:token/:layer/widget/:dataviewName/search', this.middlewares({ + action: 'search', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH + })); } middlewares ({ action, rateLimitGroup }) { From 33af2d37b36ffb67e369277eed962655414b8f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 13:30:25 +0200 Subject: [PATCH 086/185] Remove declared variables but not used --- lib/cartodb/api/map/dataview-layergroup-controller.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/api/map/dataview-layergroup-controller.js b/lib/cartodb/api/map/dataview-layergroup-controller.js index 1fad5e77..5fd666ea 100644 --- a/lib/cartodb/api/map/dataview-layergroup-controller.js +++ b/lib/cartodb/api/map/dataview-layergroup-controller.js @@ -5,7 +5,6 @@ const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const { DATAVIEW: DATAVIEW_GROUP, DATAVIEW_SEARCH: DATAVIEW_SEARCH_GROUP } = RATE_LIMIT_ENDPOINTS_GROUPS; const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); From 1cfeda8fe5ee3b891f8776c5f0e0a4db4b6c2f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 13:38:51 +0200 Subject: [PATCH 087/185] Create .middlewares() method to mount corresponding set of middlewares --- lib/cartodb/api/map/attributes-layergroup-controller.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/map/attributes-layergroup-controller.js b/lib/cartodb/api/map/attributes-layergroup-controller.js index 632cca9f..2723ebd0 100644 --- a/lib/cartodb/api/map/attributes-layergroup-controller.js +++ b/lib/cartodb/api/map/attributes-layergroup-controller.js @@ -31,8 +31,11 @@ module.exports = class AttributesLayergroupController { } register (mapRouter) { - mapRouter.get( - `/:token/:layer/attributes/:fid`, + mapRouter.get('/:token/:layer/attributes/:fid', this.middlewares()); + } + + middlewares () { + return [ layergroupToken(), credentials(), authorize(this.authBackend), @@ -50,7 +53,7 @@ module.exports = class AttributesLayergroupController { cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader() - ); + ]; } }; From 68b3cb8a34d0050edda23e845d83ee61facd2420 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 11 May 2018 13:44:43 +0200 Subject: [PATCH 088/185] Fix estimated row count with aggregations All stats are computed now pre-aggregation Code to help compute post-aggregation stats remains for testing --- .../layer-stats/mapnik-layer-stats.js | 211 ++++++++++-------- 1 file changed, 113 insertions(+), 98 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index ac62ac2b..aada928a 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,6 +1,24 @@ var queryUtils = require('../../utils/query-utils'); const PhasedExecution = require('../../utils/phased-execution'); const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); +var SubstitutionTokens = require('../../utils/substitution-tokens'); + +// Instantiate a query with tokens for a given zoom level +function queryForZoom(sql, zoom) { + const tileRes = 256; + const wmSize = 6378137.0*2*Math.PI; + const nTiles = Math.pow(2, zoom); + const tileSize = wmSize / nTiles; + const resolution = tileSize / tileRes; + const scaleDenominator = resolution / 0.00028; + const x0 = -wmSize/2, y0 = -wmSize/2; + return SubstitutionTokens.replace(sql, { + bbox: `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`, + scale_denominator: scaleDenominator, + pixel_width: resolution, + pixel_height: resolution + }); +} function MapnikLayerStats () { this._types = { @@ -9,8 +27,6 @@ function MapnikLayerStats () { }; } -// TODO: estimatedFeatureCount is post-aggregation; the rest is pre; distinguish and complement - MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; @@ -40,106 +56,109 @@ function columnAggregations(field) { return []; } +/* Helper to add a task to the queries PhasedExecution + * type can be either 'pre' (for pre-aggregation metadata) or 'post' + * zoom is used only for post-aggregation metadata + * query is a function that generates a metadata query from a data query + * assign is a function to assign the results of the metadata query + * if a assignDefault function is present, it will be used in case of error + * during the query execution and any errors will be ignored + */ +function addStat(queries, ctx, type, zoom, query, assign, assignDefault=null) { + let sql; + if (type === 'pre') { + sql = ctx. preQuery; + } + else { + sql = queryForZoom(ctx.aggrQuery, zoom); + } + sql = query(sql); + queries.task( + queryPromise( + ctx.dbConnection, + sql, + (err, res) => { + if (!err) { + assign(res); + } + else if (assignDefault !== null) { + assignDefault(); + return null; + } + return err; + } + ) + ); +} function firstPhaseQueries(queries, ctx) { + // estimatedFeatureCount if (queries.results.estimatedFeatureCount === undefined) { - queries.task( - queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.aggrQuery), function(err, res) { - if (err) { - // at least for debugging we should err - queries.results.estimatedFeatureCount = -1; - return null; - } else { - // We decided that the relation is 1 row == 1 feature - queries.results.estimatedFeatureCount = res.rows[0].rows; - return null; - } - }) + // This is always computed; a default value of -1 is used in case of error + addStat( + queries, + ctx, + 'pre', 0, + queryUtils.getQueryRowEstimation, + res => queries.results.estimatedFeatureCount = res.rows[0].rows, + () => queries.results.estimatedFeatureCount = -1 ); } + // featureCount if (ctx.metaOptions.featureCount) { - // TODO: pre/aggr - // TODO: for pre, use ctx.aggrMeta.pre_aggregation_count // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query - queries.task( - queryPromise( - ctx.dbConnection, - queryUtils.getQueryActualRowCount(ctx.preQuery), - (err, res) => { - if (err) { - queries.results.featureCount = -1; - } else { - queries.results.featureCount = res.rows[0].rows; - } - return err; - } - ) + addStat( + queries, + ctx, + 'pre', 0, + queryUtils.getQueryActualRowCount, + res => queries.results.featureCount = res.rows[0].rows ); } + // geometryType if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { - // TODO: pre/aggr - // TODO: for pre, use ctx.aggrMeta.geometry_type const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); - queries.task( - queryPromise( - ctx.dbConnection, - queryUtils.getQueryGeometryType(ctx.preQuery, geometryColumn), - (err, res) => { - if (!err) { - queries.results.geometryType = res.rows[0].geom_type; - } - return err; - } - ) + addStat( + queries, + ctx, + 'pre', 0, + sql => queryUtils.getQueryGeometryType(sql, geometryColumn), + res => queries.results.geometryType = res.rows[0].geom_type ); } + // columns (names & types) if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { - // TODO: pre/aggr - // TODO: for aggr, use layer.options.columns (will need to pass in ctx) // note: post-aggregation columns are in layer.options.columns when aggregation is present - queries.task( - // TODO: note we have getLayerColumns in aggregation mapconfig. - // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters - queryPromise( - ctx.dbConnection, - queryUtils.getQueryLimited(ctx.preQuery, 0), - (err, res) => { - if (!err) { - queries.results.columns = formatResultFields(ctx.dbConnection, res.fields); - } - return err; - } - ) + addStat( + queries, + ctx, + 'pre', 0, + sql => queryUtils.getQueryLimited(sql, 0), + res => queries.results.columns = formatResultFields(ctx.dbConnection, res.fields) ); } } function secondPhaseQueries(queries, ctx) { + // sample if (ctx.metaOptions.sample) { const numRows = queries.results.featureCount === undefined ? queries.results.estimatedFeatureCount : queries.results.featureCount; const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); - queries.task( - queryPromise( - ctx.dbConnection, - queryUtils.getQuerySample(ctx.preQuery, sampleProb), - (err, res) => { - if (err) { - queries.results.sample = []; - } else { - queries.results.sample = res.rows; - } - return err; - } - ) + addStat( + queries, + ctx, + 'pre', 0, + sql => queryUtils.getQuerySample(sql, sampleProb), + res => queries.results.sample = res.rows ); } + // columnStats if (ctx.metaOptions.columnStats) { - // TODO: pre/aggr let aggr = []; Object.keys(queries.results.columns).forEach(name => { aggr = aggr.concat( @@ -150,35 +169,27 @@ function secondPhaseQueries(queries, ctx) { const topN = ctx.metaOptions.columnStats.topCategories || 1024; // TODO: ctx.metaOptions.columnStats.maxCategories // => use PG stats to dismiss columns with more distinct values - queries.task( - queryPromise( - ctx.dbConnection, - queryUtils.getQueryTopCategories(ctx.preQuery, name, topN), - (err, res) => { - if (!err) { - queries.results.columns[name].categories = res.rows; - } - return err; - } - ) + addStat( + queries, + ctx, + 'pre', 0, + sql => queryUtils.getQueryTopCategories(sql, name, topN), + res => queries.results.columns[name].categories = res.rows ); } }); - queries.task( - queryPromise( - ctx.dbConnection, - `SELECT ${aggr.join(',')} FROM (${ctx.preQuery}) AS __cdb_query`, - (err, res) => { - if (!err) { - Object.keys(queries.results.columns).forEach(name => { - columnAggregations(queries.results.columns[name]).forEach(fn => { - queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`]; - }); - }); - } - return err; - } - ) + addStat( + queries, + ctx, + 'pre', 0, + sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`, + res => { + Object.keys(queries.results.columns).forEach(name => { + columnAggregations(queries.results.columns[name]).forEach(fn => { + queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + }); + } ); } @@ -209,6 +220,8 @@ function fieldType(cname) { return tname; } +// columns are returned as an object { columnName1: { type1: ...}, ..} +// for consistency with SQL API function formatResultFields(dbConnection, flds) { flds = flds || []; var nfields = {}; @@ -229,7 +242,9 @@ function formatResultFields(dbConnection, flds) { MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { let aggrQuery = layer.options.sql_raw || layer.options.sql; - let preQuery = layer.options.aggregation_metadata ? layer.options.aggregation_metadata.pre_aggregation_sql : aggrQuery; + let preQuery = layer.options.aggregation_metadata ? + layer.options.aggregation_metadata.pre_aggregation_sql : + aggrQuery; let context = { dbConnection, From 242224396d53615ae9b9e0ff2af8f90c9a611b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 14:05:41 +0200 Subject: [PATCH 089/185] Create .middlewares() method to compose controller actions --- .../api/map/preview-layergroup-controller.js | 52 +++++++++---------- lib/cartodb/api/middlewares/noop.js | 5 ++ 2 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 lib/cartodb/api/middlewares/noop.js diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js index 29db8729..9e84f8d9 100644 --- a/lib/cartodb/api/map/preview-layergroup-controller.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -2,6 +2,7 @@ const layergroupToken = require('../middlewares/layergroup-token'); const coordinates = require('../middlewares/coordinates'); const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); const credentials = require('../middlewares/credentials'); +const noop = require('../middlewares/noop'); const dbConnSetup = require('../middlewares/db-conn-setup'); const authorize = require('../middlewares/authorize'); const rateLimit = require('../middlewares/rate-limit'); @@ -32,34 +33,33 @@ module.exports = class PreviewLayergroupController { } register (mapRouter) { + mapRouter.get('/static/center/:token/:z/:lat/:lng/:width/:height.:format', this.middlewares({ + validateZoom: true, + previewType: 'centered' + })); + + mapRouter.get('/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', this.middlewares({ + validateZoom: false, + previewType: 'bbox' + })); + } + + middlewares ({ validateZoom, previewType }) { const forcedFormat = 'png'; - mapRouter.get( - `/static/center/:token/:z/:lat/:lng/:width/:height.:format`, - layergroupToken(), - coordinates({ z: true, x: false, y: false }), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), - cleanUpQueryParams(['layer']), - createMapStoreMapConfigProvider( - this.mapStore, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTablesCache, - forcedFormat - ), - getPreviewImageByCenter(this.previewBackend), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader() - ); + let getPreviewImage; - mapRouter.get( - `/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, + if (previewType === 'centered') { + getPreviewImage = getPreviewImageByCenter; + } + + if (previewType === 'bbox') { + getPreviewImage = getPreviewImageByBoundingBox; + } + + return [ layergroupToken(), + validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(), credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), @@ -72,12 +72,12 @@ module.exports = class PreviewLayergroupController { this.layergroupAffectedTablesCache, forcedFormat ), - getPreviewImageByBoundingBox(this.previewBackend), + getPreviewImage(this.previewBackend), cacheControlHeader(), cacheChannelHeader(), surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader() - ); + ]; } }; diff --git a/lib/cartodb/api/middlewares/noop.js b/lib/cartodb/api/middlewares/noop.js new file mode 100644 index 00000000..16ceabbe --- /dev/null +++ b/lib/cartodb/api/middlewares/noop.js @@ -0,0 +1,5 @@ +module.exports = function noop () { + return function noopMiddleware (req, res, next) { + next(); + }; +}; From 34ad3fcfe85d6b2762fa6f76031251cf473c9757 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 11 May 2018 14:18:31 +0200 Subject: [PATCH 090/185] Add aggregated stat for testing Also change aggregated stats to not filter a single tile --- .../layer-stats/mapnik-layer-stats.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index aada928a..b5818bc7 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -4,7 +4,7 @@ const AggregationMapConfig = require('../../models/aggregation/aggregation-mapco var SubstitutionTokens = require('../../utils/substitution-tokens'); // Instantiate a query with tokens for a given zoom level -function queryForZoom(sql, zoom) { +function queryForZoom(sql, zoom, singleTile=false) { const tileRes = 256; const wmSize = 6378137.0*2*Math.PI; const nTiles = Math.pow(2, zoom); @@ -12,8 +12,12 @@ function queryForZoom(sql, zoom) { const resolution = tileSize / tileRes; const scaleDenominator = resolution / 0.00028; const x0 = -wmSize/2, y0 = -wmSize/2; + let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`; + if (singleTile) { + bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`; + } return SubstitutionTokens.replace(sql, { - bbox: `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`, + bbox: bbox, scale_denominator: scaleDenominator, pixel_width: resolution, pixel_height: resolution @@ -116,6 +120,17 @@ function firstPhaseQueries(queries, ctx) { ); } + // aggrFeatureCount + if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { + addStat( + queries, + ctx, + 'post', ctx.metaOptions.aggrFeatureCount || 0, + queryUtils.getQueryActualRowCount, + res => queries.results.aggrfeatureCount = res.rows[0].rows + ); + } + // geometryType if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); From 3b4668cc19651e8591b2e985304de215ebf95d59 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 11 May 2018 14:45:12 +0200 Subject: [PATCH 091/185] Fix simple tabley sampling --- lib/cartodb/utils/query-utils.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index c7cb131d..0bbd38c0 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -89,9 +89,9 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull }; module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { - const table = simpleQueryTable(query); - if (table) { - return getTableSample(table, sampleProb, randomSeed); + const singleTable = simpleQueryTable(query); + if (singleTable) { + return getTableSample(singleTable.table, singleTable.columns, sampleProb, randomSeed); } return ` WITH __cdb_rndseed AS ( @@ -103,17 +103,17 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { `; }; -function getTableSample(table, sampleProb, randomSeed) { +function getTableSample(table, columns, sampleProb, randomSeed) { sampleProb *= 100; randomSeed *= Math.pow(2, 31) -1; return ` - SELECT * FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) + SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) `; } function simpleQueryTable(sql) { const basicQuery = - /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; + /\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i'); // queries for named maps are wrapped like this: var wrappedQuery = new RegExp( @@ -127,9 +127,10 @@ function simpleQueryTable(sql) { match = sql.match(wrappedQuery); } if (match) { - const schema = match[2]; - const table = match[3]; - return schema ? `${schema}.${table}` : table; + const columns = match[1]; + const schema = match[3]; + const table = match[4]; + return { table: schema ? `${schema}.${table}` : table, columns }; } return false; } From 3d7231929c265b24b7743d4fa3fddaeaf742478f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 15:34:10 +0200 Subject: [PATCH 092/185] Create .middlewares() method to return the middlewares to process the request --- .../api/map/preview-template-controller.js | 95 ++++++++++--------- .../api/map/tile-layergroup-controller.js | 14 ++- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index 9f5257a3..385b954d 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -22,54 +22,57 @@ function numMapper(n) { return +n; } -function PreviewTemplateController ( - namedMapProviderCache, - previewBackend, - surrogateKeysCache, - tablesExtentBackend, - metadataBackend, - pgConnection, - authBackend, - userLimitsBackend -) { - this.namedMapProviderCache = namedMapProviderCache; - this.previewBackend = previewBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.tablesExtentBackend = tablesExtentBackend; - this.metadataBackend = metadataBackend; - this.pgConnection = pgConnection; - this.authBackend = authBackend; - this.userLimitsBackend = userLimitsBackend; -} +module.exports = class PreviewTemplateController { + constructor ( + namedMapProviderCache, + previewBackend, + surrogateKeysCache, + tablesExtentBackend, + metadataBackend, + pgConnection, + authBackend, + userLimitsBackend + ) { + this.namedMapProviderCache = namedMapProviderCache; + this.previewBackend = previewBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.tablesExtentBackend = tablesExtentBackend; + this.metadataBackend = metadataBackend; + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = PreviewTemplateController; + register (mapRouter) { + mapRouter.get('/static/named/:template_id/:width/:height.:format', this.middlewares()); + } -PreviewTemplateController.prototype.register = function (mapRouter) { - mapRouter.get( - `/static/named/:template_id/:width/:height.:format`, - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), - cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), - namedMapProvider({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'STATIC_VIZ_MAP', forcedFormat: 'png' - }), - getTemplate({ label: 'STATIC_VIZ_MAP' }), - prepareLayerFilterFromPreviewLayers({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'STATIC_VIZ_MAP' - }), - getStaticImageOptions({ tablesExtentBackend: this.tablesExtentBackend }), - getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), - setContentTypeHeader(), - incrementMapViews({ metadataBackend: this.metadataBackend }), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader() - ); + middlewares () { + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), + cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'STATIC_VIZ_MAP', forcedFormat: 'png' + }), + getTemplate({ label: 'STATIC_VIZ_MAP' }), + prepareLayerFilterFromPreviewLayers({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'STATIC_VIZ_MAP' + }), + getStaticImageOptions({ tablesExtentBackend: this.tablesExtentBackend }), + getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), + setContentTypeHeader(), + incrementMapViews({ metadataBackend: this.metadataBackend }), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } }; function getTemplate ({ label }) { diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js index f882d13c..229f5d00 100644 --- a/lib/cartodb/api/map/tile-layergroup-controller.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -46,10 +46,14 @@ module.exports = class TileLayergroupController { const not = (val) => `(?!${val})([^\/]+?)`; mapRouter.get([ - `/:token/:z/:x/:y@:scale_factor?x.:format`, - `/:token/:z/:x/:y.:format`, - `/:token${not('static')}/:layer/:z/:x/:y.(:format)` - ], + `/:token/:z/:x/:y@:scale_factor?x.:format`, + `/:token/:z/:x/:y.:format`, + `/:token${not('static')}/:layer/:z/:x/:y.(:format)` + ], this.middlewares()); + } + + middlewares () { + return [ layergroupToken(), coordinates(), credentials(), @@ -72,7 +76,7 @@ module.exports = class TileLayergroupController { incrementErrorMetrics(global.statsClient), tileError(), vectorError() - ); + ]; } }; From 1f717617b0b01e4a0140e468ab4b49c0eea98431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:07:25 +0200 Subject: [PATCH 093/185] Add .middlewares() method to build middleware stack to be mounted --- .../api/template/admin-template-controller.js | 128 ++++++++++-------- 1 file changed, 74 insertions(+), 54 deletions(-) diff --git a/lib/cartodb/api/template/admin-template-controller.js b/lib/cartodb/api/template/admin-template-controller.js index b45ebca7..b2087685 100644 --- a/lib/cartodb/api/template/admin-template-controller.js +++ b/lib/cartodb/api/template/admin-template-controller.js @@ -3,70 +3,90 @@ const credentials = require('../middlewares/credentials'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -/** - * @param {AuthBackend} authBackend - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @constructor - */ -function AdminTemplateController(authBackend, templateMaps, userLimitsBackend) { - this.authBackend = authBackend; - this.templateMaps = templateMaps; - this.userLimitsBackend = userLimitsBackend; -} +module.exports = class AdminTemplateController { + /** + * @param {AuthBackend} authBackend + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @constructor + */ + constructor (authBackend, templateMaps, userLimitsBackend) { + this.authBackend = authBackend; + this.templateMaps = templateMaps; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = AdminTemplateController; + register (templateRouter) { + templateRouter.options(`/:template_id`); -AdminTemplateController.prototype.register = function (templateRouter) { - templateRouter.options(`/:template_id`); + templateRouter.post('/', this.middlewares({ + action: 'create', + label: 'POST TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE + })); - templateRouter.post( - `/`, - credentials(), - authorizedByAPIKey({ authBackend: this.authBackend, action: 'create', label: 'POST TEMPLATE' }), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), - checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), - createTemplate({ templateMaps: this.templateMaps }) - ); + templateRouter.put('/:template_id', this.middlewares({ + action: 'update', + label: 'PUT TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE + })); - templateRouter.put( - `/:template_id`, - credentials(), - authorizedByAPIKey({ authBackend: this.authBackend, action: 'update', label: 'PUT TEMPLATE' }), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), - checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), - updateTemplate({ templateMaps: this.templateMaps }) - ); + templateRouter.get('/:template_id', this.middlewares({ + action: 'get', + label: 'GET TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET + })); - templateRouter.get( - `/:template_id`, - credentials(), - authorizedByAPIKey({ authBackend: this.authBackend, action: 'get', label: 'GET TEMPLATE' }), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), - retrieveTemplate({ templateMaps: this.templateMaps }) - ); + templateRouter.delete('/:template_id', this.middlewares({ + action: 'delete', + label: 'DELETE TEMPLATE', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE + })); - templateRouter.delete( - `/:template_id`, - credentials(), - authorizedByAPIKey({ authBackend: this.authBackend, action: 'delete', label: 'DELETE TEMPLATE' }), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), - destroyTemplate({ templateMaps: this.templateMaps }) - ); + templateRouter.get('/', this.middlewares({ + action: 'list', + label: 'GET TEMPLATE LIST', + rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST + })); + } - templateRouter.get( - `/`, - credentials(), - authorizedByAPIKey({ authBackend: this.authBackend, action: 'list', label: 'GET TEMPLATE LIST' }), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), - listTemplates({ templateMaps: this.templateMaps }) - ); + middlewares ({ action, label, rateLimitGroup }) { + let template; + + if (action === 'create') { + template = createTemplate; + } + + if (action === 'update') { + template = updateTemplate; + } + + if (action === 'get') { + template = retrieveTemplate; + } + + if (action === 'delete') { + template = destroyTemplate; + } + + if (action === 'list') { + template = listTemplates; + } + + return [ + credentials(), + authorizedByAPIKey({ authBackend: this.authBackend, action, label }), + rateLimit(this.userLimitsBackend, rateLimitGroup), + checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), + template({ templateMaps: this.templateMaps }) + ]; + } }; -function checkContentType ({ action, label }) { +function checkContentType ({ label }) { return function checkContentTypeMiddleware (req, res, next) { - if (!req.is('application/json')) { - const error = new Error(`template ${action} data must be of type application/json`); + if ((req.method === 'POST' || req.method === 'PUT') && !req.is('application/json')) { + const error = new Error(`${req.method} template data must be of type application/json`); error.label = label; return next(error); } From 6eeb75a35ef30ba6265128d13cbac56200c52f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:13:34 +0200 Subject: [PATCH 094/185] Use .middlewares() method to build middleware stack to process the request --- lib/cartodb/api/map/analyses-catalog-controller.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/map/analyses-catalog-controller.js b/lib/cartodb/api/map/analyses-catalog-controller.js index 760c0e80..870d81dc 100644 --- a/lib/cartodb/api/map/analyses-catalog-controller.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -17,8 +17,11 @@ function AnalysesController(pgConnection, authBackend, userLimitsBackend) { module.exports = AnalysesController; AnalysesController.prototype.register = function (mapRouter) { - mapRouter.get( - `/analyses/catalog`, + mapRouter.get('/analyses/catalog', this.middlewares()); +}; + +AnalysesController.prototype.middlewares = function () { + return [ credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), @@ -30,7 +33,7 @@ AnalysesController.prototype.register = function (mapRouter) { prepareResponse(), cacheControlHeader({ ttl: 10, revalidate: true }), unauthorizedError() - ); + ]; }; function createPGClient () { From 9db6e2161b2a3f9858cf8769d951011de88b0c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:15:33 +0200 Subject: [PATCH 095/185] Use ES6 class syntax --- .../api/map/analyses-catalog-controller.js | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/api/map/analyses-catalog-controller.js b/lib/cartodb/api/map/analyses-catalog-controller.js index 870d81dc..2fae5ac2 100644 --- a/lib/cartodb/api/map/analyses-catalog-controller.js +++ b/lib/cartodb/api/map/analyses-catalog-controller.js @@ -8,32 +8,32 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; const cacheControlHeader = require('../middlewares/cache-control-header'); const dbParamsFromResLocals = require('../../utils/database-params'); -function AnalysesController(pgConnection, authBackend, userLimitsBackend) { - this.pgConnection = pgConnection; - this.authBackend = authBackend; - this.userLimitsBackend = userLimitsBackend; -} +module.exports = class AnalysesController { + constructor (pgConnection, authBackend, userLimitsBackend) { + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = AnalysesController; + register (mapRouter) { + mapRouter.get('/analyses/catalog', this.middlewares()); + } -AnalysesController.prototype.register = function (mapRouter) { - mapRouter.get('/analyses/catalog', this.middlewares()); -}; - -AnalysesController.prototype.middlewares = function () { - return [ - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), - cleanUpQueryParams(), - createPGClient(), - getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), - getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), - prepareResponse(), - cacheControlHeader({ ttl: 10, revalidate: true }), - unauthorizedError() - ]; + middlewares () { + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), + cleanUpQueryParams(), + createPGClient(), + getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), + getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), + prepareResponse(), + cacheControlHeader({ ttl: 10, revalidate: true }), + unauthorizedError() + ]; + } }; function createPGClient () { From 68f5ee7bde138a7589bb9fb8a59e5c67db6cf711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:18:52 +0200 Subject: [PATCH 096/185] Use ES6 class syntax --- .../api/template/named-template-controller.js | 150 +++++++++--------- 1 file changed, 72 insertions(+), 78 deletions(-) diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js index 86613c5e..f85892bd 100644 --- a/lib/cartodb/api/template/named-template-controller.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -33,88 +33,82 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {StatsBackend} statsBackend * @constructor */ -function NamedMapController ( - pgConnection, - templateMaps, - mapBackend, - metadataBackend, - surrogateKeysCache, - userLimitsBackend, - layergroupAffectedTables, - mapConfigAdapter, - statsBackend, - authBackend, - layergroupMetadata -) { - this.pgConnection = pgConnection; - this.templateMaps = templateMaps; - this.mapBackend = mapBackend; - this.metadataBackend = metadataBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsBackend = userLimitsBackend; - this.layergroupAffectedTables = layergroupAffectedTables; - this.mapConfigAdapter = mapConfigAdapter; - this.statsBackend = statsBackend; - this.authBackend = authBackend; - this.layergroupMetadata = layergroupMetadata; -} +module.exports = class NamedMapController { + constructor ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsBackend, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authBackend, + layergroupMetadata + ) { + this.pgConnection = pgConnection; + this.templateMaps = templateMaps; + this.mapBackend = mapBackend; + this.metadataBackend = metadataBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTables = layergroupAffectedTables; + this.mapConfigAdapter = mapConfigAdapter; + this.statsBackend = statsBackend; + this.authBackend = authBackend; + this.layergroupMetadata = layergroupMetadata; + } -module.exports = NamedMapController; + register (templateRouter) { + templateRouter.get(`/:template_id/jsonp`, this.composeInstantiateTemplateMiddleware()); -NamedMapController.prototype.register = function (templateRouter) { - templateRouter.get( - `/:template_id/jsonp`, - this.composeInstantiateTemplateMiddleware() - ); + templateRouter.post(`/:template_id`, this.composeInstantiateTemplateMiddleware()); + } - templateRouter.post( - `/:template_id`, - this.composeInstantiateTemplateMiddleware() - ); -}; + composeInstantiateTemplateMiddleware () { + const isTemplateInstantiation = true; + const useTemplateHash = true; + const includeQuery = false; + const label = 'NAMED MAP LAYERGROUP'; + const addContext = false; -NamedMapController.prototype.composeInstantiateTemplateMiddleware = function () { - const isTemplateInstantiation = true; - const useTemplateHash = true; - const includeQuery = false; - const label = 'NAMED MAP LAYERGROUP'; - const addContext = false; - - return [ - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), - cleanUpQueryParams(['aggregation']), - initProfiler(isTemplateInstantiation), - checkJsonContentType(), - checkInstantiteLayergroup(), - getTemplate( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsBackend, - this.mapConfigAdapter, - this.layergroupAffectedTables - ), - instantiateLayergroup( - this.mapBackend, - this.userLimitsBackend, - this.pgConnection, - this.layergroupAffectedTables - ), - incrementMapViewCount(this.metadataBackend), - augmentLayergroupData(), - cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader({ now: true }), - lastUpdatedTimeLayergroup(), - layerStats(this.pgConnection, this.statsBackend), - layergroupIdHeader(this.templateMaps ,useTemplateHash), - layergroupMetadata(this.layergroupMetadata, includeQuery), - mapError({ label, addContext }) - ]; + return [ + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED), + cleanUpQueryParams(['aggregation']), + initProfiler(isTemplateInstantiation), + checkJsonContentType(), + checkInstantiteLayergroup(), + getTemplate( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsBackend, + this.mapConfigAdapter, + this.layergroupAffectedTables + ), + instantiateLayergroup( + this.mapBackend, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTables + ), + incrementMapViewCount(this.metadataBackend), + augmentLayergroupData(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), + lastUpdatedTimeLayergroup(), + layerStats(this.pgConnection, this.statsBackend), + layergroupIdHeader(this.templateMaps ,useTemplateHash), + layergroupMetadata(this.layergroupMetadata, includeQuery), + mapError({ label, addContext }) + ]; + } }; function checkInstantiteLayergroup () { From 3bcf6d7ca09cc72e3965930a19ddde86209a41ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:20:44 +0200 Subject: [PATCH 097/185] Place comment --- .../api/template/named-template-controller.js | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js index f85892bd..5de0a91d 100644 --- a/lib/cartodb/api/template/named-template-controller.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -20,20 +20,21 @@ const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provid const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -/** - * @param {AuthBackend} authBackend - * @param {PgConnection} pgConnection - * @param {TemplateMaps} templateMaps - * @param {MapBackend} mapBackend - * @param metadataBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsBackend} userLimitsBackend - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {MapConfigAdapter} mapConfigAdapter - * @param {StatsBackend} statsBackend - * @constructor - */ module.exports = class NamedMapController { + /** + * @param {PgConnection} pgConnection + * @param {TemplateMaps} templateMaps + * @param {MapBackend} mapBackend + * @param metadataBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsBackend} userLimitsBackend + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigAdapter} mapConfigAdapter + * @param {StatsBackend} statsBackend + * @param {AuthBackend} authBackend + * @param layergroupMetadata + * @constructor + */ constructor ( pgConnection, templateMaps, @@ -62,7 +63,6 @@ module.exports = class NamedMapController { register (templateRouter) { templateRouter.get(`/:template_id/jsonp`, this.composeInstantiateTemplateMiddleware()); - templateRouter.post(`/:template_id`, this.composeInstantiateTemplateMiddleware()); } From 595d006d5b7ca0287433e01777d9caefe28e154c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:21:57 +0200 Subject: [PATCH 098/185] Rename function --- lib/cartodb/api/template/named-template-controller.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/template/named-template-controller.js b/lib/cartodb/api/template/named-template-controller.js index 5de0a91d..0af1ad65 100644 --- a/lib/cartodb/api/template/named-template-controller.js +++ b/lib/cartodb/api/template/named-template-controller.js @@ -62,11 +62,11 @@ module.exports = class NamedMapController { } register (templateRouter) { - templateRouter.get(`/:template_id/jsonp`, this.composeInstantiateTemplateMiddleware()); - templateRouter.post(`/:template_id`, this.composeInstantiateTemplateMiddleware()); + templateRouter.get('/:template_id/jsonp', this.middlewares()); + templateRouter.post('/:template_id', this.middlewares()); } - composeInstantiateTemplateMiddleware () { + middlewares () { const isTemplateInstantiation = true; const useTemplateHash = true; const includeQuery = false; From 3905ed796e4337cd982a9ea503b09a7e57d9ad5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:24:28 +0200 Subject: [PATCH 099/185] Use ES6 class syntax --- .../api/template/tile-template-controller.js | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js index de256ccb..e4d80663 100644 --- a/lib/cartodb/api/template/tile-template-controller.js +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -12,48 +12,48 @@ const vectorError = require('../middlewares/vector-error'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -function TileTemplateController ( - namedMapProviderCache, - tileBackend, - surrogateKeysCache, - pgConnection, - authBackend, - userLimitsBackend -) { - this.namedMapProviderCache = namedMapProviderCache; - this.tileBackend = tileBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.pgConnection = pgConnection; - this.authBackend = authBackend; - this.userLimitsBackend = userLimitsBackend; -} +module.exports = class TileTemplateController { + constructor ( + namedMapProviderCache, + tileBackend, + surrogateKeysCache, + pgConnection, + authBackend, + userLimitsBackend + ) { + this.namedMapProviderCache = namedMapProviderCache; + this.tileBackend = tileBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.pgConnection = pgConnection; + this.authBackend = authBackend; + this.userLimitsBackend = userLimitsBackend; + } -module.exports = TileTemplateController; - -TileTemplateController.prototype.register = function (templateRouter) { - templateRouter.get( - `/:template_id/:layer/:z/:x/:y.(:format)`, - coordinates(), - credentials(), - authorize(this.authBackend), - dbConnSetup(this.pgConnection), - rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), - cleanUpQueryParams(), - namedMapProvider({ - namedMapProviderCache: this.namedMapProviderCache, - label: 'NAMED_MAP_TILE' - }), - getTile({ - tileBackend: this.tileBackend, - label: 'NAMED_MAP_TILE' - }), - setContentTypeHeader(), - cacheControlHeader(), - cacheChannelHeader(), - surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - lastModifiedHeader(), - vectorError() - ); + register (templateRouter) { + templateRouter.get( + `/:template_id/:layer/:z/:x/:y.(:format)`, + coordinates(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + cleanUpQueryParams(), + namedMapProvider({ + namedMapProviderCache: this.namedMapProviderCache, + label: 'NAMED_MAP_TILE' + }), + getTile({ + tileBackend: this.tileBackend, + label: 'NAMED_MAP_TILE' + }), + setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + vectorError() + ); + } }; function getTile ({ tileBackend, label }) { From c6babc7dc454c7110d2aae36651409bbc5ea1a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:26:05 +0200 Subject: [PATCH 100/185] Create .middlewares() method to build midlleware stack to perform request --- lib/cartodb/api/template/tile-template-controller.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/template/tile-template-controller.js b/lib/cartodb/api/template/tile-template-controller.js index e4d80663..71a45e6e 100644 --- a/lib/cartodb/api/template/tile-template-controller.js +++ b/lib/cartodb/api/template/tile-template-controller.js @@ -30,8 +30,11 @@ module.exports = class TileTemplateController { } register (templateRouter) { - templateRouter.get( - `/:template_id/:layer/:z/:x/:y.(:format)`, + templateRouter.get('/:template_id/:layer/:z/:x/:y.(:format)', this.middlewares()); + } + + middlewares () { + return [ coordinates(), credentials(), authorize(this.authBackend), @@ -52,7 +55,7 @@ module.exports = class TileTemplateController { surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), lastModifiedHeader(), vectorError() - ); + ]; } }; From f4d60f963d76e9630e396a893223e0ed91caa375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 11 May 2018 16:37:02 +0200 Subject: [PATCH 101/185] Add comment --- lib/cartodb/api/map/tile-layergroup-controller.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/map/tile-layergroup-controller.js b/lib/cartodb/api/map/tile-layergroup-controller.js index 229f5d00..311fec45 100644 --- a/lib/cartodb/api/map/tile-layergroup-controller.js +++ b/lib/cartodb/api/map/tile-layergroup-controller.js @@ -42,12 +42,15 @@ module.exports = class TileLayergroupController { } register (mapRouter) { - // REGEXP doesn't match with `val` + // REGEXP: doesn't match with `val` const not = (val) => `(?!${val})([^\/]+?)`; + // Sadly the path that matches 1 also matches with 2 so we need to tell to express + // that performs only the middlewares of the first path that matches + // for that we use one array to group all paths. mapRouter.get([ - `/:token/:z/:x/:y@:scale_factor?x.:format`, - `/:token/:z/:x/:y.:format`, + `/:token/:z/:x/:y@:scale_factor?x.:format`, // 1 + `/:token/:z/:x/:y.:format`, // 2 `/:token${not('static')}/:layer/:z/:x/:y.(:format)` ], this.middlewares()); } From 7112341c51284994e968357163548eea5ee63882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 11 May 2018 17:41:00 +0200 Subject: [PATCH 102/185] checkStaticImageFormat middleware --- .../api/middlewares/check-static-image-format.js | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 lib/cartodb/api/middlewares/check-static-image-format.js diff --git a/lib/cartodb/api/middlewares/check-static-image-format.js b/lib/cartodb/api/middlewares/check-static-image-format.js new file mode 100644 index 00000000..d72fcb81 --- /dev/null +++ b/lib/cartodb/api/middlewares/check-static-image-format.js @@ -0,0 +1,11 @@ +const VALID_IMAGE_FORMATS = ['png', 'jpg', 'jpeg']; + +module.exports = function checkStaticImageFormat () { + return function checkStaticImageFormatMiddleware (req, res, next) { + if(!VALID_IMAGE_FORMATS.includes(req.params.format.toLowerCase())) { + return next(new Error(`Unsupported image format "${req.params.format}"`)); + } + + next(); + }; +}; From db7b4fa937c421b3d9cc241e2aeba1b796a25abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 11 May 2018 17:41:26 +0200 Subject: [PATCH 103/185] static image format tests --- test/acceptance/analysis/named-maps.js | 35 +++++++++++++++ test/acceptance/named_maps_static_view.js | 53 +++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index 42fb19d0..98316d2d 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -261,6 +261,41 @@ describe('named-maps analysis', function() { ); }); + it('should return and an error requesting unsupported image format', function(done) { + assert.response( + server, + { + url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.gif', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + assert.ifError(err); + assert.deepEqual( + JSON.parse(res.body), + { + errors:['Unsupported image format \"gif\"'], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported image format \"gif\"' + }] + } + ); + done(); + + } + ); + }); + }); describe('auto-instantiation', function() { diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index 9844c667..69c4e56c 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -282,4 +282,57 @@ describe('named maps static view', function() { }); }); + it('should return an error requesting unsupported image format', function (done) { + var view = { + zoom: 4, + center: { + lng: 40, + lat: 20 + } + }; + + templateMaps.addTemplate(username, createTemplate(view), function (err) { + if (err) { + return done(err); + } + + var url = `/api/v1/map/static/named/${templateName}/640/480.gif`; + + + var requestOptions = { + url: url, + method: 'GET', + headers: { + host: username + }, + encoding: 'binary' + }; + + var expectedResponse = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + // this could be removed once named maps are invalidated, otherwise you hits the cache + var server = new CartodbWindshaft(serverOptions); + + assert.response(server, requestOptions, expectedResponse, function (res, err) { + assert.ifError(err); + assert.deepEqual( + JSON.parse(res.body), + { + errors:['Unsupported image format \"gif\"'], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported image format \"gif\"' + }] + } + ); + done(); + }); + }); + }); + }); From e96a9f0b46745ee6878f268f9fe547a51428f4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 11 May 2018 17:42:28 +0200 Subject: [PATCH 104/185] static image format controllers --- lib/cartodb/api/map/preview-layergroup-controller.js | 2 ++ lib/cartodb/api/map/preview-template-controller.js | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js index 29db8729..5e9144c3 100644 --- a/lib/cartodb/api/map/preview-layergroup-controller.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -11,6 +11,7 @@ const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); const lastModifiedHeader = require('../middlewares/last-modified-header'); +const checkStaticImageFormat = require('../middlewares/check-static-image-format'); module.exports = class PreviewLayergroupController { constructor ( @@ -43,6 +44,7 @@ module.exports = class PreviewLayergroupController { dbConnSetup(this.pgConnection), rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), + checkStaticImageFormat(), createMapStoreMapConfigProvider( this.mapStore, this.userLimitsBackend, diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index 9f5257a3..b0359e69 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -7,6 +7,7 @@ const cacheControlHeader = require('../middlewares/cache-control-header'); const cacheChannelHeader = require('../middlewares/cache-channel-header'); const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); const lastModifiedHeader = require('../middlewares/last-modified-header'); +const checkStaticImageFormat = require('../middlewares/check-static-image-format'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -52,6 +53,7 @@ PreviewTemplateController.prototype.register = function (mapRouter) { dbConnSetup(this.pgConnection), rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), + checkStaticImageFormat(), namedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP', forcedFormat: 'png' @@ -253,7 +255,7 @@ function getImage({ previewBackend, label }) { height = +height; const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - + if (zoom !== undefined && center) { return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, (err, image, headers, stats) => { From 7f5e65573018be60bbfbbccc87539e0130b40010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 11 May 2018 17:45:17 +0200 Subject: [PATCH 105/185] static image format for last route --- lib/cartodb/api/map/preview-layergroup-controller.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/api/map/preview-layergroup-controller.js b/lib/cartodb/api/map/preview-layergroup-controller.js index 5e9144c3..29cb11bc 100644 --- a/lib/cartodb/api/map/preview-layergroup-controller.js +++ b/lib/cartodb/api/map/preview-layergroup-controller.js @@ -67,6 +67,7 @@ module.exports = class PreviewLayergroupController { dbConnSetup(this.pgConnection), rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), cleanUpQueryParams(['layer']), + checkStaticImageFormat(), createMapStoreMapConfigProvider( this.mapStore, this.userLimitsBackend, From ad4ed7a06b3ce23971aeb68f7c4f1e1d3757ba48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 11 May 2018 17:47:47 +0200 Subject: [PATCH 106/185] detail --- lib/cartodb/api/map/preview-template-controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/api/map/preview-template-controller.js b/lib/cartodb/api/map/preview-template-controller.js index b0359e69..1a292ddb 100644 --- a/lib/cartodb/api/map/preview-template-controller.js +++ b/lib/cartodb/api/map/preview-template-controller.js @@ -255,7 +255,7 @@ function getImage({ previewBackend, label }) { height = +height; const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - + if (zoom !== undefined && center) { return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, (err, image, headers, stats) => { From 53fae9fbbdb4da34f3456a48fcceef9b08063865 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 11 May 2018 18:57:14 +0200 Subject: [PATCH 107/185] Comment --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index b5818bc7..49988aa8 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -122,6 +122,9 @@ function firstPhaseQueries(queries, ctx) { // aggrFeatureCount if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { + // We expect as zoom level as the value of aggrFeatureCount + // TODO: it'd be nice to admit an array of zoom levels to + // return metadata for multiple levels. addStat( queries, ctx, From b906f88a4407fe9211605f761b33eb505d720155 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 11 May 2018 19:32:03 +0200 Subject: [PATCH 108/185] Slight refactor --- .../layer-stats/mapnik-layer-stats.js | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 49988aa8..6a71435e 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -94,8 +94,8 @@ function addStat(queries, ctx, type, zoom, query, assign, assignDefault=null) { ) ); } -function firstPhaseQueries(queries, ctx) { - // estimatedFeatureCount + +function _estimatedFeatureCount(queries, ctx) { if (queries.results.estimatedFeatureCount === undefined) { // This is always computed; a default value of -1 is used in case of error addStat( @@ -107,8 +107,9 @@ function firstPhaseQueries(queries, ctx) { () => queries.results.estimatedFeatureCount = -1 ); } +} - // featureCount +function _featureCount(queries, ctx) { if (ctx.metaOptions.featureCount) { // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query addStat( @@ -119,8 +120,9 @@ function firstPhaseQueries(queries, ctx) { res => queries.results.featureCount = res.rows[0].rows ); } +} - // aggrFeatureCount +function _aggrFeatureCount(queries, ctx) { if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { // We expect as zoom level as the value of aggrFeatureCount // TODO: it'd be nice to admit an array of zoom levels to @@ -133,8 +135,9 @@ function firstPhaseQueries(queries, ctx) { res => queries.results.aggrfeatureCount = res.rows[0].rows ); } +} - // geometryType +function _geometryType(queries, ctx) { if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); addStat( @@ -145,8 +148,9 @@ function firstPhaseQueries(queries, ctx) { res => queries.results.geometryType = res.rows[0].geom_type ); } +} - // columns (names & types) +function _columns(queries, ctx) { if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { // note: post-aggregation columns are in layer.options.columns when aggregation is present addStat( @@ -159,8 +163,15 @@ function firstPhaseQueries(queries, ctx) { } } -function secondPhaseQueries(queries, ctx) { - // sample +function firstPhaseQueries(queries, ctx) { + _estimatedFeatureCount(queries, ctx); + _featureCount(queries, ctx); + _aggrFeatureCount(queries, ctx); + _geometryType(queries, ctx); + _columns(queries, ctx); +} + +function _sample(queries, ctx) { if (ctx.metaOptions.sample) { const numRows = queries.results.featureCount === undefined ? queries.results.estimatedFeatureCount : @@ -174,8 +185,9 @@ function secondPhaseQueries(queries, ctx) { res => queries.results.sample = res.rows ); } +} - // columnStats +function _columnStats(queries, ctx) { if (ctx.metaOptions.columnStats) { let aggr = []; Object.keys(queries.results.columns).forEach(name => { @@ -210,7 +222,11 @@ function secondPhaseQueries(queries, ctx) { } ); } +} +function secondPhaseQueries(queries, ctx) { + _sample(queries, ctx); + _columnStats(queries, ctx); } // This is adapted from SQL API: From 5e09c80b717e6b7261e48e97850b54887028f81c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 11 May 2018 19:57:49 +0200 Subject: [PATCH 109/185] Remove comment --- .../models/mapconfig/adapter/aggregation-mapconfig-adapter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 8639b85b..b3df3d96 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -73,7 +73,7 @@ module.exports = class AggregationMapConfigAdapter { if (adapted) { requestMapConfig.layers[index] = layer; } - const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); // <<- + const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); context.aggregation.layers.push(aggregatedFormats); }); From b8109401d1bf83d723e21249d4af9c71fa13bf4f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Sun, 13 May 2018 13:05:39 +0200 Subject: [PATCH 110/185] Tests for metadata with aggregation --- test/acceptance/aggregation.js | 176 ++++++++++++++++++ .../stats/mapnik_stats_layergroup.js | 1 - 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 6658eb81..6aab8d95 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -100,6 +100,18 @@ describe('aggregation', function () { from generate_series(-3, 3) x `; + const POINTS_SQL_PAIRS = ` + -- Generate pairs of near points + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0), 4326) as the_geom, + st_transform( + st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0),4326), + 3857) as the_geom_webmercator, + x as value + from generate_series(-6, 6) x + `; + function createVectorMapConfig (layers = [ { type: 'cartodb', @@ -130,6 +142,7 @@ describe('aggregation', function () { before(function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; }); after(function (){ @@ -138,6 +151,7 @@ describe('aggregation', function () { afterEach(function (done) { this.testClient.drain(done); + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); it('should return a layergroup indicating the mapconfig was aggregated', function (done) { @@ -2173,6 +2187,168 @@ describe('aggregation', function () { }); }); + ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`default pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.ok(body.metadata.layers[0].meta.stats.estimatedFeatureCount > 0); + + done(); + }); + }); + + it(`on demand post-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + aggrFeatureCount: 10 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrfeatureCount, 13); + + done(); + }); + }); + + it(`post-aggregation count adapts to zoom level with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + aggrFeatureCount: 0 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrfeatureCount, 9); + + done(); + }); + }); + + it(`on-demand pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + featureCount: true + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.featureCount, 13); + + done(); + }); + }); + }); }); }); diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index d8a9a06b..f74272d0 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -475,5 +475,4 @@ describe('Create mapnik layergroup', function() { }); }); - // TODO: add tests for metadata with aggregation }); From 5a397afd0609611fd8ee6a38a5378aed2d7feaea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 14 May 2018 11:50:48 +0200 Subject: [PATCH 111/185] In order to extract common middlewares between routers and controlles we need to activate mergeParams options to preserve the req.params from the parent router --- lib/cartodb/api/api-router.js | 2 +- lib/cartodb/api/map/map-router.js | 2 +- lib/cartodb/api/template/template-router.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index 9d53285b..b7534559 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -191,7 +191,7 @@ module.exports = class ApiRouter { Object.keys(this.serverOptions.routes).forEach(apiVersion => { const routes = this.serverOptions.routes[apiVersion]; - const apiRouter = router(); + const apiRouter = router({ mergeParams: true }); apiRouter.use(logger(this.serverOptions)); apiRouter.use(initializeStatusCode()); diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index 03f19937..46e2be5e 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -113,7 +113,7 @@ module.exports = class MapRouter { } register (apiRouter, mapPaths) { - const mapRouter = router(); + const mapRouter = router({ mergeParams: true }); this.analysisLayergroupController.register(mapRouter); this.attributesLayergroupController.register(mapRouter); diff --git a/lib/cartodb/api/template/template-router.js b/lib/cartodb/api/template/template-router.js index 9ddcf9fc..4234bb22 100644 --- a/lib/cartodb/api/template/template-router.js +++ b/lib/cartodb/api/template/template-router.js @@ -53,7 +53,7 @@ module.exports = class TemplateRouter { } register (apiRouter, templatePaths) { - const templateRouter = router(); + const templateRouter = router({ mergeParams: true }); this.namedMapController.register(templateRouter); this.tileTemplateController.register(templateRouter); From 63a9d58c6713c065818c09813007c5d87cee7ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 14 May 2018 12:28:57 +0200 Subject: [PATCH 112/185] Add regression test --- test/acceptance/regressions.js | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/acceptance/regressions.js b/test/acceptance/regressions.js index 698d1401..e49065d3 100644 --- a/test/acceptance/regressions.js +++ b/test/acceptance/regressions.js @@ -1,7 +1,10 @@ require('../support/test_helper'); var assert = require('../support/assert'); +const helper = require('../support/test_helper'); var TestClient = require('../support/test-client'); const LayergroupToken = require('../../lib/cartodb/models/layergroup-token'); +const CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +const serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); describe('regressions', function() { @@ -38,6 +41,49 @@ describe('regressions', function() { }); }); + // See: https://github.com/CartoDB/Windshaft-cartodb/pull/956 + it('"/user/localhost/api/v1/map" should create an anonymous map', function (done) { + const server = new CartodbWindshaft(serverOptions); + const layergroup = { + version: '1.7.0', + layers: [ + { + type: 'mapnik', + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + }; + + const keysToDelete = {}; + + assert.response(server, + { + url: '/user/localhost/api/v1/map', + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + function(res, err) { + if (err) { + return done(err); + } + + const body = JSON.parse(res.body); + assert.ok(body.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(body.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + helper.deleteRedisKeys(keysToDelete, done); + } + ); + }); + describe('map instantiation', function () { const apikeyToken = 'regular1'; const mapConfig = { From 1c5344ba6e450ea710410d3dacd493910f8134ae Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 14 May 2018 14:54:29 +0200 Subject: [PATCH 113/185] Update Windshaft to 4.7.3 --- NEWS.md | 8 +++++--- package.json | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 74bbe860..2267e7ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,12 +5,14 @@ Released 2018-mm-dd New features: - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 - +- Upgrades Windshaft to [4.7.3](https://github.com/CartoDB/Windshaft/blob/4.7.3/NEWS.md#version-473) which includes: + - A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type. + - Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible). + - MVT: Disable simplify_distance to avoid multiple simplifications. + - Fix a bug with zero length lines not being rendered when using the marker symbolizer. Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. -- Upgrades Windshaft to 4.7.1, which includes a fix in mapnik-vector-tile to avoid grouping together properties with the same value but different type. - ## 6.1.0 Released 2018-04-16 diff --git a/package.json b/package.json index 1387632c..a0ee81dd 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.7.1", + "windshaft": "4.7.3", "yargs": "~5.0.0" }, "devDependencies": { From 0f48d510629d0e8b0ad09bc33f96cae6f6331836 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 14 May 2018 15:01:04 +0200 Subject: [PATCH 114/185] Update yarn.lock --- yarn.lock | 160 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 121 insertions(+), 39 deletions(-) diff --git a/yarn.lock b/yarn.lock index 02cc4826..8491e430 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,20 +2,20 @@ # yarn lockfile v1 -"@carto/mapnik@3.6.2-carto.8": - version "3.6.2-carto.8" - resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.8.tgz#70448689d9b14d644bebd079f5714871c458a46d" +"@carto/mapnik@3.6.2-carto.10": + version "3.6.2-carto.10" + resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.10.tgz#a97c951dcdac09d0eb35b3ea71e5eeaa206c1af6" dependencies: mapnik-vector-tile cartodb/mapnik-vector-tile#v1.6.1-carto.1 nan "2.10.0" - node-pre-gyp "0.7.0" + node-pre-gyp "0.10.0" protozero "1.5.1" -"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb7": - version "2.5.1-cdb7" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/ec881cb9ac52113f895f23857430e2d434bb99a6" +"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb9": + version "2.5.1-cdb9" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/5129e43223cb55daed31373c7a36c98eb6178fc1" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.10" "@mapbox/sphericalmercator" "~1.0.1" mapnik-pool "~0.1.3" @@ -23,11 +23,11 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -abaculus@cartodb/abaculus#2.0.3-cdb8: - version "2.0.3-cdb8" - resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/31c03f2442943d4c47740fa154cda753b5cccd8a" +"abaculus@github:cartodb/abaculus#2.0.3-cdb10": + version "2.0.3-cdb10" + resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/90d537028bb8af8a35e7a40c46493066dd8a76b3" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.10" d3-queue "^2.0.2" sphericalmercator "1.0.x" @@ -249,7 +249,7 @@ camshaft@0.61.8: dot "^1.0.3" request "2.85.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -275,7 +275,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -338,6 +338,10 @@ chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +chownr@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" + chroma-js@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.1.1.tgz#9bb9434959336ece75700aaadfeedc71806d8c05" @@ -455,7 +459,7 @@ debug@2.6.0: dependencies: ms "0.7.2" -debug@2.6.9, debug@^2.2.0: +debug@2.6.9, debug@^2.1.2, debug@^2.2.0: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -748,6 +752,12 @@ fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" +fs-minipass@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d" + dependencies: + minipass "^2.2.1" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1005,6 +1015,18 @@ iconv-lite@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" +iconv-lite@^0.4.4: + version "0.4.23" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" + dependencies: + safer-buffer ">= 2.1.2 < 3" + +ignore-walk@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.1.tgz#a83e62e7d272ac0e3b551aaa82831a19b69f82f8" + dependencies: + minimatch "^3.0.4" + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -1292,7 +1314,7 @@ mapnik-reference@~8.5.3: dependencies: semver "^5.1.0" -mapnik-vector-tile@cartodb/mapnik-vector-tile#v1.6.1-carto.1: +"mapnik-vector-tile@github:cartodb/mapnik-vector-tile#v1.6.1-carto.1": version "1.6.1-carto.1" resolved "https://codeload.github.com/cartodb/mapnik-vector-tile/tar.gz/0111f7117946179d62ec7a6eba2f4e9fb355d05e" @@ -1363,6 +1385,19 @@ minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" +minipass@^2.2.1, minipass@^2.2.4: + version "2.3.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.0.tgz#2e11b1c46df7fe7f1afbe9a490280add21ffe384" + dependencies: + safe-buffer "^5.1.1" + yallist "^3.0.0" + +minizlib@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.0.tgz#11e13658ce46bc3a70a267aac58359d1e0c29ceb" + dependencies: + minipass "^2.2.1" + mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" @@ -1429,6 +1464,14 @@ ncp@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3" +needle@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/needle/-/needle-2.2.1.tgz#b5e325bd3aae8c2678902fa296f729455d1d3a7d" + dependencies: + debug "^2.1.2" + iconv-lite "^0.4.4" + sax "^1.2.4" + negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -1444,20 +1487,20 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.7.0.tgz#55aeffbaed93b50d0a4657d469198cd80ac9df36" +node-pre-gyp@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.10.0.tgz#6e4ef5bb5c5203c6552448828c852c40111aac46" dependencies: detect-libc "^1.0.2" mkdirp "^0.5.1" + needle "^2.2.0" nopt "^4.0.1" + npm-packlist "^1.1.6" npmlog "^4.0.2" rc "^1.1.7" - request "2.83.0" rimraf "^2.6.1" semver "^5.3.0" - tar "^2.2.1" - tar-pack "^3.4.0" + tar "^4" node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: version "0.6.39" @@ -1501,6 +1544,17 @@ normalize-package-data@^2.3.2: semver "2 || 3 || 4 || 5" validate-npm-package-license "^3.0.1" +npm-bundled@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308" + +npm-packlist@^1.1.6: + version "1.1.10" + resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.10.tgz#1039db9e985727e464df066f4cf0ab6ef85c398a" + dependencies: + ignore-walk "^3.0.1" + npm-bundled "^1.0.1" + npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" @@ -1904,9 +1958,9 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.83.0, request@2.x, request@^2.55.0: - version "2.83.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" +request@2.85.0: + version "2.85.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1931,9 +1985,9 @@ request@2.83.0, request@2.x, request@^2.55.0: tunnel-agent "^0.6.0" uuid "^3.1.0" -request@2.85.0: - version "2.85.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa" +request@2.x, request@^2.55.0: + version "2.83.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" dependencies: aws-sign2 "~0.7.0" aws4 "^1.6.0" @@ -1992,10 +2046,22 @@ safe-buffer@5.1.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, s version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" +safe-buffer@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" +"safer-buffer@>= 2.1.2 < 3": + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + +sax@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + "semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -2247,6 +2313,18 @@ tar@^2.2.1: fstream "^1.0.2" inherits "2" +tar@^4: + version "4.4.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.2.tgz#60685211ba46b38847b1ae7ee1a24d744a2cd462" + dependencies: + chownr "^1.0.1" + fs-minipass "^1.2.5" + minipass "^2.2.4" + minizlib "^1.1.0" + mkdirp "^0.5.0" + safe-buffer "^5.1.2" + yallist "^3.0.2" + through2@~0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/through2/-/through2-0.2.3.tgz#eb3284da4ea311b6cc8ace3653748a52abf25a3f" @@ -2258,11 +2336,11 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb12: - version "0.6.18-cdb12" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/e3c0d80e604ca4a5dfad648ee6f6fb355d415a88" +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb14": + version "0.6.18-cdb14" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/6d06f728833d3e34d1adcd05567b3f4379f547bb" dependencies: - "@carto/mapnik" "3.6.2-carto.8" + "@carto/mapnik" "3.6.2-carto.10" generic-pool "~2.4.0" mime "~1.6.0" sphericalmercator "~1.0.4" @@ -2419,13 +2497,13 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.7.1: - version "4.7.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.1.tgz#2b92753b2f6e97b239e15e1576ec312cc7dfeb13" +windshaft@4.7.3: + version "4.7.3" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.3.tgz#86b6c9ae21f5ff9ad7a37bc344151e3d13da4f06" dependencies: - "@carto/mapnik" "3.6.2-carto.8" - "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb7 - abaculus cartodb/abaculus#2.0.3-cdb8 + "@carto/mapnik" "3.6.2-carto.10" + "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb9 + abaculus cartodb/abaculus#2.0.3-cdb10 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 cartodb-psql "^0.10.1" @@ -2439,7 +2517,7 @@ windshaft@4.7.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb12 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb14 torque.js "~2.11.0" underscore "~1.6.0" @@ -2480,6 +2558,10 @@ y18n@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" +yallist@^3.0.0, yallist@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" + yargs-parser@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4" From 56aa1b39f008d82edcc7a4c2bcf1ab03abd0af51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 16 May 2018 12:00:40 +0200 Subject: [PATCH 115/185] removing 'jpeg' as valid format and no checking agains lowercase --- lib/cartodb/api/middlewares/check-static-image-format.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/api/middlewares/check-static-image-format.js b/lib/cartodb/api/middlewares/check-static-image-format.js index d72fcb81..9b9c8b24 100644 --- a/lib/cartodb/api/middlewares/check-static-image-format.js +++ b/lib/cartodb/api/middlewares/check-static-image-format.js @@ -1,8 +1,8 @@ -const VALID_IMAGE_FORMATS = ['png', 'jpg', 'jpeg']; +const VALID_IMAGE_FORMATS = ['png', 'jpg']; module.exports = function checkStaticImageFormat () { return function checkStaticImageFormatMiddleware (req, res, next) { - if(!VALID_IMAGE_FORMATS.includes(req.params.format.toLowerCase())) { + if(!VALID_IMAGE_FORMATS.includes(req.params.format)) { return next(new Error(`Unsupported image format "${req.params.format}"`)); } From 3af118220644de57cefebf331987ae5c13f02ef0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 May 2018 14:45:19 +0200 Subject: [PATCH 116/185] Rename misleading function argument --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 6a71435e..50e3bc16 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -35,10 +35,10 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; -function queryPromise(dbConnection, query, callback) { +function queryPromise(dbConnection, query, setResults) { return new Promise(function(resolve, reject) { dbConnection.query(query, function (err, res) { - err = callback(err, res); + err = setResults(err, res); if (err) { reject(err); } From 012fa91e83407051aa19938dd8730392f1cc8967 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 May 2018 14:45:34 +0200 Subject: [PATCH 117/185] Typo --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 50e3bc16..f9e0000f 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -71,7 +71,7 @@ function columnAggregations(field) { function addStat(queries, ctx, type, zoom, query, assign, assignDefault=null) { let sql; if (type === 'pre') { - sql = ctx. preQuery; + sql = ctx.preQuery; } else { sql = queryForZoom(ctx.aggrQuery, zoom); From d9e047e20e2fc511ed2ab508f25b0defad4a33a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Thu, 17 May 2018 12:44:31 +0200 Subject: [PATCH 118/185] NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 2267e7ac..ebd6b2d6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -13,6 +13,8 @@ New features: Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. +- Static maps fails for unsupported formats +- Handling errors extracting the column type on dataviews ## 6.1.0 Released 2018-04-16 From d76a0d9f22e5c001db48c0c67c267b2f59e5c6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 17 May 2018 13:47:52 +0200 Subject: [PATCH 119/185] Upgrade camshaft to version 0.61.9 --- package.json | 2 +- yarn.lock | 38 +++++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index a0ee81dd..e7350e01 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "basic-auth": "^2.0.0", "body-parser": "^1.18.2", - "camshaft": "0.61.8", + "camshaft": "0.61.9", "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", "cartodb-redis": "1.0.0", diff --git a/yarn.lock b/yarn.lock index 8491e430..57201e3d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11,7 +11,7 @@ node-pre-gyp "0.10.0" protozero "1.5.1" -"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb9": +"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb9": version "2.5.1-cdb9" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/5129e43223cb55daed31373c7a36c98eb6178fc1" dependencies: @@ -23,7 +23,7 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -"abaculus@github:cartodb/abaculus#2.0.3-cdb10": +abaculus@cartodb/abaculus#2.0.3-cdb10: version "2.0.3-cdb10" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/90d537028bb8af8a35e7a40c46493066dd8a76b3" dependencies: @@ -238,9 +238,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.61.8: - version "0.61.8" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.8.tgz#75669c6c14791a93433e79a8892298e88cb0fce2" +camshaft@0.61.9: + version "0.61.9" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.9.tgz#f3d399dfacf51b6a492c579e925c8b1141b22974" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -249,7 +249,7 @@ camshaft@0.61.8: dot "^1.0.3" request "2.85.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -275,7 +275,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -555,7 +555,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -1314,7 +1318,7 @@ mapnik-reference@~8.5.3: dependencies: semver "^5.1.0" -"mapnik-vector-tile@github:cartodb/mapnik-vector-tile#v1.6.1-carto.1": +mapnik-vector-tile@cartodb/mapnik-vector-tile#v1.6.1-carto.1: version "1.6.1-carto.1" resolved "https://codeload.github.com/cartodb/mapnik-vector-tile/tar.gz/0111f7117946179d62ec7a6eba2f4e9fb355d05e" @@ -1420,7 +1424,11 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@~2.18.1: +moment@^2.10.6: + version "2.22.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.22.1.tgz#529a2e9bf973f259c9643d237fda84de3a26e8ad" + +moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1440,11 +1448,11 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@2.10.0: +nan@2.10.0, nan@^2.0.8: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0: +nan@^2.3.4, nan@^2.4.0: version "2.8.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" @@ -2051,8 +2059,8 @@ safe-buffer@^5.1.2: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" safe-json-stringify@~1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" + version "1.1.0" + resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz#bd2b6dad1ebafab3c24672a395527f01804b7e19" "safer-buffer@>= 2.1.2 < 3": version "2.1.2" @@ -2336,7 +2344,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb14": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb14: version "0.6.18-cdb14" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/6d06f728833d3e34d1adcd05567b3f4379f547bb" dependencies: From dd0436e68e9d9443d1179523543649e7c11f5ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 17 May 2018 13:55:57 +0200 Subject: [PATCH 120/185] Update NEWS --- NEWS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ebd6b2d6..141e082e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,11 +10,13 @@ New features: - Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible). - MVT: Disable simplify_distance to avoid multiple simplifications. - Fix a bug with zero length lines not being rendered when using the marker symbolizer. +- Upgrades Camshaft to [0.61.9](https://github.com/CartoDB/camshaft/releases/tag/0.61.9): + - Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses. Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. - Static maps fails for unsupported formats -- Handling errors extracting the column type on dataviews +- Handling errors extracting the column type on dataviews ## 6.1.0 Released 2018-04-16 From 4bc8fb207ab05395e780b0e6a4cae814b325922c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 18 May 2018 15:29:46 +0200 Subject: [PATCH 121/185] Use sql_raw for query without aggregation --- .../backends/layer-stats/mapnik-layer-stats.js | 7 ++----- .../adapter/aggregation-mapconfig-adapter.js | 17 ++++++++--------- .../mapconfig/adapter/turbo-carto-adapter.js | 4 +++- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index f9e0000f..365bb3a5 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -275,16 +275,13 @@ function formatResultFields(dbConnection, flds) { MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - let aggrQuery = layer.options.sql_raw || layer.options.sql; - let preQuery = layer.options.aggregation_metadata ? - layer.options.aggregation_metadata.pre_aggregation_sql : - aggrQuery; + let aggrQuery = layer.options.sql; + let preQuery = layer.options.sql_raw || aggrQuery; let context = { dbConnection, preQuery, aggrQuery, - aggrMeta: layer.options.aggregation_metadata, metaOptions: layer.options.metadata || {} }; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index b3df3d96..9752ab57 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -84,7 +84,7 @@ module.exports = class AggregationMapConfigAdapter { _adaptLayer (connection, mapConfig, layer, index) { return new Promise((resolve, reject) => { - this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt, aggrMeta) => { + this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => { if (err) { return reject(err); } @@ -93,7 +93,6 @@ module.exports = class AggregationMapConfigAdapter { return resolve({ layer, index, adapted: shouldAdapt }); } - const sqlQuery = layer.options.sql; const sqlQueryWrap = layer.options.sql_wrap; let aggregationSql = mapConfig.getAggregatedQuery(index); @@ -102,6 +101,12 @@ module.exports = class AggregationMapConfigAdapter { aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); } + if (!layer.options.sql_raw) { + // if sql_wrap is present, the original query should already be + // in sql_raw (with sql being the wrapped query); + // otherwise we keep the now the original query in sql_raw + layer.options.sql_raw = layer.options.sql; + } layer.options.sql = aggregationSql; mapConfig.getLayerAggregationColumns(index, (err, columns) => { @@ -111,12 +116,6 @@ module.exports = class AggregationMapConfigAdapter { layer.options.columns = columns; - layer.options.aggregation_metadata = { - pre_aggregation_sql: sqlQueryWrap || sqlQuery, - geometry_type: aggrMeta.type, - pre_aggregation_count: aggrMeta.count - }; - return resolve({ layer, index, adapted: shouldAdapt }); }); }); @@ -161,7 +160,7 @@ module.exports = class AggregationMapConfigAdapter { return callback(null, false); } - callback(null, true, result); + callback(null, true); }); } diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index 2bd30152..31c4f7c4 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -117,7 +117,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, var layerSql = layer.options.sql; var layerRawSql = layer.options.sql_raw; - if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) { + if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql && layer.options.sql_wrap) { + // For wrapped queries we'll derive the tokens from the data extent + // instead of the whole Earth/root tile. var self = this; var tokensQuery = tokensQueryTpl({_sql: layerRawSql}); return pg.query(tokensQuery, function(err, resultSet) { From 391ac51f0f1d40d4fd58aafee1ab7ef2d452d57c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 18 May 2018 15:33:07 +0200 Subject: [PATCH 122/185] Implement metadata queries with plain Promises Remove usage of PhasedExecution This achives better query execution granularity and removes questionable usage of shared results object. It introduces a couple of behavior changes: * estimatedFeatureCount desn't ignore errors now * sample always uses estimatedFeatureCount,even if the actual count is also computed. --- .../layer-stats/mapnik-layer-stats.js | 251 +++++++++--------- lib/cartodb/utils/phased-execution.js | 94 ------- test/acceptance/aggregation.js | 4 +- 3 files changed, 129 insertions(+), 220 deletions(-) delete mode 100644 lib/cartodb/utils/phased-execution.js diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 365bb3a5..36f4824a 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,5 +1,4 @@ var queryUtils = require('../../utils/query-utils'); -const PhasedExecution = require('../../utils/phased-execution'); const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); var SubstitutionTokens = require('../../utils/substitution-tokens'); @@ -35,15 +34,14 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; -function queryPromise(dbConnection, query, setResults) { +function queryPromise(dbConnection, query, adaptResults) { return new Promise(function(resolve, reject) { dbConnection.query(query, function (err, res) { - err = setResults(err, res); if (err) { reject(err); } else { - resolve(); + resolve(adaptResults(res)); } }); @@ -60,15 +58,7 @@ function columnAggregations(field) { return []; } -/* Helper to add a task to the queries PhasedExecution - * type can be either 'pre' (for pre-aggregation metadata) or 'post' - * zoom is used only for post-aggregation metadata - * query is a function that generates a metadata query from a data query - * assign is a function to assign the results of the metadata query - * if a assignDefault function is present, it will be used in case of error - * during the query execution and any errors will be ignored - */ -function addStat(queries, ctx, type, zoom, query, assign, assignDefault=null) { +function _getSQL(ctx, type, zoom, query) { let sql; if (type === 'pre') { sql = ctx.preQuery; @@ -76,157 +66,164 @@ function addStat(queries, ctx, type, zoom, query, assign, assignDefault=null) { else { sql = queryForZoom(ctx.aggrQuery, zoom); } - sql = query(sql); - queries.task( - queryPromise( - ctx.dbConnection, - sql, - (err, res) => { - if (!err) { - assign(res); - } - else if (assignDefault !== null) { - assignDefault(); - return null; - } - return err; - } - ) + return query(sql); +} + +function _estimatedFeatureCount(ctx) { + // TODO: restore -1 on errors behavior? + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, queryUtils.getQueryRowEstimation), + res => ({ estimatedFeatureCount: res.rows[0].rows }) ); } -function _estimatedFeatureCount(queries, ctx) { - if (queries.results.estimatedFeatureCount === undefined) { - // This is always computed; a default value of -1 is used in case of error - addStat( - queries, - ctx, - 'pre', 0, - queryUtils.getQueryRowEstimation, - res => queries.results.estimatedFeatureCount = res.rows[0].rows, - () => queries.results.estimatedFeatureCount = -1 - ); - } -} - -function _featureCount(queries, ctx) { +function _featureCount(ctx) { if (ctx.metaOptions.featureCount) { // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query - addStat( - queries, - ctx, - 'pre', 0, - queryUtils.getQueryActualRowCount, - res => queries.results.featureCount = res.rows[0].rows + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, queryUtils.getQueryActualRowCount), + res => ({ featureCount: res.rows[0].rows }) ); } + return Promise.resolve(); } -function _aggrFeatureCount(queries, ctx) { +function _aggrFeatureCount(ctx) { if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { // We expect as zoom level as the value of aggrFeatureCount // TODO: it'd be nice to admit an array of zoom levels to // return metadata for multiple levels. - addStat( - queries, - ctx, - 'post', ctx.metaOptions.aggrFeatureCount || 0, - queryUtils.getQueryActualRowCount, - res => queries.results.aggrfeatureCount = res.rows[0].rows + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'post', ctx.metaOptions.aggrFeatureCount || 0, queryUtils.getQueryActualRowCount), + res => ({ aggrFeatureCount: res.rows[0].rows }) ); } + return Promise.resolve(); } -function _geometryType(queries, ctx) { - if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { +function _geometryType(ctx) { + if (ctx.metaOptions.geometryType) { const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); - addStat( - queries, - ctx, - 'pre', 0, - sql => queryUtils.getQueryGeometryType(sql, geometryColumn), - res => queries.results.geometryType = res.rows[0].geom_type + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)), + res => ({ geometryType: res.rows[0].geom_type }) ); } + return Promise.resolve(); } -function _columns(queries, ctx) { +function _columns(ctx) { if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { // note: post-aggregation columns are in layer.options.columns when aggregation is present - addStat( - queries, - ctx, - 'pre', 0, - sql => queryUtils.getQueryLimited(sql, 0), - res => queries.results.columns = formatResultFields(ctx.dbConnection, res.fields) + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, sql => queryUtils.getQueryLimited(sql, 0)), + res => formatResultFields(ctx.dbConnection, res.fields) ); } + return Promise.resolve(); +} + +// combine a list of results merging the properties of all the objects +// undefined results are admitted and ignored +function mergeResults(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + if (a === undefined) { + return b; + } + if (b === undefined) { + return a; + } + return Object.assign({}, a, b); + }); + } } -function firstPhaseQueries(queries, ctx) { - _estimatedFeatureCount(queries, ctx); - _featureCount(queries, ctx); - _aggrFeatureCount(queries, ctx); - _geometryType(queries, ctx); - _columns(queries, ctx); +// deeper (1 level) combination of a list of objects: +// mergeColumns([{ col1: { a: 1 }, col2: { a: 2 } }, { col1: { b: 3 } }]) => { col1: { a: 1, b: 3 }, col2: { a: 2 } } +function mergeColumns(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + let c = Object.assign({}, b || {}, a || {}); + Object.keys(c).forEach(key => { + if (b.hasOwnProperty(key)) { + c[key] = Object.assign(c[key], b[key]); + } + }); + return c; + }); + } } -function _sample(queries, ctx) { + +function _sample(ctx, numRows) { if (ctx.metaOptions.sample) { - const numRows = queries.results.featureCount === undefined ? - queries.results.estimatedFeatureCount : - queries.results.featureCount; const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); - addStat( - queries, - ctx, - 'pre', 0, - sql => queryUtils.getQuerySample(sql, sampleProb), - res => queries.results.sample = res.rows + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, sql => queryUtils.getQuerySample(sql, sampleProb)), + res => ({ sample: res.rows }) ); } + return Promise.resolve(); } -function _columnStats(queries, ctx) { +function _columnStats(ctx, columns) { + if (!columns) { + return Promise.resolve(); + } if (ctx.metaOptions.columnStats) { + let queries = []; let aggr = []; - Object.keys(queries.results.columns).forEach(name => { + queries.push(new Promise(resolve => resolve(columns))); // add columns as first result + Object.keys(columns).forEach(name => { aggr = aggr.concat( - columnAggregations(queries.results.columns[name]) + columnAggregations(columns[name]) .map(fn => `${fn}(${name}) AS ${name}_${fn}`) ); - if (queries.results.columns[name].type === 'string') { + if (columns[name].type === 'string') { const topN = ctx.metaOptions.columnStats.topCategories || 1024; // TODO: ctx.metaOptions.columnStats.maxCategories // => use PG stats to dismiss columns with more distinct values - addStat( - queries, - ctx, - 'pre', 0, - sql => queryUtils.getQueryTopCategories(sql, name, topN), - res => queries.results.columns[name].categories = res.rows + queries.push( + queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, sql => queryUtils.getQueryTopCategories(sql, name, topN)), + res => ({ [name]: { categories: res.rows } }) + ) ); } }); - addStat( - queries, - ctx, - 'pre', 0, - sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`, - res => { - Object.keys(queries.results.columns).forEach(name => { - columnAggregations(queries.results.columns[name]).forEach(fn => { - queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`]; + queries.push( + queryPromise( + ctx.dbConnection, + _getSQL(ctx, 'pre', 0, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`), + res => { + let stats = {}; + Object.keys(columns).forEach(name => { + stats[name] = {}; + columnAggregations(columns[name]).forEach(fn => { + stats[name][fn] = res.rows[0][`${name}_${fn}`]; + }); }); - }); - } + return stats; + } + ) ); + return Promise.all(queries).then(results => ({ columns: mergeColumns(results) })); } -} - -function secondPhaseQueries(queries, ctx) { - _sample(queries, ctx); - _columnStats(queries, ctx); + return Promise.resolve({ columns }); } // This is adapted from SQL API: @@ -278,27 +275,33 @@ function (layer, dbConnection, callback) { let aggrQuery = layer.options.sql; let preQuery = layer.options.sql_raw || aggrQuery; - let context = { + let ctx = { dbConnection, preQuery, aggrQuery, metaOptions: layer.options.metadata || {} }; - let queries = new PhasedExecution(); - // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere // we would set queries.results.estimatedFeatureCount and queries.results.geometryType // (if metaOptions.geometryType) from it. - // Queries will be executed in two phases, with results from the first phase needed - // to define the queries of the second phase - queries.phase(() => firstPhaseQueries(queries, context)); - queries.phase(() => secondPhaseQueries(queries, context)); - queries.run() - .then(results => callback(null, results)) - .catch(error => callback(error)); + // TODO: compute _sample with _featureCount when available + Promise.all([ + _estimatedFeatureCount(ctx).then( + ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount) + .then(s => mergeResults([s, { estimatedFeatureCount }])) + ), + _featureCount(ctx), + _aggrFeatureCount(ctx), + _geometryType(ctx), + _columns(ctx).then(columns => _columnStats(ctx, columns)) + ]).then(results => { + callback(null, mergeResults(results)); + }).catch(error => { + callback(error); + }); }; module.exports = MapnikLayerStats; diff --git a/lib/cartodb/utils/phased-execution.js b/lib/cartodb/utils/phased-execution.js deleted file mode 100644 index 46dfbb39..00000000 --- a/lib/cartodb/utils/phased-execution.js +++ /dev/null @@ -1,94 +0,0 @@ -/** - * PhasedExecution handles the execution of async tasks (via Promises) - * which have dependencies between them in a simplified manner. - * Instead of using the complete task dependency graph, tasks - * are organized into execution phases. So that tasks from a latter - * phase will be initialized after tasks from previous phases have - * finished. - * - * All tasks place their results in a shared object to make them - * available to tasks of latter phases. - * - * Each phase is defined by a function that defines its tasks. - * - * Example: - * - * let p = new PhasedExecution(); - * // Define first phase with tasks 1 & 2 - * p.phase(() => { - * console.log('At phase I', p.results);* - * p.results.phase1 = 1 - * p.task(new Promise((resolve) => { - * setTimeout( () => { - * console.log('At task 1:', p.results); - * p.results.task1 = 100; - * resolve(); - * }, 400); - * })); - * p.task(new Promise((resolve) => { - * setTimeout( () => { - * console.log('At task 2:', p.results); - * p.results.task2 = 200; - * resolve(); - * }, 100); - * })); - * }); - * // Define second phase with tasks 3 & 4 - * p.phase(() => { - * console.log('At phase II', p.results); - * p.results.phase2 = 2 - * p.task(new Promise((resolve) => { - * setTimeout( () => { - * console.log('At task 3:', p.results); - * p.results.task3 = 300; - * resolve(); - * }, 50); - * })); - * p.task(new Promise((resolve) => { - * setTimeout( () => { - * console.log('At task 4:', p.results); - * p.results.task4 = 400; - * resolve(); - * }, 100); - * })); - * }); - * // Define third phase with task 5 - * p.phase(() => { - * console.log('At phase III', p.results); - * p.results.phase3 = 3 - * p.task(new Promise((resolve) => { - * setTimeout( () => { - * console.log('At task 5:', p.results); - * p.results.task5 = 500; - * resolve(); - * }, 50); - * })); - * }); - * // Execute all tasks - * p.run().then((results) => { - * console.log("RESULTS:", results); - * }).catch((err) => { - * console.log("ERROR:", error); - * }); - */ -module.exports = class PhasedExecution { - constructor() { - this.results = {}; - this.phases = []; - } - phase(phasegenerator) { - this.phases.push(phasegenerator); - } - task(promise) { - this.tasks.push(promise); - } - run() { - this.tasks = []; - let phase = this.phases.shift(); - if (phase) { - phase(this); - return Promise.all(this.tasks).then(() => this.run()); - } - return this.results; - } -}; diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 6aab8d95..96401f15 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -2261,7 +2261,7 @@ describe('aggregation', function () { assert.equal(typeof body.metadata, 'object'); assert.ok(Array.isArray(body.metadata.layers)); assert.ok(body.metadata.layers[0].meta.aggregation.mvt); - assert.equal(body.metadata.layers[0].meta.stats.aggrfeatureCount, 13); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 13); done(); }); @@ -2302,7 +2302,7 @@ describe('aggregation', function () { assert.equal(typeof body.metadata, 'object'); assert.ok(Array.isArray(body.metadata.layers)); assert.ok(body.metadata.layers[0].meta.aggregation.mvt); - assert.equal(body.metadata.layers[0].meta.stats.aggrfeatureCount, 9); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 9); done(); }); From 4e99ff1c39c7389008e886a519d3938bb217b089 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 18 May 2018 22:25:32 +0200 Subject: [PATCH 123/185] Fix token substitution for stat queries --- .../layer-stats/mapnik-layer-stats.js | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 36f4824a..e3d11070 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -58,14 +58,15 @@ function columnAggregations(field) { return []; } -function _getSQL(ctx, type, zoom, query) { +function _getSQL(ctx, query, type='pre', zoom=0) { let sql; if (type === 'pre') { sql = ctx.preQuery; } else { - sql = queryForZoom(ctx.aggrQuery, zoom); + sql = ctx.aggrQuery; } + sql = queryForZoom(sql, zoom || 0); return query(sql); } @@ -73,7 +74,7 @@ function _estimatedFeatureCount(ctx) { // TODO: restore -1 on errors behavior? return queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, queryUtils.getQueryRowEstimation), + _getSQL(ctx, queryUtils.getQueryRowEstimation), res => ({ estimatedFeatureCount: res.rows[0].rows }) ); } @@ -83,7 +84,7 @@ function _featureCount(ctx) { // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query return queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, queryUtils.getQueryActualRowCount), + _getSQL(ctx, queryUtils.getQueryActualRowCount), res => ({ featureCount: res.rows[0].rows }) ); } @@ -97,7 +98,7 @@ function _aggrFeatureCount(ctx) { // return metadata for multiple levels. return queryPromise( ctx.dbConnection, - _getSQL(ctx, 'post', ctx.metaOptions.aggrFeatureCount || 0, queryUtils.getQueryActualRowCount), + _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount), res => ({ aggrFeatureCount: res.rows[0].rows }) ); } @@ -109,7 +110,7 @@ function _geometryType(ctx) { const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); return queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)), + _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)), res => ({ geometryType: res.rows[0].geom_type }) ); } @@ -121,7 +122,7 @@ function _columns(ctx) { // note: post-aggregation columns are in layer.options.columns when aggregation is present return queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, sql => queryUtils.getQueryLimited(sql, 0)), + _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0)), res => formatResultFields(ctx.dbConnection, res.fields) ); } @@ -172,7 +173,7 @@ function _sample(ctx, numRows) { const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); return queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, sql => queryUtils.getQuerySample(sql, sampleProb)), + _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb)), res => ({ sample: res.rows }) ); } @@ -199,7 +200,7 @@ function _columnStats(ctx, columns) { queries.push( queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, sql => queryUtils.getQueryTopCategories(sql, name, topN)), + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN)), res => ({ [name]: { categories: res.rows } }) ) ); @@ -208,7 +209,7 @@ function _columnStats(ctx, columns) { queries.push( queryPromise( ctx.dbConnection, - _getSQL(ctx, 'pre', 0, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`), + _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`), res => { let stats = {}; Object.keys(columns).forEach(name => { From 9daaf5bb6ac63dfd901cea668f9c0f776d2ec1cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 10:15:22 +0200 Subject: [PATCH 124/185] Use development branch of windshaft --- package.json | 2 +- yarn.lock | 163 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 122 insertions(+), 43 deletions(-) diff --git a/package.json b/package.json index e7350e01..94fdb836 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.7.3", + "windshaft": "cartodb/windshaft#upgrade-dependencies", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 57201e3d..4efe941a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -267,14 +267,6 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -283,6 +275,14 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#master": + version "0.15.1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/31abb8bee02df605521247b0223d508320f7d4c8" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "1.8.3" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -443,6 +443,10 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" +d3@3.5.17: + version "3.5.17" + resolved "https://registry.yarnpkg.com/d3/-/d3-3.5.17.tgz#bc46748004378b21a360c9fc7cf5231790762fb8" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -453,6 +457,12 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" +debug@2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" + dependencies: + ms "0.7.1" + debug@2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" @@ -465,15 +475,15 @@ debug@2.6.9, debug@^2.1.2, debug@^2.2.0: dependencies: ms "2.0.0" -debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" +debug@3.1.0, debug@^3.1.0, debug@~3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" dependencies: ms "2.0.0" -debug@^3.1.0, debug@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" +debug@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" dependencies: ms "2.0.0" @@ -555,7 +565,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: +dot@1.1.2, dot@^1.0.3: version "1.1.2" resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" @@ -1432,6 +1442,10 @@ moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" +ms@0.7.1: + version "0.7.1" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" + ms@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" @@ -1839,6 +1853,10 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +queue-async@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.1.0.tgz#d5f88e32fd01fee4f6d946d21d17abd16ac65f35" + queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" @@ -1993,6 +2011,31 @@ request@2.85.0: tunnel-agent "^0.6.0" uuid "^3.1.0" +request@2.87.0: + version "2.87.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.87.0.tgz#32f00235cd08d482b4d0d68db93a829c0ed5756e" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + tough-cookie "~2.3.3" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + request@2.x, request@^2.55.0: version "2.83.0" resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356" @@ -2078,6 +2121,10 @@ semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" +semver@5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" @@ -2133,6 +2180,10 @@ signal-exit@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" +simple-statistics@^0.9.0: + version "0.9.2" + resolved "https://registry.yarnpkg.com/simple-statistics/-/simple-statistics-0.9.2.tgz#3e35cb10308fc76963aa4ee7252e6a6ea10d28e4" + single-line-log@~0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/single-line-log/-/single-line-log-0.3.1.tgz#a7ad6507f218ce5dfe16c4bf2d659246419e7a06" @@ -2183,7 +2234,11 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" @@ -2234,6 +2289,10 @@ step-profiler@~0.3.0: dependencies: debug "^2.2.0" +step@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" + step@~0.0.5, step@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" @@ -2354,20 +2413,23 @@ tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb14: sphericalmercator "~1.0.4" step "~0.0.5" -tilelive@5.12.2: - version "5.12.2" - resolved "https://registry.yarnpkg.com/tilelive/-/tilelive-5.12.2.tgz#9ede3aefded4b04e0be32e0dfc6a1ce01247eef7" +tilelive@5.12.3: + version "5.12.3" + resolved "https://registry.yarnpkg.com/tilelive/-/tilelive-5.12.3.tgz#9c8c770e1194aa8d353d9a1d03147404307cdebd" dependencies: minimist "~0.2.0" progress-stream "~0.5.x" queue-async "~1.0.7" sphericalmercator "~1.0.1" -torque.js@~2.11.0: - version "2.11.5" - resolved "https://registry.yarnpkg.com/torque.js/-/torque.js-2.11.5.tgz#8fe8dfb53fb76497a3c427fcb7ba18e427c1ef6b" +torque.js@2.16.2: + version "2.16.2" + resolved "https://registry.yarnpkg.com/torque.js/-/torque.js-2.16.2.tgz#fadeaeb2adce161eca03ea2fe16ef0e80aa02967" dependencies: - carto CartoDB/carto#0.15.1-cdb1 + carto cartodb/carto#master + d3 "3.5.17" + turbo-carto "0.19.0" + turf-jenks "~1.0.1" tough-cookie@~2.3.0, tough-cookie@~2.3.3: version "2.3.3" @@ -2381,6 +2443,17 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turbo-carto@0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.0.tgz#83fb1932acd42acb426312eef216b5f6ac34708e" + dependencies: + cartocolor "4.0.0" + colorbrewer "1.0.0" + debug "2.2.0" + es6-promise "3.1.2" + postcss "5.0.19" + postcss-value-parser "3.3.0" + turbo-carto@0.20.2: version "0.20.2" resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.2.tgz#2b737597a65c2918432f70ea414f12fbec2b6a6f" @@ -2392,6 +2465,12 @@ turbo-carto@0.20.2: postcss "5.0.19" postcss-value-parser "3.3.0" +turf-jenks@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/turf-jenks/-/turf-jenks-1.0.1.tgz#5c72aea13a716eff2f0023a72cfa549a00202d7f" + dependencies: + simple-statistics "^0.9.0" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -2434,6 +2513,10 @@ uid-number@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" +underscore@1.6.0, underscore@~1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" + underscore@1.8.2: version "1.8.2" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.2.tgz#64df2eb590899de950782f3735190ba42ebf311d" @@ -2442,10 +2525,6 @@ underscore@1.8.3: version "1.8.3" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" -underscore@~1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.6.0.tgz#8b38b10cacdef63337b8b24e4ff86d45aea529a8" - unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -2505,29 +2584,29 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.7.3: - version "4.7.3" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.7.3.tgz#86b6c9ae21f5ff9ad7a37bc344151e3d13da4f06" +windshaft@cartodb/windshaft#upgrade-dependencies: + version "4.7.4" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/cdb544257b17afc8f6c2c8a3438a6294c52385ba" dependencies: "@carto/mapnik" "3.6.2-carto.10" "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb9 abaculus cartodb/abaculus#2.0.3-cdb10 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "^0.10.1" - debug "^3.1.0" - dot "~1.0.2" + cartodb-psql "0.10.2" + debug "3.1.0" + dot "1.1.2" grainstore "1.9.0" - queue-async "~1.0.7" - redis-mpool "^0.5.0" - request "2.85.0" - semver "~5.0.3" - sphericalmercator "1.0.4" - step "~0.0.6" - tilelive "5.12.2" + queue-async "1.1.0" + redis-mpool "0.5.0" + request "2.87.0" + semver "5.5.0" + sphericalmercator "1.0.5" + step "1.0.0" + tilelive "5.12.3" tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb14 - torque.js "~2.11.0" - underscore "~1.6.0" + torque.js "2.16.2" + underscore "1.6.0" wordwrap@0.0.2: version "0.0.2" From 8479198268b1e30b872796e3dcc7e3a1254e39c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 10:25:04 +0200 Subject: [PATCH 125/185] Upgrade body-parser version to 1.18.3 --- package.json | 2 +- yarn.lock | 66 ++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 94fdb836..0b599ead 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ ], "dependencies": { "basic-auth": "^2.0.0", - "body-parser": "^1.18.2", + "body-parser": "1.18.3", "camshaft": "0.61.9", "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", diff --git a/yarn.lock b/yarn.lock index 4efe941a..d5a69815 100644 --- a/yarn.lock +++ b/yarn.lock @@ -165,7 +165,7 @@ block-stream@*: dependencies: inherits "~2.0.0" -body-parser@1.18.2, body-parser@^1.18.2: +body-parser@1.18.2: version "1.18.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.2.tgz#87678a19d84b47d859b83199bd59bce222b10454" dependencies: @@ -180,6 +180,21 @@ body-parser@1.18.2, body-parser@^1.18.2: raw-body "2.3.2" type-is "~1.6.15" +body-parser@1.18.3: + version "1.18.3" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.18.3.tgz#5b292198ffdd553b3a0f20ded0592b956955c8b4" + dependencies: + bytes "3.0.0" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.2" + http-errors "~1.6.3" + iconv-lite "0.4.23" + on-finished "~2.3.0" + qs "6.5.2" + raw-body "2.3.3" + type-is "~1.6.16" + boom@2.x.x: version "2.10.1" resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" @@ -275,7 +290,7 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#master": +carto@cartodb/carto#master: version "0.15.1" resolved "https://codeload.github.com/cartodb/carto/tar.gz/31abb8bee02df605521247b0223d508320f7d4c8" dependencies: @@ -521,7 +536,7 @@ depd@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" -depd@~1.1.1: +depd@~1.1.1, depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -1005,6 +1020,15 @@ http-errors@1.6.2, http-errors@~1.6.2: setprototypeof "1.0.3" statuses ">= 1.3.1 < 2" +http-errors@1.6.3, http-errors@~1.6.3: + version "1.6.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.3.tgz#8b55680bb4be283a0b5bf4ea2e38580be1d9320d" + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.0" + statuses ">= 1.4.0 < 2" + http-signature@~1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" @@ -1029,7 +1053,7 @@ iconv-lite@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" -iconv-lite@^0.4.4: +iconv-lite@0.4.23, iconv-lite@^0.4.4: version "0.4.23" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" dependencies: @@ -1363,12 +1387,22 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" +mime-db@~1.33.0: + version "1.33.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" + mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: mime-db "~1.30.0" +mime-types@~2.1.18: + version "2.1.18" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.18.tgz#6f323f60a83d11146f831ff11fd66e2fe5503bb8" + dependencies: + mime-db "~1.33.0" + mime@1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" @@ -1849,6 +1883,10 @@ qs@6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" +qs@6.5.2: + version "6.5.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" + qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" @@ -1874,6 +1912,15 @@ raw-body@2.3.2: iconv-lite "0.4.19" unpipe "1.0.0" +raw-body@2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.3.tgz#1b324ece6b5706e153855bc1148c65bb7f6ea0c3" + dependencies: + bytes "3.0.0" + http-errors "1.6.3" + iconv-lite "0.4.23" + unpipe "1.0.0" + rc@^1.1.7: version "1.2.5" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd" @@ -2283,6 +2330,10 @@ sshpk@^1.7.0: version "1.3.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" +"statuses@>= 1.4.0 < 2": + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + step-profiler@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3" @@ -2496,6 +2547,13 @@ type-is@~1.6.15: media-typer "0.3.0" mime-types "~2.1.15" +type-is@~1.6.16: + version "1.6.16" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.16.tgz#f89ce341541c672b25ee7ae3c73dee3b2be50194" + dependencies: + media-typer "0.3.0" + mime-types "~2.1.18" + uglify-js@^2.6: version "2.8.29" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" From 9b971aa12448cb21d513fc1c1030ec7b780ef3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 10:46:29 +0200 Subject: [PATCH 126/185] Upgrade dot version to 1.1.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0b599ead..3638d5d8 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "cartodb-query-tables": "0.3.0", "cartodb-redis": "1.0.0", "debug": "^3.1.0", - "dot": "~1.0.2", + "dot": "1.1.2", "express": "~4.16.0", "fastly-purge": "~1.0.1", "glob": "^7.1.2", From 73b837f4d93a09a3ae53774038248dd866ce10e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 10:53:29 +0200 Subject: [PATCH 127/185] Upgrade request to version 2.87.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3638d5d8..a942ee0b 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "on-headers": "^1.0.1", "queue-async": "~1.0.7", "redis-mpool": "0.5.0", - "request": "2.85.0", + "request": "2.87.0", "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", From 38e55367b1ec179da31b6b9582c3d9c3bb0f0bfc Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 11:44:52 +0200 Subject: [PATCH 128/185] Revert error behaviour for estimatedFeatureCount Keep current production behavior of ignoreing errors when computing this stat and returning -1. This is done as to no introduce any instability in production at the moment. --- .../backends/layer-stats/mapnik-layer-stats.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index e3d11070..9ce411ed 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -34,11 +34,16 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; -function queryPromise(dbConnection, query, adaptResults) { +function queryPromise(dbConnection, query, adaptResults, errorHandler) { return new Promise(function(resolve, reject) { dbConnection.query(query, function (err, res) { if (err) { - reject(err); + if (errorHandler) { + errorHandler(err); + } + else { + reject(err); + } } else { resolve(adaptResults(res)); @@ -71,11 +76,11 @@ function _getSQL(ctx, query, type='pre', zoom=0) { } function _estimatedFeatureCount(ctx) { - // TODO: restore -1 on errors behavior? return queryPromise( ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryRowEstimation), - res => ({ estimatedFeatureCount: res.rows[0].rows }) + res => ({ estimatedFeatureCount: res.rows[0].rows }), + () => ({ estimatedFeatureCount: -1 }) ); } From a32613c8549868fc79a000bb925aa308e762d321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 11:58:23 +0200 Subject: [PATCH 129/185] Upgrade queue-async to version 1.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a942ee0b..72d4bf2b 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "lzma": "~2.3.2", "node-statsd": "~0.0.7", "on-headers": "^1.0.1", - "queue-async": "~1.0.7", + "queue-async": "1.1.0", "redis-mpool": "0.5.0", "request": "2.87.0", "semver": "~5.3.0", From fecd63e5824545ccb53b1954fa65d2bfe4d0afa0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 11:59:49 +0200 Subject: [PATCH 130/185] Fix bug --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 9ce411ed..689edb9b 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -39,7 +39,7 @@ function queryPromise(dbConnection, query, adaptResults, errorHandler) { dbConnection.query(query, function (err, res) { if (err) { if (errorHandler) { - errorHandler(err); + resolve(errorHandler(err)); } else { reject(err); From 1f3f7b456071ddd75d7e40d2526b0d6a5c5d1109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:04:23 +0200 Subject: [PATCH 131/185] Upgrade semver version to 5.5.0 --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 72d4bf2b..5dfb7786 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "queue-async": "1.1.0", "redis-mpool": "0.5.0", "request": "2.87.0", - "semver": "~5.3.0", + "semver": "5.5.0", "step": "~0.0.6", "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", diff --git a/yarn.lock b/yarn.lock index d5a69815..026107bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2160,7 +2160,7 @@ sax@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" From 15f512a3c7de5054f792bacd6bc586364bf25273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:08:29 +0200 Subject: [PATCH 132/185] Upgrade step version to 1.0.0 --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 5dfb7786..965db876 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "redis-mpool": "0.5.0", "request": "2.87.0", "semver": "5.5.0", - "step": "~0.0.6", + "step": "1.0.0", "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", diff --git a/yarn.lock b/yarn.lock index 026107bd..dfb976f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2344,7 +2344,7 @@ step@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" -step@~0.0.5, step@~0.0.6: +step@~0.0.5: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" From 985e61b3c6e5f6a4492f6d14f2833738584b087a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:16:30 +0200 Subject: [PATCH 133/185] Upgrade strftime version to 0.10.0 --- package.json | 2 +- yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 965db876..cf3c5b5c 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "moment": "~2.18.1", "nock": "~2.11.0", "redis": "2.8.0", - "strftime": "~0.8.2" + "strftime": "0.10.0" }, "scripts": { "lint": "jshint lib test", diff --git a/yarn.lock b/yarn.lock index dfb976f9..d0445d14 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2348,9 +2348,9 @@ step@~0.0.5: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" -strftime@~0.8.2: - version "0.8.4" - resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" +strftime@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.10.0.tgz#b3f0fa419295202a5a289f6d6be9f4909a617193" string-width@^1.0.1, string-width@^1.0.2: version "1.0.2" From 69d5aef59b2539e2c3f7b58792df1aadf704a1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:21:40 +0200 Subject: [PATCH 134/185] Upgrade node-statsd version to 0.1.1 --- package.json | 2 +- yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index cf3c5b5c..6025df49 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "log4js": "cartodb/log4js-node#cdb", "lru-cache": "2.6.5", "lzma": "~2.3.2", - "node-statsd": "~0.0.7", + "node-statsd": "0.1.1", "on-headers": "^1.0.1", "queue-async": "1.1.0", "redis-mpool": "0.5.0", diff --git a/yarn.lock b/yarn.lock index d0445d14..42c9e4ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1574,9 +1574,9 @@ node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: tar "^2.2.1" tar-pack "^3.4.0" -node-statsd@~0.0.7: - version "0.0.7" - resolved "https://registry.yarnpkg.com/node-statsd/-/node-statsd-0.0.7.tgz#96d4bbd21dff2d798b3374a3e9329cfecce3afa8" +node-statsd@0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/node-statsd/-/node-statsd-0.1.1.tgz#27a59348763d0af7a037ac2a031fef3f051013d3" nopt@3.x: version "3.0.6" From 7bafc54280314f327aac0ed8a2ca72a74b93be7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:27:21 +0200 Subject: [PATCH 135/185] Upgrade moment version to 2.22.1 --- package.json | 2 +- yarn.lock | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 6025df49..4f68143f 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "istanbul": "~0.4.3", "jshint": "~2.9.4", "mocha": "~3.4.1", - "moment": "~2.18.1", + "moment": "2.22.1", "nock": "~2.11.0", "redis": "2.8.0", "strftime": "0.10.0" diff --git a/yarn.lock b/yarn.lock index 42c9e4ca..db0cb547 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1468,14 +1468,10 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6: +moment@2.22.1, moment@^2.10.6: version "2.22.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.22.1.tgz#529a2e9bf973f259c9643d237fda84de3a26e8ad" -moment@~2.18.1: - version "2.18.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" - ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" From 1e9ec9e0536c3f7dccddadc4777a7674b4e61d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:30:28 +0200 Subject: [PATCH 136/185] Upgrade express version to 4.16.3 --- package.json | 2 +- yarn.lock | 88 +++++++++++++++++++++++++++------------------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/package.json b/package.json index 4f68143f..9c28e30e 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "cartodb-redis": "1.0.0", "debug": "^3.1.0", "dot": "1.1.2", - "express": "~4.16.0", + "express": "4.16.3", "fastly-purge": "~1.0.1", "glob": "^7.1.2", "log4js": "cartodb/log4js-node#cdb", diff --git a/yarn.lock b/yarn.lock index db0cb547..be09f92f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -35,11 +35,11 @@ abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" -accepts@~1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.4.tgz#86246758c7dd6d21a6474ff084a4740ec05eb21f" +accepts@~1.3.5: + version "1.3.5" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.5.tgz#eb777df6011723a3b14e8a72c0805c8e86746bd2" dependencies: - mime-types "~2.1.16" + mime-types "~2.1.18" negotiator "0.6.1" ajv@^4.9.1: @@ -608,7 +608,7 @@ ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" -encodeurl@~1.0.1: +encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -673,11 +673,11 @@ exit@0.1.2, exit@0.1.x: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" -express@~4.16.0: - version "4.16.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" +express@4.16.3: + version "4.16.3" + resolved "https://registry.yarnpkg.com/express/-/express-4.16.3.tgz#6af8a502350db3246ecc4becf6b5a34d22f7ed53" dependencies: - accepts "~1.3.4" + accepts "~1.3.5" array-flatten "1.1.1" body-parser "1.18.2" content-disposition "0.5.2" @@ -685,26 +685,26 @@ express@~4.16.0: cookie "0.3.1" cookie-signature "1.0.6" debug "2.6.9" - depd "~1.1.1" - encodeurl "~1.0.1" + depd "~1.1.2" + encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.1.0" + finalhandler "1.1.1" fresh "0.5.2" merge-descriptors "1.0.1" methods "~1.1.2" on-finished "~2.3.0" parseurl "~1.3.2" path-to-regexp "0.1.7" - proxy-addr "~2.0.2" + proxy-addr "~2.0.3" qs "6.5.1" range-parser "~1.2.0" safe-buffer "5.1.1" - send "0.16.1" - serve-static "1.13.1" + send "0.16.2" + serve-static "1.13.2" setprototypeof "1.1.0" - statuses "~1.3.1" - type-is "~1.6.15" + statuses "~1.4.0" + type-is "~1.6.16" utils-merge "1.0.1" vary "~1.1.2" @@ -734,16 +734,16 @@ fastly-purge@~1.0.1: dependencies: request "^2.55.0" -finalhandler@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.0.tgz#ce0b6855b45853e791b2fcc680046d88253dd7f5" +finalhandler@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.1.tgz#eebf4ed840079c83f4249038c9d703008301b105" dependencies: debug "2.6.9" - encodeurl "~1.0.1" + encodeurl "~1.0.2" escape-html "~1.0.3" on-finished "~2.3.0" parseurl "~1.3.2" - statuses "~1.3.1" + statuses "~1.4.0" unpipe "~1.0.0" find-up@^1.0.0: @@ -1084,9 +1084,9 @@ invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" -ipaddr.js@1.5.2: - version "1.5.2" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.5.2.tgz#d4b505bde9946987ccf0fc58d9010ff9607e3fa0" +ipaddr.js@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.6.0.tgz#e3fa357b773da619f26e95f049d055c72796f86b" is-arrayish@^0.2.1: version "0.2.1" @@ -1391,7 +1391,7 @@ mime-db@~1.33.0: version "1.33.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.16, mime-types@~2.1.17, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: @@ -1864,12 +1864,12 @@ protozero@1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.5.1.tgz#5a27df6fb6e1ed743f510812ae76c082f5b16638" -proxy-addr@~2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.2.tgz#6571504f47bb988ec8180253f85dd7e14952bdec" +proxy-addr@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.3.tgz#355f262505a621646b3130a728eb647e22055341" dependencies: forwarded "~0.1.2" - ipaddr.js "1.5.2" + ipaddr.js "1.6.0" punycode@^1.4.1: version "1.4.1" @@ -2176,14 +2176,14 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -send@0.16.1: - version "0.16.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" +send@0.16.2: + version "0.16.2" + resolved "https://registry.yarnpkg.com/send/-/send-0.16.2.tgz#6ecca1e0f8c156d141597559848df64730a6bbc1" dependencies: debug "2.6.9" - depd "~1.1.1" + depd "~1.1.2" destroy "~1.0.4" - encodeurl "~1.0.1" + encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" @@ -2192,16 +2192,16 @@ send@0.16.1: ms "2.0.0" on-finished "~2.3.0" range-parser "~1.2.0" - statuses "~1.3.1" + statuses "~1.4.0" -serve-static@1.13.1: - version "1.13.1" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.1.tgz#4c57d53404a761d8f2e7c1e8a18a47dbf278a719" +serve-static@1.13.2: + version "1.13.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.2.tgz#095e8472fd5b46237db50ce486a43f4b86c6cec1" dependencies: - encodeurl "~1.0.1" + encodeurl "~1.0.2" escape-html "~1.0.3" parseurl "~1.3.2" - send "0.16.1" + send "0.16.2" set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" @@ -2322,7 +2322,7 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -"statuses@>= 1.3.1 < 2", statuses@~1.3.1: +"statuses@>= 1.3.1 < 2": version "1.3.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e" @@ -2330,6 +2330,10 @@ sshpk@^1.7.0: version "1.5.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" +statuses@~1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" + step-profiler@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3" From 11cdcc65ad0e570e57ede263d65d82c0dd062899 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 12:45:16 +0200 Subject: [PATCH 137/185] Add safety limit to sample metadata The sampling probability is now being computed using an estimate of the table row count This could led to too high probabilities (to large samples) if the estimate is not accurate. To avoid potential problems with large samples we've added a LIMIT to the sampling queries. --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 5 +++-- lib/cartodb/utils/query-utils.js | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 689edb9b..0390536b 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -172,13 +172,14 @@ function mergeColumns(results) { } } - function _sample(ctx, numRows) { if (ctx.metaOptions.sample) { const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); + // We'll use a safety limit just in case numRows is a bad estimate + const limit = Math.ceil(ctx.metaOptions.sample * 1.5); return queryPromise( ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb)), + _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)), res => ({ sample: res.rows }) ); } diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index 0bbd38c0..a1276f5e 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -88,11 +88,12 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull `; }; -module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { +module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) { const singleTable = simpleQueryTable(query); if (singleTable) { return getTableSample(singleTable.table, singleTable.columns, sampleProb, randomSeed); } + const limitClause = limit ? `LIMIT ${limit}` : ''; return ` WITH __cdb_rndseed AS ( SELECT setseed(${randomSeed}) @@ -100,14 +101,16 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) { SELECT * FROM (${query}) AS __cdb_query WHERE random() < ${sampleProb} + ${limitClause} `; }; -function getTableSample(table, columns, sampleProb, randomSeed) { +function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) { + const limitClause = limit ? `LIMIT ${limit}` : ''; sampleProb *= 100; randomSeed *= Math.pow(2, 31) -1; return ` - SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) + SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause} `; } From c95b080267e2eed67944557b6f4f44c8f4714ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 12:47:48 +0200 Subject: [PATCH 138/185] Upgrade yargs version to 11.1.0 --- package.json | 2 +- yarn.lock | 208 ++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 183 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 9c28e30e..a22ddbc3 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "turbo-carto": "0.20.2", "underscore": "~1.6.0", "windshaft": "cartodb/windshaft#upgrade-dependencies", - "yargs": "~5.0.0" + "yargs": "11.1.0" }, "devDependencies": { "istanbul": "~0.4.3", diff --git a/yarn.lock b/yarn.lock index be09f92f..c71acc8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -74,6 +74,10 @@ ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" +ansi-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" + ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" @@ -253,6 +257,10 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" +camelcase@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" + camshaft@0.61.9: version "0.61.9" resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.9.tgz#f3d399dfacf51b6a492c579e925c8b1141b22974" @@ -384,6 +392,14 @@ cliui@^3.2.0: strip-ansi "^3.0.1" wrap-ansi "^2.0.0" +cliui@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49" + dependencies: + string-width "^2.1.1" + strip-ansi "^4.0.0" + wrap-ansi "^2.0.0" + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -442,6 +458,14 @@ core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" +cross-spawn@^5.0.1: + version "5.1.0" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-5.1.0.tgz#e8bd0efee58fcff6f8f94510a0a554bbfa235449" + dependencies: + lru-cache "^4.0.1" + shebang-command "^1.2.0" + which "^1.2.9" + cryptiles@2.x.x: version "2.0.5" resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8" @@ -669,6 +693,18 @@ etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" +execa@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777" + dependencies: + cross-spawn "^5.0.1" + get-stream "^3.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + exit@0.1.2, exit@0.1.x: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -753,6 +789,12 @@ find-up@^1.0.0: path-exists "^2.0.0" pinkie-promise "^2.0.0" +find-up@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + dependencies: + locate-path "^2.0.0" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -844,6 +886,10 @@ get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" +get-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -1108,6 +1154,14 @@ is-fullwidth-code-point@^1.0.0: dependencies: number-is-nan "^1.0.0" +is-fullwidth-code-point@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" + +is-stream@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" + is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" @@ -1251,6 +1305,13 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + lodash._baseassign@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz#8c38a099500f215ad09e59f1722fd0c52bfe0a4e" @@ -1274,7 +1335,7 @@ lodash._isiterateecall@^3.0.0: version "3.0.9" resolved "https://registry.yarnpkg.com/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz#5203ad7ba425fae842460e696db9cf3e6aac057c" -lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.1.0, lodash.assign@^4.2.0: +lodash.assign@^4.0.3, lodash.assign@^4.0.6: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" @@ -1331,6 +1392,13 @@ lru-cache@2.6.5: version "2.6.5" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" +lru-cache@^4.0.1: + version "4.1.3" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c" + dependencies: + pseudomap "^1.0.2" + yallist "^2.1.2" + lzma@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/lzma/-/lzma-2.3.2.tgz#3783b24858b9c0e747a0df3cbf1fb5fcaa92c441" @@ -1360,6 +1428,12 @@ media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" +mem@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/mem/-/mem-1.1.0.tgz#5edd52b485ca1d900fe64895505399a0dfa45f76" + dependencies: + mimic-fn "^1.0.0" + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -1415,6 +1489,10 @@ mime@~1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" +mimic-fn@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" + "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -1607,6 +1685,12 @@ npm-packlist@^1.1.6: ignore-walk "^3.0.1" npm-bundled "^1.0.1" +npm-run-path@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" + dependencies: + path-key "^2.0.0" + npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" @@ -1676,6 +1760,14 @@ os-locale@^1.4.0: dependencies: lcid "^1.0.0" +os-locale@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-2.1.0.tgz#42bc2900a6b5b8bd17376c8e882b65afccf24bf2" + dependencies: + execa "^0.7.0" + lcid "^1.0.0" + mem "^1.1.0" + os-tmpdir@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" @@ -1687,6 +1779,26 @@ osenv@^0.1.4: os-homedir "^1.0.0" os-tmpdir "^1.0.0" +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + +p-limit@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.2.0.tgz#0e92b6bedcb59f022c13d0f1949dc82d15909f1c" + dependencies: + p-try "^1.0.0" + +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + dependencies: + p-limit "^1.1.0" + +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + packet-reader@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.2.0.tgz#819df4d010b82d5ea5671f8a1a3acf039bcd7700" @@ -1707,10 +1819,18 @@ path-exists@^2.0.0: dependencies: pinkie-promise "^2.0.0" +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" +path-key@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -1871,6 +1991,10 @@ proxy-addr@~2.0.3: forwarded "~0.1.2" ipaddr.js "1.6.0" +pseudomap@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" + punycode@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" @@ -2215,6 +2339,16 @@ setprototypeof@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" +shebang-command@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" + dependencies: + shebang-regex "^1.0.0" + +shebang-regex@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" + shelljs@0.3.x: version "0.3.0" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.3.0.tgz#3596e6307a781544f591f37da618360f31db57b1" @@ -2360,6 +2494,13 @@ string-width@^1.0.1, string-width@^1.0.2: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +string-width@^2.0.0, string-width@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" + dependencies: + is-fullwidth-code-point "^2.0.0" + strip-ansi "^4.0.0" + string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" @@ -2380,12 +2521,22 @@ strip-ansi@^3.0.0, strip-ansi@^3.0.1: dependencies: ansi-regex "^2.0.0" +strip-ansi@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" + dependencies: + ansi-regex "^3.0.0" + strip-bom@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" dependencies: is-utf8 "^0.2.0" +strip-eof@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" + strip-json-comments@1.0.x: version "1.0.4" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91" @@ -2622,7 +2773,11 @@ which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" -which@^1.1.1: +which-module@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" + +which@^1.1.1, which@^1.2.9: version "1.3.0" resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" dependencies: @@ -2703,6 +2858,10 @@ y18n@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" +yallist@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" + yallist@^3.0.0, yallist@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" @@ -2714,12 +2873,28 @@ yargs-parser@^2.4.1: camelcase "^3.0.0" lodash.assign "^4.0.6" -yargs-parser@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-3.2.0.tgz#5081355d19d9d0c8c5d81ada908cb4e6d186664f" +yargs-parser@^9.0.2: + version "9.0.2" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-9.0.2.tgz#9ccf6a43460fe4ed40a9bb68f48d43b8a68cc077" dependencies: - camelcase "^3.0.0" - lodash.assign "^4.1.0" + camelcase "^4.1.0" + +yargs@11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-11.1.0.tgz#90b869934ed6e871115ea2ff58b03f4724ed2d77" + dependencies: + cliui "^4.0.0" + decamelize "^1.1.1" + find-up "^2.1.0" + get-caller-file "^1.0.1" + os-locale "^2.0.0" + require-directory "^2.1.1" + require-main-filename "^1.0.1" + set-blocking "^2.0.0" + string-width "^2.0.0" + which-module "^2.0.0" + y18n "^3.2.1" + yargs-parser "^9.0.2" yargs@^4.2.0: version "4.8.1" @@ -2749,25 +2924,6 @@ yargs@~3.10.0: decamelize "^1.0.0" window-size "0.1.0" -yargs@~5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-5.0.0.tgz#3355144977d05757dbb86d6e38ec056123b3a66e" - dependencies: - cliui "^3.2.0" - decamelize "^1.1.1" - get-caller-file "^1.0.1" - lodash.assign "^4.2.0" - os-locale "^1.4.0" - read-pkg-up "^1.0.1" - require-directory "^2.1.1" - require-main-filename "^1.0.1" - set-blocking "^2.0.0" - string-width "^1.0.2" - which-module "^1.0.0" - window-size "^0.2.0" - y18n "^3.2.1" - yargs-parser "^3.2.0" - zipfile@~0.5.5: version "0.5.11" resolved "https://registry.yarnpkg.com/zipfile/-/zipfile-0.5.11.tgz#a46c1b76706138fb9bcf83890193caec679f685e" From bdc52204e4e28f66c8e965d55220cb5afb40ebf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 13:01:01 +0200 Subject: [PATCH 139/185] Upgrade nock version to 9.2.6 --- package.json | 2 +- yarn.lock | 86 ++++++++++++++++++++++++++++------------------------ 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/package.json b/package.json index a22ddbc3..12202477 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "jshint": "~2.9.4", "mocha": "~3.4.1", "moment": "2.22.1", - "nock": "~2.11.0", + "nock": "9.2.6", "redis": "2.8.0", "strftime": "0.10.0" }, diff --git a/yarn.lock b/yarn.lock index c71acc8a..35bd4f74 100644 --- a/yarn.lock +++ b/yarn.lock @@ -343,13 +343,16 @@ center-align@^0.1.1: align-text "^0.1.3" lazy-cache "^1.0.3" -"chai@>=1.9.2 <4.0.0": - version "3.5.0" - resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247" +chai@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chai/-/chai-4.1.2.tgz#0f64584ba642f0f2ace2806279f4f06ca23ad73c" dependencies: assertion-error "^1.0.1" - deep-eql "^0.1.3" - type-detect "^1.0.0" + check-error "^1.0.1" + deep-eql "^3.0.0" + get-func-name "^2.0.0" + pathval "^1.0.0" + type-detect "^4.0.0" chalk@^1.1.3: version "1.1.3" @@ -361,6 +364,10 @@ chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +check-error@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" + chownr@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" @@ -520,21 +527,15 @@ debug@3.1.0, debug@^3.1.0, debug@~3.1.0: dependencies: ms "2.0.0" -debug@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-1.0.5.tgz#f7241217430f99dec4c2b473eab92228e874c2ac" - dependencies: - ms "2.0.0" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" -deep-eql@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" +deep-eql@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df" dependencies: - type-detect "0.1.1" + type-detect "^4.0.0" deep-equal@^1.0.0: version "1.0.1" @@ -886,6 +887,10 @@ get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" +get-func-name@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41" + get-stream@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" @@ -1251,7 +1256,7 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" -json-stringify-safe@~5.0.1: +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -1363,14 +1368,14 @@ lodash.keys@^3.0.0: lodash.isarguments "^3.0.0" lodash.isarray "^3.0.0" -lodash@2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.1.tgz#5b7723034dda4d262e5a46fb2c58d7cc22f71420" - lodash@3.7.x: version "3.7.0" resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.7.0.tgz#3678bd8ab995057c07ade836ed2ef087da811d45" +lodash@^4.17.5: + version "4.17.10" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7" + lodash@^4.5.1: version "4.17.5" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511" @@ -1606,16 +1611,19 @@ negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" -nock@~2.11.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/nock/-/nock-2.11.0.tgz#2554cc09f0c3077100aae185b88d030bad950fcd" +nock@9.2.6: + version "9.2.6" + resolved "https://registry.yarnpkg.com/nock/-/nock-9.2.6.tgz#496ddb2c32e6d0848cda1b3ea51fbe054d4454d3" dependencies: - chai ">=1.9.2 <4.0.0" - debug "^1.0.4" + chai "^4.1.2" + debug "^3.1.0" deep-equal "^1.0.0" - lodash "2.4.1" + json-stringify-safe "^5.0.1" + lodash "^4.17.5" mkdirp "^0.5.0" - propagate "0.3.x" + propagate "^1.0.0" + qs "^6.5.1" + semver "^5.5.0" node-pre-gyp@0.10.0: version "0.10.0" @@ -1843,6 +1851,10 @@ path-type@^1.0.0: pify "^2.0.0" pinkie-promise "^2.0.0" +pathval@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.0.tgz#b942e6d4bde653005ef6b71361def8727d0645e0" + performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" @@ -1976,9 +1988,9 @@ progress-stream@~0.5.x: speedometer "~0.1.2" through2 "~0.2.3" -propagate@0.3.x: - version "0.3.1" - resolved "https://registry.yarnpkg.com/propagate/-/propagate-0.3.1.tgz#e3a84404a7ece820dd6bbea9f6d924e3135ae09c" +propagate@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-1.0.0.tgz#00c2daeedda20e87e3782b344adba1cddd6ad709" protozero@1.5.1: version "1.5.1" @@ -2003,7 +2015,7 @@ qs@6.5.1, qs@~6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" -qs@6.5.2: +qs@6.5.2, qs@^6.5.1: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -2288,7 +2300,7 @@ semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" -semver@5.5.0: +semver@5.5.0, semver@^5.5.0: version "5.5.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" @@ -2683,13 +2695,9 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -type-detect@0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-0.1.1.tgz#0ba5ec2a885640e470ea4e8505971900dac58822" - -type-detect@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" +type-detect@^4.0.0: + version "4.0.8" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" type-is@~1.6.15: version "1.6.15" From 7f841d49b245bae6a06ff45859398d758e5d65bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 13:18:44 +0200 Subject: [PATCH 140/185] Upgrade mocha version to 3.5.3 --- package.json | 2 +- yarn.lock | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 12202477..8ccabe2e 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "devDependencies": { "istanbul": "~0.4.3", "jshint": "~2.9.4", - "mocha": "~3.4.1", + "mocha": "3.5.3", "moment": "2.22.1", "nock": "9.2.6", "redis": "2.8.0", diff --git a/yarn.lock b/yarn.lock index 35bd4f74..b1e64aa8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -509,11 +509,11 @@ debug@2.2.0: dependencies: ms "0.7.1" -debug@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b" +debug@2.6.8: + version "2.6.8" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" dependencies: - ms "0.7.2" + ms "2.0.0" debug@2.6.9, debug@^2.1.2, debug@^2.2.0: version "2.6.9" @@ -1033,6 +1033,10 @@ hawk@~6.0.2: hoek "4.x.x" sntp "2.x.x" +he@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -1535,17 +1539,18 @@ mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdi dependencies: minimist "0.0.8" -mocha@~3.4.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.4.2.tgz#d0ef4d332126dbf18d0d640c9b382dd48be97594" +mocha@3.5.3: + version "3.5.3" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-3.5.3.tgz#1e0480fe36d2da5858d1eb6acc38418b26eaa20d" dependencies: browser-stdout "1.3.0" commander "2.9.0" - debug "2.6.0" + debug "2.6.8" diff "3.2.0" escape-string-regexp "1.0.5" glob "7.1.1" growl "1.9.2" + he "1.1.1" json3 "3.3.2" lodash.create "3.1.1" mkdirp "0.5.1" @@ -1559,10 +1564,6 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" -ms@0.7.2: - version "0.7.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" - ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" From 83e944c985eaff430b279fc42ffb2f63bf247474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 13:29:07 +0200 Subject: [PATCH 141/185] Upgrade lru-cache version to 4.1.3 --- package.json | 2 +- yarn.lock | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 8ccabe2e..31636992 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "fastly-purge": "~1.0.1", "glob": "^7.1.2", "log4js": "cartodb/log4js-node#cdb", - "lru-cache": "2.6.5", + "lru-cache": "4.1.3", "lzma": "~2.3.2", "node-statsd": "0.1.1", "on-headers": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index b1e64aa8..cb6d5f2c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1397,11 +1397,7 @@ longest@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097" -lru-cache@2.6.5: - version "2.6.5" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.6.5.tgz#e56d6354148ede8d7707b58d143220fd08df0fd5" - -lru-cache@^4.0.1: +lru-cache@4.1.3, lru-cache@^4.0.1: version "4.1.3" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.3.tgz#a1175cf3496dfc8436c156c334b4955992bce69c" dependencies: From a8da7e60c353a1734c450a953a1acd3a227f0726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 13:58:17 +0200 Subject: [PATCH 142/185] Do not define dependencies in package.json with version ranges --- package.json | 20 +++++++------- yarn.lock | 74 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 57 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index 31636992..d8d76146 100644 --- a/package.json +++ b/package.json @@ -24,36 +24,36 @@ "Simon Martin " ], "dependencies": { - "basic-auth": "^2.0.0", + "basic-auth": "2.0.0", "body-parser": "1.18.3", "camshaft": "0.61.9", "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", "cartodb-redis": "1.0.0", - "debug": "^3.1.0", + "debug": "3.1.0", "dot": "1.1.2", "express": "4.16.3", - "fastly-purge": "~1.0.1", - "glob": "^7.1.2", + "fastly-purge": "1.0.1", + "glob": "7.1.2", "log4js": "cartodb/log4js-node#cdb", "lru-cache": "4.1.3", - "lzma": "~2.3.2", + "lzma": "2.3.2", "node-statsd": "0.1.1", - "on-headers": "^1.0.1", + "on-headers": "1.0.1", "queue-async": "1.1.0", "redis-mpool": "0.5.0", "request": "2.87.0", "semver": "5.5.0", "step": "1.0.0", - "step-profiler": "~0.3.0", + "step-profiler": "0.3.0", "turbo-carto": "0.20.2", - "underscore": "~1.6.0", + "underscore": "1.6.0", "windshaft": "cartodb/windshaft#upgrade-dependencies", "yargs": "11.1.0" }, "devDependencies": { - "istanbul": "~0.4.3", - "jshint": "~2.9.4", + "istanbul": "0.4.3", + "jshint": "2.9.4", "mocha": "3.5.3", "moment": "2.22.1", "nock": "9.2.6", diff --git a/yarn.lock b/yarn.lock index cb6d5f2c..a9f15e5d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -147,7 +147,7 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" -basic-auth@^2.0.0: +basic-auth@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.0.tgz#015db3f353e02e56377755f962742e8981e7bbba" dependencies: @@ -217,6 +217,13 @@ boom@5.x.x: dependencies: hoek "4.x.x" +brace-expansion@^1.0.0: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -765,12 +772,19 @@ fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" -fastly-purge@~1.0.1: +fastly-purge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/fastly-purge/-/fastly-purge-1.0.1.tgz#3bdfe9ea1d0fbf2a65712f2f5fe2eca63fcb5960" dependencies: request "^2.55.0" +fileset@0.2.x: + version "0.2.1" + resolved "https://registry.yarnpkg.com/fileset/-/fileset-0.2.1.tgz#588ef8973c6623b2a76df465105696b96aac8067" + dependencies: + glob "5.x" + minimatch "2.x" + finalhandler@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.1.tgz#eebf4ed840079c83f4249038c9d703008301b105" @@ -901,6 +915,16 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" +glob@5.x: + version "5.0.15" + resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" + dependencies: + inflight "^1.0.4" + inherits "2" + minimatch "2 || 3" + once "^1.3.0" + path-is-absolute "^1.0.0" + glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" @@ -912,13 +936,14 @@ glob@7.1.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^5.0.15: - version "5.0.15" - resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" +glob@7.1.2, glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: + fs.realpath "^1.0.0" inflight "^1.0.4" inherits "2" - minimatch "2 || 3" + minimatch "^3.0.4" once "^1.3.0" path-is-absolute "^1.0.0" @@ -932,17 +957,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1195,15 +1209,15 @@ isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" -istanbul@~0.4.3: - version "0.4.5" - resolved "https://registry.yarnpkg.com/istanbul/-/istanbul-0.4.5.tgz#65c7d73d4c4da84d4f3ac310b918fb0b8033733b" +istanbul@0.4.3: + version "0.4.3" + resolved "https://registry.yarnpkg.com/istanbul/-/istanbul-0.4.3.tgz#5b714ee0ae493ac5ef204b99f3872bceef73d53a" dependencies: abbrev "1.0.x" async "1.x" escodegen "1.8.x" esprima "2.7.x" - glob "^5.0.15" + fileset "0.2.x" handlebars "^4.0.1" js-yaml "3.x" mkdirp "0.5.x" @@ -1233,9 +1247,9 @@ jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" -jshint@~2.9.4: - version "2.9.5" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" +jshint@2.9.4: + version "2.9.4" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.4.tgz#5e3ba97848d5290273db514aee47fe24cf592934" dependencies: cli "~1.0.0" console-browserify "1.1.x" @@ -1404,7 +1418,7 @@ lru-cache@4.1.3, lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" -lzma@~2.3.2: +lzma@2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/lzma/-/lzma-2.3.2.tgz#3783b24858b9c0e747a0df3cbf1fb5fcaa92c441" @@ -1504,6 +1518,12 @@ mimic-fn@^1.0.0: dependencies: brace-expansion "^1.1.7" +minimatch@2.x: + version "2.0.10" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-2.0.10.tgz#8d087c39c6b38c001b97fca7ce6d0e1e80afbac7" + dependencies: + brace-expansion "^1.0.0" + minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1727,7 +1747,7 @@ on-finished@~2.3.0: dependencies: ee-first "1.1.1" -on-headers@^1.0.1: +on-headers@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.1.tgz#928f5d0f470d49342651ea6794b0857c100693f7" @@ -2477,7 +2497,7 @@ statuses@~1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" -step-profiler@~0.3.0: +step-profiler@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/step-profiler/-/step-profiler-0.3.0.tgz#841368ce44f2330c862edd5ec60e7fbec148f0b3" dependencies: From b7b52eee808c5b03c6cd37b8c4835d0d8c2ef69c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 14:04:04 +0200 Subject: [PATCH 143/185] Upgrade istanbul and jshint version to 0.4.5 and 2.9.5 respectively --- package.json | 4 ++-- yarn.lock | 54 +++++++++++++++++----------------------------------- 2 files changed, 19 insertions(+), 39 deletions(-) diff --git a/package.json b/package.json index d8d76146..0e2e807c 100644 --- a/package.json +++ b/package.json @@ -52,8 +52,8 @@ "yargs": "11.1.0" }, "devDependencies": { - "istanbul": "0.4.3", - "jshint": "2.9.4", + "istanbul": "0.4.5", + "jshint": "2.9.5", "mocha": "3.5.3", "moment": "2.22.1", "nock": "9.2.6", diff --git a/yarn.lock b/yarn.lock index a9f15e5d..e3872fe7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -217,13 +217,6 @@ boom@5.x.x: dependencies: hoek "4.x.x" -brace-expansion@^1.0.0: - version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" - dependencies: - balanced-match "^1.0.0" - concat-map "0.0.1" - brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -778,13 +771,6 @@ fastly-purge@1.0.1: dependencies: request "^2.55.0" -fileset@0.2.x: - version "0.2.1" - resolved "https://registry.yarnpkg.com/fileset/-/fileset-0.2.1.tgz#588ef8973c6623b2a76df465105696b96aac8067" - dependencies: - glob "5.x" - minimatch "2.x" - finalhandler@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.1.tgz#eebf4ed840079c83f4249038c9d703008301b105" @@ -915,16 +901,6 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@5.x: - version "5.0.15" - resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" - dependencies: - inflight "^1.0.4" - inherits "2" - minimatch "2 || 3" - once "^1.3.0" - path-is-absolute "^1.0.0" - glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" @@ -947,6 +923,16 @@ glob@7.1.2, glob@^7.0.5, glob@^7.1.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^5.0.15: + version "5.0.15" + resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" + dependencies: + inflight "^1.0.4" + inherits "2" + minimatch "2 || 3" + once "^1.3.0" + path-is-absolute "^1.0.0" + glob@^6.0.1: version "6.0.4" resolved "https://registry.yarnpkg.com/glob/-/glob-6.0.4.tgz#0f08860f6a155127b2fadd4f9ce24b1aab6e4d22" @@ -1209,15 +1195,15 @@ isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" -istanbul@0.4.3: - version "0.4.3" - resolved "https://registry.yarnpkg.com/istanbul/-/istanbul-0.4.3.tgz#5b714ee0ae493ac5ef204b99f3872bceef73d53a" +istanbul@0.4.5: + version "0.4.5" + resolved "https://registry.yarnpkg.com/istanbul/-/istanbul-0.4.5.tgz#65c7d73d4c4da84d4f3ac310b918fb0b8033733b" dependencies: abbrev "1.0.x" async "1.x" escodegen "1.8.x" esprima "2.7.x" - fileset "0.2.x" + glob "^5.0.15" handlebars "^4.0.1" js-yaml "3.x" mkdirp "0.5.x" @@ -1247,9 +1233,9 @@ jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" -jshint@2.9.4: - version "2.9.4" - resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.4.tgz#5e3ba97848d5290273db514aee47fe24cf592934" +jshint@2.9.5: + version "2.9.5" + resolved "https://registry.yarnpkg.com/jshint/-/jshint-2.9.5.tgz#1e7252915ce681b40827ee14248c46d34e9aa62c" dependencies: cli "~1.0.0" console-browserify "1.1.x" @@ -1518,12 +1504,6 @@ mimic-fn@^1.0.0: dependencies: brace-expansion "^1.1.7" -minimatch@2.x: - version "2.0.10" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-2.0.10.tgz#8d087c39c6b38c001b97fca7ce6d0e1e80afbac7" - dependencies: - brace-expansion "^1.0.0" - minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" From 32092d212ebe4777683066ae3db0fce09b9de1ea Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 14:05:27 +0200 Subject: [PATCH 144/185] Fix bug --- lib/cartodb/utils/query-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index a1276f5e..22d99eef 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -91,7 +91,7 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) { const singleTable = simpleQueryTable(query); if (singleTable) { - return getTableSample(singleTable.table, singleTable.columns, sampleProb, randomSeed); + return getTableSample(singleTable.table, singleTable.columns, sampleProb, limit, randomSeed); } const limitClause = limit ? `LIMIT ${limit}` : ''; return ` From b233f18a0ff36810a8b0707011f108575d4c8541 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 15:43:23 +0200 Subject: [PATCH 145/185] Modernize code copied from SQL API --- .../backends/layer-stats/mapnik-layer-stats.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 0390536b..ad415c55 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -260,19 +260,18 @@ function fieldType(cname) { // columns are returned as an object { columnName1: { type1: ...}, ..} // for consistency with SQL API -function formatResultFields(dbConnection, flds) { - flds = flds || []; - var nfields = {}; - for (var i=0; i Date: Mon, 21 May 2018 15:43:32 +0200 Subject: [PATCH 146/185] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 141e082e..9df94a67 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,11 +12,13 @@ New features: - Fix a bug with zero length lines not being rendered when using the marker symbolizer. - Upgrades Camshaft to [0.61.9](https://github.com/CartoDB/camshaft/releases/tag/0.61.9): - Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses. +- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952) Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. - Static maps fails for unsupported formats - Handling errors extracting the column type on dataviews +- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens ## 6.1.0 Released 2018-04-16 From d7a90e6be42b7e2a1001c92204a61ff92c6b5855 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 15:54:52 +0200 Subject: [PATCH 147/185] Remove debugging comment --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index ad415c55..40590156 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -264,7 +264,6 @@ function formatResultFields(dbConnection, fields = []) { let nfields = {}; for (let field of fields) { const cname = dbConnection.typeName(field.dataTypeID); - console.log("FIELD",field); let tname; if ( ! cname ) { tname = 'unknown(' + field.dataTypeID + ')'; From d828a92ea33b74598db8a1a0f1c7abc71efd11d7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 16:59:36 +0200 Subject: [PATCH 148/185] Use ifError to check for errors --- test/acceptance/stats/mapnik_stats_layergroup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index f74272d0..1ec7371f 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -346,7 +346,7 @@ describe('Create mapnik layergroup', function() { } testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); const expectedColumns = { From befedfd80a62b63020b7d927ffe51cfeded3c74b Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 17:03:16 +0200 Subject: [PATCH 149/185] Use ifError to check for errors --- .../stats/mapnik_stats_layergroup.js | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 1ec7371f..f6cad84e 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -90,7 +90,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); @@ -107,7 +107,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -127,7 +127,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -147,7 +147,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); @@ -164,7 +164,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -183,7 +183,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); @@ -204,7 +204,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); @@ -224,7 +224,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function (err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); @@ -245,7 +245,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); @@ -265,7 +265,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); @@ -295,7 +295,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); const expectedColumns = { @@ -404,7 +404,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); @@ -423,7 +423,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); @@ -442,7 +442,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); @@ -464,7 +464,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); From 6384f5538cbd500188fa8013bc15e1426371ee78 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 17:06:53 +0200 Subject: [PATCH 150/185] Rename variable for clarity --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 40590156..d1e292de 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -296,7 +296,7 @@ function (layer, dbConnection, callback) { Promise.all([ _estimatedFeatureCount(ctx).then( ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount) - .then(s => mergeResults([s, { estimatedFeatureCount }])) + .then(sampleResults => mergeResults([sampleResults, { estimatedFeatureCount }])) ), _featureCount(ctx), _aggrFeatureCount(ctx), From ebc086106f651f545f8d19fddda4452519b0b424 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 21 May 2018 17:33:03 +0200 Subject: [PATCH 151/185] Refactor metadata queryPromise helper This function to adapt query execution as a Promise was unnecessary complex. --- .../layer-stats/mapnik-layer-stats.js | 89 ++++++------------- 1 file changed, 29 insertions(+), 60 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index d1e292de..69336147 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -34,24 +34,11 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; -function queryPromise(dbConnection, query, adaptResults, errorHandler) { - return new Promise(function(resolve, reject) { - dbConnection.query(query, function (err, res) { - if (err) { - if (errorHandler) { - resolve(errorHandler(err)); - } - else { - reject(err); - } - } - else { - resolve(adaptResults(res)); - } - }); - +function queryPromise(dbConnection, query) { + return new Promise((resolve, reject) => { + dbConnection.query(query, (err, res) => err ? reject(err) : resolve(res)); }); -} + } function columnAggregations(field) { if (field.type === 'number') { @@ -76,22 +63,16 @@ function _getSQL(ctx, query, type='pre', zoom=0) { } function _estimatedFeatureCount(ctx) { - return queryPromise( - ctx.dbConnection, - _getSQL(ctx, queryUtils.getQueryRowEstimation), - res => ({ estimatedFeatureCount: res.rows[0].rows }), - () => ({ estimatedFeatureCount: -1 }) - ); + return queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryRowEstimation)) + .then(res => ({ estimatedFeatureCount: res.rows[0].rows })) + .catch(() => ({ estimatedFeatureCount: -1 })); } function _featureCount(ctx) { if (ctx.metaOptions.featureCount) { // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query - return queryPromise( - ctx.dbConnection, - _getSQL(ctx, queryUtils.getQueryActualRowCount), - res => ({ featureCount: res.rows[0].rows }) - ); + return queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryActualRowCount)) + .then(res => ({ featureCount: res.rows[0].rows })); } return Promise.resolve(); } @@ -103,9 +84,8 @@ function _aggrFeatureCount(ctx) { // return metadata for multiple levels. return queryPromise( ctx.dbConnection, - _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount), - res => ({ aggrFeatureCount: res.rows[0].rows }) - ); + _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount) + ).then(res => ({ aggrFeatureCount: res.rows[0].rows })); } return Promise.resolve(); } @@ -113,11 +93,8 @@ function _aggrFeatureCount(ctx) { function _geometryType(ctx) { if (ctx.metaOptions.geometryType) { const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); - return queryPromise( - ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)), - res => ({ geometryType: res.rows[0].geom_type }) - ); + return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn))) + .then(res => ({ geometryType: res.rows[0].geom_type })); } return Promise.resolve(); } @@ -125,11 +102,8 @@ function _geometryType(ctx) { function _columns(ctx) { if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { // note: post-aggregation columns are in layer.options.columns when aggregation is present - return queryPromise( - ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0)), - res => formatResultFields(ctx.dbConnection, res.fields) - ); + return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0))) + .then(res => formatResultFields(ctx.dbConnection, res.fields)); } return Promise.resolve(); } @@ -177,11 +151,8 @@ function _sample(ctx, numRows) { const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); // We'll use a safety limit just in case numRows is a bad estimate const limit = Math.ceil(ctx.metaOptions.sample * 1.5); - return queryPromise( - ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)), - res => ({ sample: res.rows }) - ); + return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit))) + .then(res => ({ sample: res.rows })); } return Promise.resolve(); } @@ -206,27 +177,25 @@ function _columnStats(ctx, columns) { queries.push( queryPromise( ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN)), - res => ({ [name]: { categories: res.rows } }) - ) + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN)) + ).then(res => ({ [name]: { categories: res.rows } })) ); } }); queries.push( queryPromise( ctx.dbConnection, - _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`), - res => { - let stats = {}; - Object.keys(columns).forEach(name => { - stats[name] = {}; - columnAggregations(columns[name]).forEach(fn => { - stats[name][fn] = res.rows[0][`${name}_${fn}`]; - }); + _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`) + ).then(res => { + let stats = {}; + Object.keys(columns).forEach(name => { + stats[name] = {}; + columnAggregations(columns[name]).forEach(fn => { + stats[name][fn] = res.rows[0][`${name}_${fn}`]; }); - return stats; - } - ) + }); + return stats; + }) ); return Promise.all(queries).then(results => ({ columns: mergeColumns(results) })); } From 094a0ea76d9a5a3dfa1eae3d2d9f9a02f8ffe976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 17:41:19 +0200 Subject: [PATCH 152/185] Upgrade windshaft version to 4.8.0 --- package.json | 2 +- yarn.lock | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 0e2e807c..40d75a38 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "step-profiler": "0.3.0", "turbo-carto": "0.20.2", "underscore": "1.6.0", - "windshaft": "cartodb/windshaft#upgrade-dependencies", + "windshaft": "4.8.0", "yargs": "11.1.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index e3872fe7..deb13479 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11,7 +11,7 @@ node-pre-gyp "0.10.0" protozero "1.5.1" -"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb9": +"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb9": version "2.5.1-cdb9" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/5129e43223cb55daed31373c7a36c98eb6178fc1" dependencies: @@ -23,7 +23,7 @@ version "1.0.5" resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2" -abaculus@cartodb/abaculus#2.0.3-cdb10: +"abaculus@github:cartodb/abaculus#2.0.3-cdb10": version "2.0.3-cdb10" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/90d537028bb8af8a35e7a40c46493066dd8a76b3" dependencies: @@ -272,7 +272,7 @@ camshaft@0.61.9: dot "^1.0.3" request "2.85.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -290,17 +290,17 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@cartodb/carto#0.15.1-cdb3: - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" +carto@cartodb/carto#master: + version "0.15.1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/31abb8bee02df605521247b0223d508320f7d4c8" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "1.8.3" -carto@cartodb/carto#master: - version "0.15.1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/31abb8bee02df605521247b0223d508320f7d4c8" +"carto@github:cartodb/carto#0.15.1-cdb3": + version "0.15.1-cdb3" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" @@ -2614,7 +2614,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb14: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb14": version "0.6.18-cdb14" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/6d06f728833d3e34d1adcd05567b3f4379f547bb" dependencies: @@ -2802,9 +2802,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#upgrade-dependencies: - version "4.7.4" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/cdb544257b17afc8f6c2c8a3438a6294c52385ba" +windshaft@4.8.0: + version "4.8.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.8.0.tgz#79ea03ee78fa68288e07c7aa01a9fdabe3cc5af0" dependencies: "@carto/mapnik" "3.6.2-carto.10" "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb9 From 87c5844704acd13315e820a0cb3631cd31abff57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 21 May 2018 17:47:51 +0200 Subject: [PATCH 153/185] Update NEWS --- NEWS.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 141e082e..44723922 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,17 +1,36 @@ # Changelog -## 6.1.1 +## 6.2.0 Released 2018-mm-dd New features: - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 -- Upgrades Windshaft to [4.7.3](https://github.com/CartoDB/Windshaft/blob/4.7.3/NEWS.md#version-473) which includes: +- Upgrades Windshaft to [4.8.0](https://github.com/CartoDB/Windshaft/blob/4.8.0/NEWS.md#version-480) which includes: + - Update internal deps. - A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type. - Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible). - MVT: Disable simplify_distance to avoid multiple simplifications. - Fix a bug with zero length lines not being rendered when using the marker symbolizer. - Upgrades Camshaft to [0.61.9](https://github.com/CartoDB/camshaft/releases/tag/0.61.9): - Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses. +- Update other deps: + - body-parser: 1.18.3 + - dot: 1.1.2 + - express: 4.16.3 + - lru-cache: 4.1.3 + - node-statsd: 0.1.1, + - queue-async: 1.1.0 + - request: 2.87.0 + - semver: 5.5.0 + - step: 1.0.0 + - yargs: 11.1.0 +- Update devel deps: + - istanbul: 0.4.5 + - jshint: 2.9.5 + - mocha: 3.5.3 + - moment: 2.22.1 + - nock: 9.2.6 + - strftime: 0.10.0 Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. From 97b4e197776237e1f849c71f9de3e9eaa8e90f9f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 22 May 2018 13:00:18 +0200 Subject: [PATCH 154/185] Add metadata parameter for NULL categories --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index d1e292de..a04d68bc 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -201,12 +201,17 @@ function _columnStats(ctx, columns) { ); if (columns[name].type === 'string') { const topN = ctx.metaOptions.columnStats.topCategories || 1024; + let includeNulls = true; + if (ctx.metaOptions.columnStats.hasOwnProperty('includeNulls')) { + includeNulls = ctx.metaOptions.columnStats.includeNulls; + } + // TODO: ctx.metaOptions.columnStats.maxCategories // => use PG stats to dismiss columns with more distinct values queries.push( queryPromise( ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN)), + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls)), res => ({ [name]: { categories: res.rows } }) ) ); From 7041039572e47f7267cca7c2a0fab0af9729ca9e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 22 May 2018 13:42:29 +0200 Subject: [PATCH 155/185] Add tests for category metadata parameters --- .../stats/mapnik_stats_layergroup.js | 129 ++++++++++++++---- 1 file changed, 103 insertions(+), 26 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index f6cad84e..46c4368a 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -74,6 +74,26 @@ describe('Create mapnik layergroup', function() { } }; + var mapnikLayerNullCats = { + type: 'mapnik', + options: { + sql: ` + WITH geom AS ( + SELECT + 'SRID=4326;POINT(0 0)'::geometry AS the_geom, + 'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator + ) + SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom + UNION + SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom + UNION + SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom + `, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + function mapnikBasicLayerId(index) { return 'layer' + index; } @@ -309,6 +329,32 @@ describe('Create mapnik layergroup', function() { }); }); + // metadata categories are ordered only partially by descending frequency; + // this orders them completely to avoid ambiguities when comparing + function withSortedCategories(columns) { + function catOrder(a, b) { + if (a.frequency !== b.frequency) { + return b.frequency - a.frequency; + } + if (a.category < b.category) { + return -1; + } + if (a.category > b.category) { + return +1; + } + return 0; + } + let sorted = {}; + Object.keys(columns).forEach(name => { + let data = columns[name]; + if (data.hasOwnProperty('categories')) { + data = Object.assign(data, { categories: data.categories.sort(catOrder)}); + } + sorted[name] = data; + }); + return sorted; + } + it('should provide column stats as optional metadata', function(done) { var testClient = new TestClient({ version: '1.4.0', @@ -319,32 +365,6 @@ describe('Create mapnik layergroup', function() { ] }); - // metadata categories are ordered only partially by descending frequency; - // this orders them completely to avoid ambiguities when comparing - function withSortedCategories(columns) { - function catOrder(a, b) { - if (a.frequency !== b.frequency) { - return b.frequency - a.frequency; - } - if (a.category < b.category) { - return -1; - } - if (a.category > b.category) { - return +1; - } - return 0; - } - let sorted = {}; - Object.keys(columns).forEach(name => { - let data = columns[name]; - if (data.hasOwnProperty('categories')) { - data = Object.assign(data, { categories: data.categories.sort(catOrder)}); - } - sorted[name] = data; - }); - return sorted; - } - testClient.getLayergroup(function(err, layergroup) { assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); @@ -393,6 +413,63 @@ describe('Create mapnik layergroup', function() { }); }); + it('should limit the number of categories as requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: { topCategories: 2 } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.address.categories.length, 2); + testClient.drain(done); + }); + }); + + it('should include null categories if requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayerNullCats, { + columnStats: { includeNulls: true } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.cat.categories.length, 3); + testClient.drain(done); + }); + }); + + it('should not include null categories if not requested', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayerNullCats, { + columnStats: { includeNulls: false } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns; + assert.equal(columnsMetadata.cat.categories.length, 2); + testClient.drain(done); + }); + }); + it('should provide row count as optional metadata', function(done) { var testClient = new TestClient({ version: '1.4.0', From ef36466b3b78f40e668777354c6f4478ed39e6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 18:19:41 +0200 Subject: [PATCH 156/185] Fix test to use the right helper function to set the user render timeout --- test/acceptance/user-render-timeout-limit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 93770080..6089468a 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -213,12 +213,12 @@ describe('user render timeout limit', function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; const mapconfig = createMapConfig(); this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserDatabaseTimeoutLimit(50, done); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); }); afterEach(function (done) { serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { if (err) { return done(err); } From dfc354550c0991f9d7b8ca67f2b7c7d4876c55e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 18:27:50 +0200 Subject: [PATCH 157/185] Add method to override cache buster provided by server while fetching tiles and static images --- test/support/test-client.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 5e08394a..2a5840f5 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -125,6 +125,14 @@ function resErr2errRes(callback) { }; } +function layergroupidTemplate (layergroupId, params) { + const { token, signer, cacheBuster } = LayergroupToken.parse(layergroupId); + + // {user}@{token}:{cache_buster} + // {token}:{cache_buster} + return `${signer ? signer + '@' : ''}${token}:${params.cacheBuster ? Date.now() : cacheBuster }`; +} + TestClient.prototype.getWidget = function(widgetName, params, callback) { var self = this; @@ -726,7 +734,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = '/api/v1/map/' + layergroupId + '/'; + url = `/api/v1/map/${layergroupidTemplate(layergroupId, params)}/`; var layers = params.layers; @@ -769,7 +777,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }, params.response); - var isPng = format.match(/png$/); if (isPng) { @@ -954,7 +961,9 @@ TestClient.prototype.getStaticCenter = function (params, callback) { self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; self.keysToDelete['user:localhost:mapviews:global'] = 5; - url = `/api/v1/map/static/center/${layergroupId}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`; + const layergroupid = layergroupidTemplate(layergroupId, params); + + url = `/api/v1/map/static/center/${layergroupid}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`; if (self.apiKey) { url += '?' + qs.stringify({api_key: self.apiKey}); From 87838bd4ce8bb71356523910dba6e875d936601c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 18:28:57 +0200 Subject: [PATCH 158/185] Override cache buster in order to not hit any internal cache --- test/acceptance/user-render-timeout-limit.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 6089468a..150ca5e9 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -139,7 +139,7 @@ describe('user render timeout limit', function () { }); it('layergroup creation works but tile request fails due to render timeout', function (done) { - this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => { + this.testClient.getTile(0, 0, 0, { cacheBuster: true }, (err, res, tile) => { assert.ifError(err); assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => { @@ -180,7 +180,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { @@ -234,7 +235,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/x-protobuf' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -277,7 +279,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { @@ -365,7 +368,7 @@ describe('user render timeout limit', function () { }); }); - it('layergroup creation works and render tile fails', function (done) { + it('layergroup creation works and render static center tile fails', function (done) { const params = { zoom: 0, lat: 0, @@ -378,7 +381,8 @@ describe('user render timeout limit', function () { headers: { 'Content-Type': 'application/json; charset=utf-8' } - } + }, + cacheBuster: true }; this.testClient.getStaticCenter(params, function (err, res, timeoutError) { From 8febc78d0e25e5cdc97338b64950a491b1665e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 19:20:37 +0200 Subject: [PATCH 159/185] Do not test against Postgis renderer because it does not implement renderer timeout option --- test/acceptance/user-render-timeout-limit.js | 89 +++++++++----------- 1 file changed, 42 insertions(+), 47 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 150ca5e9..79822c7b 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -202,57 +202,52 @@ describe('user render timeout limit', function () { }); }); - if (process.env.POSTGIS_VERSION === '2.4') { - describe('vector (PostGIS)', vector(true)); - } - - describe('vector (mapnik)', vector(false)); - function vector(usePostGIS) { + describe('vector tile via mapnik renderer', function () { + const usePostGIS = false; const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; - return function () { - beforeEach(function (done) { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); - }); - afterEach(function (done) { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); + }); + + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { + if (err) { + return done(err); + } + this.testClient.drain(done); + }); + }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/x-protobuf' } - this.testClient.drain(done); - }); + }, + cacheBuster: true + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); }); - - it('layergroup creation works but vector tile request fails due to render timeout', function (done) { - const params = { - format: 'mvt', - response: { - status: 429, - headers: { - 'Content-Type': 'application/x-protobuf' - } - }, - cacheBuster: true - }; - - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { - assert.ifError(err); - - var tileJSON = tile.toJSON(); - assert.equal(Array.isArray(tileJSON), true); - assert.equal(tileJSON.length, 2); - assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); - assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - - done(); - }); - }); - }; - } + }); + }); describe('interativity', function () { beforeEach(function (done) { From cc81c8ff4c8c237f336dd7fa3f961260de0c7ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 19:22:08 +0200 Subject: [PATCH 160/185] Only test user database timeout against Postgis Renderer if available (Postgis >= 2.4) --- .../acceptance/user-database-timeout-limit.js | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 52ead026..bdf70a12 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -2,6 +2,7 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; @@ -438,6 +439,78 @@ describe('user database timeout limit', function () { }); }); + + if (process.env.POSTGIS_VERSION === '2.4') { + describe.only('fetching vector tiles via PostGIS renderer', function() { + const usePostGIS = true; + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + const expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { + if (err) { + return done(err); + } + + this.layergroupid = res.layergroupid; + + done(); + }); + }); + + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.drain(done); + }); + + describe('with user\'s timeout of 200 ms', function () { + beforeEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(200, done); + }); + + afterEach(function (done) { + this.testClient.setUserDatabaseTimeoutLimit(0, done); + }); + + it('"mvt" fails due to statement timeout', function (done) { + const params = { + layergroupid: this.layergroupid, + format: 'mvt', + layers: [ 0 ], + response: { + status: 429, + headers: { + 'Content-Type': 'application/x-protobuf' + } + }, + cacheBuster: true + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); + }); + }); + }); + }); + } }); }); From 4701decfcdf6741810ffe055adbf92580b07d493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 19:37:46 +0200 Subject: [PATCH 161/185] No need to drain test-client --- test/acceptance/user-database-timeout-limit.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index bdf70a12..eecc679e 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -468,9 +468,8 @@ describe('user database timeout limit', function () { }); }); - afterEach(function (done) { + afterEach(function () { serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - this.testClient.drain(done); }); describe('with user\'s timeout of 200 ms', function () { From 54601db44afa1d5d7731cad003e6e35ed0d1f2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 22 May 2018 19:41:20 +0200 Subject: [PATCH 162/185] Remove test runner filter --- test/acceptance/user-database-timeout-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index eecc679e..6e5423e1 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -441,7 +441,7 @@ describe('user database timeout limit', function () { }); if (process.env.POSTGIS_VERSION === '2.4') { - describe.only('fetching vector tiles via PostGIS renderer', function() { + describe('fetching vector tiles via PostGIS renderer', function() { const usePostGIS = true; const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; From 94c34eeb23cae0cfb8aaa3070e2c06f8f5d63e31 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 23 May 2018 11:15:20 +0200 Subject: [PATCH 163/185] Small change to use const --- lib/cartodb/backends/layer-stats/mapnik-layer-stats.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index a04d68bc..129e4937 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -201,10 +201,9 @@ function _columnStats(ctx, columns) { ); if (columns[name].type === 'string') { const topN = ctx.metaOptions.columnStats.topCategories || 1024; - let includeNulls = true; - if (ctx.metaOptions.columnStats.hasOwnProperty('includeNulls')) { - includeNulls = ctx.metaOptions.columnStats.includeNulls; - } + const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ? + ctx.metaOptions.columnStats.includeNulls : + true; // TODO: ctx.metaOptions.columnStats.maxCategories // => use PG stats to dismiss columns with more distinct values From 1668392296c0512a86aad99250704cf7c70598b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 24 May 2018 13:42:32 +0200 Subject: [PATCH 164/185] Move 'postgres' attributes inside Mapnik's renderer configuration and keep params relative to user's connection --- config/environments/development.js.example | 47 +++++++++++----------- config/environments/production.js.example | 41 +++++++++++-------- config/environments/staging.js.example | 41 +++++++++++-------- lib/cartodb/server_options.js | 11 ++++- 4 files changed, 82 insertions(+), 58 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index f9c58abb..574eafed 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -115,32 +115,10 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS - type: "postgis", user: "publicuser", password: "public", host: '127.0.0.1', - port: 5432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - /* experimental - geometry_field: "the_geom", - extent: "-180,-90,180,90", - srid: 4326, - */ - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + port: 6432 } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -221,6 +199,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 6432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 89c5e5bd..fee05568 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -115,26 +115,10 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - max_size: 500 + port: 6432 } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -215,6 +199,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 6432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 76353f9b..8e806946 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -115,26 +115,10 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: '<%= user_password %>' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + port: 6432 } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -215,6 +199,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 6432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index c3d0fdb8..739f6bca 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -16,6 +16,15 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, { snapToGrid: false, clipByBox2d: false, metrics: false, + postgis: { + simplify_geometries: false, + extent: '-20037508.3,-20037508.3,20037508.3,20037508.3', + row_limit: 65535, + persist_connection: false, + use_overviews: true, + max_size: 500, + twkb_encoding: true + }, limits: {} }, http: {} @@ -118,7 +127,7 @@ module.exports = { // TODO: allow to specify in configuration srid: 3857 }, - datasource: global.environment.postgres, + datasource: rendererConfig.mapnik.postgis, cachedir: global.environment.millstone.cache_basedir, use_workers: rendererConfig.mapnik.useCartocssWorkers || false, mapnik_version: global.environment.mapnik_version, From ee38c717a5b0618dade286f377cd6a12f2d430d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 24 May 2018 13:53:29 +0200 Subject: [PATCH 165/185] Remove geojson renderer config --- config/environments/development.js.example | 19 ------------------- config/environments/production.js.example | 20 -------------------- config/environments/staging.js.example | 20 -------------------- config/environments/test.js.example | 20 -------------------- lib/cartodb/server_options.js | 11 +---------- 5 files changed, 1 insertion(+), 89 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 574eafed..9eff6616 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -235,25 +235,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, diff --git a/config/environments/production.js.example b/config/environments/production.js.example index fee05568..9c2647c7 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -235,26 +235,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 8e806946..6e396893 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -235,26 +235,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 1f4e34a6..56d1f9ef 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -227,26 +227,6 @@ var config = { cacheOnTimeout: true }, - geojson: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - }, - - // SQL queries will be wrapped with ST_ClipByBox2D - // Returning the portion of a geometry falling within a rectangle - // It will only work if snapToGrid is enabled - clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 - // geometries will be simplified using ST_RemoveRepeatedPoints - // which cost is no more expensive than snapping and results are - // much closer to the original geometry - removeRepeatedPoints: false // this requires postgis >=2.2 - }, - // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer 'cache-features': true, diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 739f6bca..2a2850b7 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -141,16 +141,7 @@ module.exports = { }, renderer: { mvt: rendererConfig.mvt, - mapnik: _.defaults(rendererConfig.mapnik, { - geojson: { - dbPoolParams: { - size: 16, - idleTimeout: 3000, - reapInterval: 1000 - }, - clipByBox2d: false - } - }), + mapnik: rendererConfig.mapnik, torque: rendererConfig.torque, http: rendererConfig.http }, From f4042851403a8333c7a221eafefbf9305876d1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 24 May 2018 18:36:45 +0200 Subject: [PATCH 166/185] Use postgres pool configuration to have the same configuration for all renderers. --- config/environments/development.js.example | 21 ++------------------- config/environments/production.js.example | 11 +---------- config/environments/staging.js.example | 21 ++------------------- config/environments/test.js.example | 11 +---------- lib/cartodb/server_options.js | 4 ++-- 5 files changed, 8 insertions(+), 60 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 9eff6616..d5b3327a 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -137,15 +137,7 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + usePostGIS: false }, mapnik: { // The size of the pool of internal mapnik backend @@ -261,16 +253,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 9c2647c7..97d97302 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -261,16 +261,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 6e396893..0dcd87b6 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -137,15 +137,7 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + usePostGIS: false }, mapnik: { // The size of the pool of internal mapnik backend @@ -261,16 +253,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 56d1f9ef..082d4aa4 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -250,16 +250,7 @@ var config = { src: __dirname + '/../../assets/default-placeholder.png' } }, - torque: { - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } - } + torque: {} } // anything analyses related ,analysis: { diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 2a2850b7..9410b2f4 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -140,9 +140,9 @@ module.exports = { statsInterval: rendererConfig.statsInterval }, renderer: { - mvt: rendererConfig.mvt, + mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt), mapnik: rendererConfig.mapnik, - torque: rendererConfig.torque, + torque: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.torque), http: rendererConfig.http }, From 92255797d917adc44b80e38671d4408cf9cf5431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 13:37:34 +0200 Subject: [PATCH 167/185] Add torque options to renderer factory --- lib/cartodb/api/api-router.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index b7534559..f69652a3 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -311,7 +311,8 @@ function createRendererFactory ({ redisPool, serverOptions, environmentOptions } mapnik: serverOptions.renderer.mapnik }, http: serverOptions.renderer.http, - mvt: serverOptions.renderer.mvt + mvt: serverOptions.renderer.mvt, + torque: serverOptions.renderer.torque }); From 8fe192267d6616d5658a169a45b1f69c99c88ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 13:37:57 +0200 Subject: [PATCH 168/185] cosmetic --- lib/cartodb/api/api-router.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index f69652a3..b0f42f03 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -315,6 +315,5 @@ function createRendererFactory ({ redisPool, serverOptions, environmentOptions } torque: serverOptions.renderer.torque }); - return rendererFactory; } From 1d0fa9a5f62ddf7fa3af4cff7a318f805fd8c085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 14:23:19 +0200 Subject: [PATCH 169/185] Use the right port --- config/environments/development.js.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index d5b3327a..024c0013 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -118,7 +118,7 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432 + port: 5432 } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' From d003790e4d9b5b746bae5407ce526a0de0ee1a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 14:25:43 +0200 Subject: [PATCH 170/185] Use the right port again --- config/environments/development.js.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 024c0013..503ac0d4 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -197,7 +197,7 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, + port: 5432, extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", // max number of rows to return when querying data, 0 means no limit row_limit: 65535, From 5938836ff55909bc8bafa00d91c693ae28f0c501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 14:36:06 +0200 Subject: [PATCH 171/185] Use default postgres port --- config/environments/production.js.example | 4 ++-- config/environments/staging.js.example | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 97d97302..0dc6275f 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -118,7 +118,7 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432 + port: 5432 } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -205,7 +205,7 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, + port: 5432, extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", // max number of rows to return when querying data, 0 means no limit row_limit: 65535, diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 0dcd87b6..67a49e78 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -118,7 +118,7 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432 + port: 5432 } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' @@ -197,7 +197,7 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 6432, + port: 5432, extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", // max number of rows to return when querying data, 0 means no limit row_limit: 65535, From bbab9c1a6bbfbff6e7b561950ba12a17e092db4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 14:44:58 +0200 Subject: [PATCH 172/185] Avoid breaking changes --- lib/cartodb/server_options.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 9410b2f4..dff1280e 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -127,7 +127,7 @@ module.exports = { // TODO: allow to specify in configuration srid: 3857 }, - datasource: rendererConfig.mapnik.postgis, + datasource: rendererConfig.mapnik.postgis || global.environment.postgres || {}, cachedir: global.environment.millstone.cache_basedir, use_workers: rendererConfig.mapnik.useCartocssWorkers || false, mapnik_version: global.environment.mapnik_version, From 4e4399b727f9a3da8d8742547bf838c84a679da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 18:00:53 +0200 Subject: [PATCH 173/185] Missing postgis configuration --- config/environments/test.js.example | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 082d4aa4..5619c8ac 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -119,21 +119,7 @@ var config = { user: "test_windshaft_publicuser", password: "public", host: '127.0.0.1', - port: 5432, - extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", - // max number of rows to return when querying data, 0 means no limit - row_limit: 65535, - simplify_geometries: true, - use_overviews: true, // use overviews to retrieve raster - /* - * Set persist_connection to false if you want - * database connections to be closed on renderer - * expiration (1 minute after last use). - * Setting to true (the default) would never - * close any connection for the server's lifetime - */ - persist_connection: false, - max_size: 500 + port: 5432 } ,mapnik_version: '' ,mapnik_tile_format: 'png8:m=h' @@ -214,6 +200,29 @@ var config = { // It will only work if snapToGrid is enabled clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5 + postgis: { + // Parameters to pass to datasource plugin of mapnik + // See http://github.com/mapnik/mapnik/wiki/PostGIS + user: "publicuser", + password: "public", + host: '127.0.0.1', + port: 5432, + extent: "-20037508.3,-20037508.3,20037508.3,20037508.3", + // max number of rows to return when querying data, 0 means no limit + row_limit: 65535, + /* + * Set persist_connection to false if you want + * database connections to be closed on renderer + * expiration (1 minute after last use). + * Setting to true (the default) would never + * close any connection for the server's lifetime + */ + persist_connection: false, + simplify_geometries: true, + use_overviews: true, // use overviews to retrieve raster + max_size: 500 + }, + limits: { // Time in milliseconds a render request can take before it fails, some notes: // - 0 means no render limit From b311f0e09196ec1fa652f1cff45fc756c853890a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 25 May 2018 18:38:31 +0200 Subject: [PATCH 174/185] Add generic pool configuration to postgres global config --- config/environments/development.js.example | 10 +++++++++- config/environments/production.js.example | 10 +++++++++- config/environments/staging.js.example | 10 +++++++++- config/environments/test.js.example | 10 +++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 503ac0d4..2a849fd0 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -118,7 +118,15 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 5432 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 0dc6275f..5b1d50b8 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -118,7 +118,15 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 5432 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 67a49e78..805f4ae7 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -118,7 +118,15 @@ var config = { user: "publicuser", password: "public", host: '127.0.0.1', - port: 5432 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 5619c8ac..f0484a6b 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -119,7 +119,15 @@ var config = { user: "test_windshaft_publicuser", password: "public", host: '127.0.0.1', - port: 5432 + port: 5432, + pool: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } } ,mapnik_version: '' ,mapnik_tile_format: 'png8:m=h' From 4a8bfcf6478a1c9fdc440181da4bb4edf09f3234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 28 May 2018 10:03:07 +0200 Subject: [PATCH 175/185] Revert this commit: try to see what's happening here --- test/acceptance/ported/server_gettile.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index 4dc089c9..ce8f0e96 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -39,13 +39,13 @@ describe('server_gettile', function() { // --{ //////////////////////////////////////////////////////////////////// - it("get'ing a tile with default style should return an expected tile", function(done){ + it.only("get'ing a tile with default style should return an expected tile", function(done){ testClient.getTile(testClient.defaultTableMapConfig('test_table'), 13, 4011, 3088, imageCompareFn('test_table_13_4011_3088.png', done) ); }); - it("response of get tile can be served by renderer cache", function(done) { + it.only("response of get tile can be served by renderer cache", function(done) { var tileUrl = '/13/4011/3088.png'; var lastXwc; var mapConfig = testClient.defaultTableMapConfig('test_table'); @@ -60,7 +60,7 @@ describe('server_gettile', function() { var xwc = res.headers['x-windshaft-cache']; assert.ok(xwc); assert.ok(xwc > 0); - assert.ok(xwc >= lastXwc); + assert.ok(xwc >= lastXwc, `xwc: ${xwc}, lastXwc: ${lastXwc}`); requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) { var xwc = res.headers['x-windshaft-cache']; From 4861d35628c66452363810b5247168bf9642631b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 28 May 2018 10:31:48 +0200 Subject: [PATCH 176/185] Cast to integer before comparing cache headers --- test/acceptance/ported/server_gettile.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index ce8f0e96..5a31cd69 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -51,19 +51,19 @@ describe('server_gettile', function() { var mapConfig = testClient.defaultTableMapConfig('test_table'); testClient.withLayergroup(mapConfig, function (err, requestTile, finish) { requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); lastXwc = xwc; requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); - assert.ok(xwc >= lastXwc, `xwc: ${xwc}, lastXwc: ${lastXwc}`); + assert.ok(xwc >= lastXwc); requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); finish(done); @@ -103,7 +103,7 @@ describe('server_gettile', function() { testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) { requestTile(tileUrl, function(err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); requestTile(tileUrl, function (err, res) { @@ -111,7 +111,7 @@ describe('server_gettile', function() { res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time" ); - assert.ok(res.headers['x-windshaft-cache'] >= 0); + assert.ok(parseInt(res.headers['x-windshaft-cache'], 10) >= 0); assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) { From 8b05d75f97dba64bb8dc9d4d3993b081bedf18fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 28 May 2018 10:32:31 +0200 Subject: [PATCH 177/185] Revert test runner filters --- test/acceptance/ported/server_gettile.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index 5a31cd69..b83a9720 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -39,13 +39,13 @@ describe('server_gettile', function() { // --{ //////////////////////////////////////////////////////////////////// - it.only("get'ing a tile with default style should return an expected tile", function(done){ + it("get'ing a tile with default style should return an expected tile", function(done){ testClient.getTile(testClient.defaultTableMapConfig('test_table'), 13, 4011, 3088, imageCompareFn('test_table_13_4011_3088.png', done) ); }); - it.only("response of get tile can be served by renderer cache", function(done) { + it("response of get tile can be served by renderer cache", function(done) { var tileUrl = '/13/4011/3088.png'; var lastXwc; var mapConfig = testClient.defaultTableMapConfig('test_table'); From 46bb400ffd514df232c2f68ddd52ad7ca5cbd31c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 28 May 2018 10:32:31 +0200 Subject: [PATCH 178/185] Cast x-windshaft-cache header in order to fix assertions --- test/acceptance/ported/server_gettile.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index 4dc089c9..b83a9720 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -51,19 +51,19 @@ describe('server_gettile', function() { var mapConfig = testClient.defaultTableMapConfig('test_table'); testClient.withLayergroup(mapConfig, function (err, requestTile, finish) { requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); lastXwc = xwc; requestTile(tileUrl, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(xwc); assert.ok(xwc > 0); assert.ok(xwc >= lastXwc); requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); finish(done); @@ -103,7 +103,7 @@ describe('server_gettile', function() { testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) { requestTile(tileUrl, function(err, res) { - var xwc = res.headers['x-windshaft-cache']; + var xwc = parseInt(res.headers['x-windshaft-cache'], 10); assert.ok(!xwc); requestTile(tileUrl, function (err, res) { @@ -111,7 +111,7 @@ describe('server_gettile', function() { res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time" ); - assert.ok(res.headers['x-windshaft-cache'] >= 0); + assert.ok(parseInt(res.headers['x-windshaft-cache'], 10) >= 0); assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) { From 4bb8914d9a02488ddf37bbc726eff74ed30ef4fd Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 28 May 2018 16:08:31 +0200 Subject: [PATCH 179/185] Add parameters to select metadata sample columns --- .../layer-stats/mapnik-layer-stats.js | 35 ++++++++++++++++--- lib/cartodb/utils/query-utils.js | 20 ++++++++--- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 3a7a3358..0caf2d0e 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -146,13 +146,36 @@ function mergeColumns(results) { } } -function _sample(ctx, numRows) { +const SAMPLE_SEED = 0.5; +const DEFAULT_SAMPLE_ROWS = 100; + +function exclude(items, excludedItems) { + if (excludedItems) { + return items.filter(item => !excludedItems.includes(item)); + } + return items; +} + +function _sample(ctx, numRows, availableColumns = null) { if (ctx.metaOptions.sample) { - const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); + const sampleProb = Math.min(ctx.metaOptions.sample.num_rows / numRows, 1); // We'll use a safety limit just in case numRows is a bad estimate - const limit = Math.ceil(ctx.metaOptions.sample * 1.5); - return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit))) - .then(res => ({ sample: res.rows })); + const requestedRows = ctx.metaOptions.sample.num_rows || DEFAULT_SAMPLE_ROWS; + const limit = Math.ceil(requestedRows * 1.5); + let columns = ctx.metaOptions.sample.include_columns; + if (columns) { + columns = exclude(columns, ctx.metaOptions.sample.exclude_columns); + } + else if (ctx.metaOptions.sample.exclude_columns) { + if (availableColumns === null) { + return Promise.reject(new Error('Column stats are needed to use the sample exclude columns options')); + } + columns = exclude(availableColumns, ctx.metaOptions.sample.exclude_columns); + } + return queryPromise(ctx.dbConnection, _getSQL( + ctx, + sql => queryUtils.getQuerySample(sql, sampleProb, limit, SAMPLE_SEED, columns) + )).then(res => ({ sample: res.rows })); } return Promise.resolve(); } @@ -265,6 +288,8 @@ function (layer, dbConnection, callback) { // (if metaOptions.geometryType) from it. // TODO: compute _sample with _featureCount when available + // TODO: add support for sample.exclude option by, in that case, forcing the columns query and + // passing the results to the sample query function. Promise.all([ _estimatedFeatureCount(ctx).then( diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index 22d99eef..ae1a2e2d 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -88,17 +88,27 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull `; }; -module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) { +function columnSelector(columns) { + if (!columns) { + return '*'; + } + if (typeof(columns) === 'string') { + return columns; + } + return columns.map(name => `"${name}"`).join(', '); +} + +module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5, columns = null) { const singleTable = simpleQueryTable(query); if (singleTable) { - return getTableSample(singleTable.table, singleTable.columns, sampleProb, limit, randomSeed); + return getTableSample(singleTable.table, columns || singleTable.columns, sampleProb, limit, randomSeed); } const limitClause = limit ? `LIMIT ${limit}` : ''; return ` WITH __cdb_rndseed AS ( SELECT setseed(${randomSeed}) ) - SELECT * + SELECT ${columnSelector(columns)} FROM (${query}) AS __cdb_query WHERE random() < ${sampleProb} ${limitClause} @@ -110,7 +120,9 @@ function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0 sampleProb *= 100; randomSeed *= Math.pow(2, 31) -1; return ` - SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause} + SELECT ${columnSelector(columns)} + FROM ${table} + TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause} `; } From 34a2f3b32bde01c847683fa3c8faea7874487680 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 28 May 2018 16:50:53 +0200 Subject: [PATCH 180/185] Tests were missing in previous commit --- .../stats/mapnik_stats_layergroup.js | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 46c4368a..4e8413ae 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -513,7 +513,7 @@ describe('Create mapnik layergroup', function() { version: '1.4.0', layers: [ layerWithMetadata(mapnikLayer4, { - sample: 3 + sample: { num_rows: 3 } }) ] }); @@ -529,6 +529,31 @@ describe('Create mapnik layergroup', function() { }); }); + it('can specify sample columns', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + sample: { + num_rows: 3, + include_columns: [ 'cartodb_id', 'address', 'the_geom' ] + } + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'address', 'the_geom' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + it('should only provide requested optional metadata', function(done) { var testClient = new TestClient({ version: '1.4.0', From 8a1d5d3a489cc26e94a4c181f99c64f33adea303 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 28 May 2018 17:36:58 +0200 Subject: [PATCH 181/185] Be careful and detect type invalid types --- lib/cartodb/utils/query-utils.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index ae1a2e2d..867db0cd 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -92,10 +92,13 @@ function columnSelector(columns) { if (!columns) { return '*'; } - if (typeof(columns) === 'string') { + if (typeof columns === 'string') { return columns; } - return columns.map(name => `"${name}"`).join(', '); + if (Array.isArray(columns)) { + return columns.map(name => `"${name}"`).join(', '); + } + throw new TypeError(`Bad argument type for columns: ${typeof columns}`); } module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5, columns = null) { From 26da872704afdbb89c82d8f241c76a3921b5ee12 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 28 May 2018 17:37:45 +0200 Subject: [PATCH 182/185] Leave sample exclude_columns for later --- .../backends/layer-stats/mapnik-layer-stats.js | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 0caf2d0e..d8320334 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -149,29 +149,13 @@ function mergeColumns(results) { const SAMPLE_SEED = 0.5; const DEFAULT_SAMPLE_ROWS = 100; -function exclude(items, excludedItems) { - if (excludedItems) { - return items.filter(item => !excludedItems.includes(item)); - } - return items; -} - -function _sample(ctx, numRows, availableColumns = null) { +function _sample(ctx, numRows) { if (ctx.metaOptions.sample) { const sampleProb = Math.min(ctx.metaOptions.sample.num_rows / numRows, 1); // We'll use a safety limit just in case numRows is a bad estimate const requestedRows = ctx.metaOptions.sample.num_rows || DEFAULT_SAMPLE_ROWS; const limit = Math.ceil(requestedRows * 1.5); let columns = ctx.metaOptions.sample.include_columns; - if (columns) { - columns = exclude(columns, ctx.metaOptions.sample.exclude_columns); - } - else if (ctx.metaOptions.sample.exclude_columns) { - if (availableColumns === null) { - return Promise.reject(new Error('Column stats are needed to use the sample exclude columns options')); - } - columns = exclude(availableColumns, ctx.metaOptions.sample.exclude_columns); - } return queryPromise(ctx.dbConnection, _getSQL( ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit, SAMPLE_SEED, columns) From 794c3efb7d7d9663f7ab583db41b46a3d6ab772e Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 28 May 2018 12:52:03 +0200 Subject: [PATCH 183/185] Update Windshaft, camshaft and cartodb-psql to use cartodb-psql 0.11.0 --- NEWS.md | 5 +++-- package.json | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 02c66777..fb2d8ea2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,16 +5,17 @@ Released 2018-mm-dd New features: - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5 -- Upgrades Windshaft to [4.8.0](https://github.com/CartoDB/Windshaft/blob/4.8.0/NEWS.md#version-480) which includes: +- Upgrades Windshaft to [4.8.1](https://github.com/CartoDB/Windshaft/blob/4.8.1/NEWS.md#version-481) which includes: - Update internal deps. - A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type. - Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible). - MVT: Disable simplify_distance to avoid multiple simplifications. - Fix a bug with zero length lines not being rendered when using the marker symbolizer. -- Upgrades Camshaft to [0.61.9](https://github.com/CartoDB/camshaft/releases/tag/0.61.9): +- Upgrades Camshaft to [0.61.10](https://github.com/CartoDB/camshaft/releases/tag/0.61.10): - Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses. - Update other deps: - body-parser: 1.18.3 + - cartodb-psql: 0.11.0 - dot: 1.1.2 - express: 4.16.3 - lru-cache: 4.1.3 diff --git a/package.json b/package.json index 40d75a38..488b59c3 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,8 @@ "dependencies": { "basic-auth": "2.0.0", "body-parser": "1.18.3", - "camshaft": "0.61.9", - "cartodb-psql": "0.10.2", + "camshaft": "0.61.10", + "cartodb-psql": "0.11.0", "cartodb-query-tables": "0.3.0", "cartodb-redis": "1.0.0", "debug": "3.1.0", @@ -48,7 +48,7 @@ "step-profiler": "0.3.0", "turbo-carto": "0.20.2", "underscore": "1.6.0", - "windshaft": "4.8.0", + "windshaft": "4.8.1", "yargs": "11.1.0" }, "devDependencies": { From 9c3b3e698e97abe54f22a96b3dfeab517e912d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 29 May 2018 16:28:23 +0200 Subject: [PATCH 184/185] Update yarn.lock --- yarn.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/yarn.lock b/yarn.lock index deb13479..4f98787a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -261,13 +261,13 @@ camelcase@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" -camshaft@0.61.9: - version "0.61.9" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.9.tgz#f3d399dfacf51b6a492c579e925c8b1141b22974" +camshaft@0.61.10: + version "0.61.10" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.10.tgz#9055bbc577dbd87d38a0e712bf6157afd7704dca" dependencies: async "^1.5.2" bunyan "1.8.1" - cartodb-psql "^0.10.1" + cartodb-psql "0.11.0" debug "^3.1.0" dot "^1.0.3" request "2.85.0" @@ -312,12 +312,12 @@ cartocolor@4.0.0: dependencies: colorbrewer "1.0.0" -cartodb-psql@0.10.2, cartodb-psql@^0.10.1: - version "0.10.2" - resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.10.2.tgz#8c505066e4a635cfa0ee4c603769c83f6e2187dd" +cartodb-psql@0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.11.0.tgz#6b4eae0876ee56944a61fe5f4acc6a8b0b11233f" dependencies: debug "^3.1.0" - pg cartodb/node-postgres#6.1.6-cdb1 + pg CartoDB/node-postgres#6.4.2-cdb1 underscore "~1.6.0" cartodb-query-tables@0.3.0: @@ -1804,9 +1804,9 @@ p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" -packet-reader@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.2.0.tgz#819df4d010b82d5ea5671f8a1a3acf039bcd7700" +packet-reader@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.3.1.tgz#cd62e60af8d7fea8a705ec4ff990871c46871f27" parse-json@^2.2.0: version "2.2.0" @@ -1885,20 +1885,20 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: - version "6.1.6" - resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" +"pg@github:CartoDB/node-postgres#6.4.2-cdb1": + version "6.4.2" + resolved "https://codeload.github.com/CartoDB/node-postgres/tar.gz/449fac1d6da711ffcc6694ae3c89f85244f48bdc" dependencies: buffer-writer "1.0.1" js-string-escape "1.0.1" - packet-reader "0.2.0" + packet-reader "0.3.1" pg-connection-string "0.1.3" pg-pool "1.*" pg-types "1.*" - pgpass "1.x" + pgpass "1.*" semver "4.3.2" -pgpass@1.x: +pgpass@1.*: version "1.0.2" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" dependencies: @@ -2802,16 +2802,16 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.8.0: - version "4.8.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.8.0.tgz#79ea03ee78fa68288e07c7aa01a9fdabe3cc5af0" +windshaft@4.8.1: + version "4.8.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.8.1.tgz#7c1ef21e4f885643f6347bc5871274c533fce206" dependencies: "@carto/mapnik" "3.6.2-carto.10" "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb9 abaculus cartodb/abaculus#2.0.3-cdb10 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.10.2" + cartodb-psql "0.11.0" debug "3.1.0" dot "1.1.2" grainstore "1.9.0" From 9a60ab07a829a2aa8210d759c280a356ccfa11cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 30 May 2018 16:51:37 +0200 Subject: [PATCH 185/185] Remove deprecated base url configuration --- config/environments/development.js.example | 15 --------------- config/environments/production.js.example | 15 --------------- config/environments/staging.js.example | 15 --------------- config/environments/test.js.example | 15 --------------- 4 files changed, 60 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 2a849fd0..d6345b82 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -13,21 +13,6 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.localhost' - // DEPRECATED: use routes property instead - // --------------------------------------- - // Base URLs for the APIs - // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' - // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 5b1d50b8..d601dbc5 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -13,21 +13,6 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' - // DEPRECATED: use routes property instead - // --------------------------------------- - // Base URLs for the APIs - // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' - // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 805f4ae7..76260b8f 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -13,21 +13,6 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '^(.*)\\.cartodb\\.com$' - // DEPRECATED: use routes property instead - // --------------------------------------- - // Base URLs for the APIs - // - // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/maps/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "/api/v1/maps" is the the new API, - // "/tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' - // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API diff --git a/config/environments/test.js.example b/config/environments/test.js.example index f0484a6b..0dd475fe 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -13,21 +13,6 @@ var config = { // from hostname. Must have a single grabbing block. ,user_from_host: '(.*)' - // DEPRECATED: use routes property instead - // --------------------------------------- - // Base URLs for the APIs - // - // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API - // - // Base url for the Templated Maps API - // "/api/v1/map/named" is the new API, - // "/tiles/template" is for compatibility with versions up to 1.6.x - ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)' - // Base url for the Detached Maps API - // "maps" is the the new API, - // "tiles/layergroup" is for compatibility with versions up to 1.6.x - ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' - // Base URLs for the APIs // // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API