From 10feea0d48ee75bc79b0cfb02887be31fc8c7052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 23 Sep 2016 17:11:04 +0200 Subject: [PATCH 1/3] Pass logger configuration to analysis backend and create a stream based on config --- config/environments/development.js.example | 6 ++++++ config/environments/production.js.example | 6 ++++++ config/environments/staging.js.example | 6 ++++++ config/environments/test.js.example | 6 ++++++ lib/cartodb/backends/analysis.js | 25 ++++++++++++++++++---- lib/cartodb/server.js | 1 + lib/cartodb/server_options.js | 6 ++++++ 7 files changed, 52 insertions(+), 4 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 98404825..1432b31f 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -210,6 +210,12 @@ var config = { endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', // the template to use for adding the host header in the batch api requests hostHeaderTemplate: '{{=it.username}}.localhost.lan' + }, + logger: { + // If filename is given logs comming from analysis client will be written + // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). + // Log file will be re-opened on receiving the HUP signal + filename: '/tmp/analysis.log' } } ,millstone: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 691b62e7..e7f8eb21 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -204,6 +204,12 @@ var config = { endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', // the template to use for adding the host header in the batch api requests hostHeaderTemplate: '{{=it.username}}.localhost.lan' + }, + logger: { + // If filename is given logs comming from analysis client will be written + // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). + // Log file will be re-opened on receiving the HUP signal + filename: 'logs/analysis.log' } } ,millstone: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index f223013a..08fd5c98 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -204,6 +204,12 @@ var config = { endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', // the template to use for adding the host header in the batch api requests hostHeaderTemplate: '{{=it.username}}.localhost.lan' + }, + logger: { + // If filename is given logs comming from analysis client will be written + // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). + // Log file will be re-opened on receiving the HUP signal + filename: 'logs/analysis.log' } } ,millstone: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 057be949..8a5fbce2 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -205,6 +205,12 @@ var config = { endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', // the template to use for adding the host header in the batch api requests hostHeaderTemplate: '{{=it.username}}.localhost.lan' + }, + logger: { + // If filename is given logs comming from analysis client will be written + // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). + // Log file will be re-opened on receiving the HUP signal + filename: 'logs/analysis.log' } } ,millstone: { diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js index 5855da46..b4a16888 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis.js @@ -1,19 +1,36 @@ var camshaft = require('camshaft'); +var fs = require('fs'); -function AnalysisBackend(options) { - var batchConfig = options.batch || {}; +function AnalysisBackend (options) { + options = options || {}; + this.setBatchConfig(options.batch); + this.setLoggerConfig(options.logger); +} + +module.exports = AnalysisBackend; + +AnalysisBackend.prototype.setBatchConfig = function (options) { + var batchConfig = options || {}; batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job'; batchConfig.inlineExecution = batchConfig.inlineExecution || false; batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan'; this.batchConfig = batchConfig; -} +}; -module.exports = AnalysisBackend; +AnalysisBackend.prototype.setLoggerConfig = function (options) { + var loggerConfig = options || {}; + loggerConfig.filename = loggerConfig.filename; + + this.loggerConfig = loggerConfig; +}; AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) { analysisConfiguration.batch.endpoint = this.batchConfig.endpoint; analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution; analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate; + analysisConfiguration.logger = { + stream: this.loggerConfig.filename ? fs.createWriteStream(this.loggerConfig.filename) : process.stdout + }; camshaft.create(analysisConfiguration, analysisDefinition, callback); }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e51b39c8..e302b4bc 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -147,6 +147,7 @@ module.exports = function(serverOptions) { var tileBackend = new windshaft.backend.Tile(rendererCache); var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + var analysisBackend = new AnalysisBackend(serverOptions.analysis); var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 22039b8c..052e30cc 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -36,6 +36,9 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, { inlineExecution: false, endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', hostHeaderTemplate: '{{=it.username}}.localhost.lan' + }, + logger: { + filename: undefined } }); @@ -95,6 +98,9 @@ module.exports = { inlineExecution: analysisConfig.batch.inlineExecution, endpoint: analysisConfig.batch.endpoint, hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate + }, + logger: { + filename: analysisConfig.logger.filename } }, // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 From f963fb321edf9f36f8e07bc027d9abc36355fa71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 23 Sep 2016 17:35:46 +0200 Subject: [PATCH 2/3] Set default config of analysis logger for test env --- config/environments/test.js.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 8a5fbce2..1350f7e6 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -210,7 +210,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'logs/analysis.log' + filename: 'node-windshaft.log' } } ,millstone: { From 2d75985cb3308e9e4ae3b632ac68d7b7b0a5bd42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 26 Sep 2016 17:39:27 +0200 Subject: [PATCH 3/3] Recreate stream on SIGHUP event --- lib/cartodb/backends/analysis.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js index b4a16888..4ac7ba2b 100644 --- a/lib/cartodb/backends/analysis.js +++ b/lib/cartodb/backends/analysis.js @@ -19,17 +19,31 @@ AnalysisBackend.prototype.setBatchConfig = function (options) { AnalysisBackend.prototype.setLoggerConfig = function (options) { var loggerConfig = options || {}; + loggerConfig.filename = loggerConfig.filename; this.loggerConfig = loggerConfig; + + if (this.loggerConfig.filename) { + this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' }); + + process.on('SIGHUP', function () { + if (this.stream) { + this.stream.destroy(); + } + + this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' }); + }.bind(this)); + } }; AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) { analysisConfiguration.batch.endpoint = this.batchConfig.endpoint; analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution; analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate; + analysisConfiguration.logger = { - stream: this.loggerConfig.filename ? fs.createWriteStream(this.loggerConfig.filename) : process.stdout + stream: this.stream ? this.stream : process.stdout }; camshaft.create(analysisConfiguration, analysisDefinition, callback);