cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1170 from CartoDB/dgaubert/ch78384/maps-api-replace-log4js-logger-by-pino

Browse Source 
Replace log4js logger by pino
This commit is contained in:
Daniel G. Aubert
2020-06-23 11:56:03 +02:00
committed by GitHub
parent b8d3971c8a 360b98254b
commit d65565c091
83 changed files with 1081 additions and 1155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
NEWS.md
View File

@@ -1,8 +1,22 @@
# Changelog
## 9.0.1
## 10.0.0
Released 2020-mm-dd
Breaking changes:
- Log system revamp:
- Logs to stdout, disabled while testing
- Upgrade `camshaft` to version [`0.66.0`](https://github.com/CartoDB/camshaft/releases/tag/0.66.0)
- Use header `X-Request-Id`, or create a new `uuid` when no present, to identyfy log entries
- Be able to set log level from env variable `LOG_LEVEL`, useful while testing: `LOG_LEVEL=info npm test`; even more human-readable: `LOG_LEVEL=info npm t | ./node_modules/.bin/pino-pretty`
- Stop responding with `X-Tiler-Errors` header. Now errors are properly logged and will end up in ELK as usual.
- Stop responding with `X-Tiler-Profiler` header. Now profiling stats are properly logged and will end up in ELK as usual.
- Be able to reduce the footprint in the final log file depending on the environment
- Be able to pass the logger to the analysis creation (camshaft) while instantiating a named map with analysis.
- Be able to tag requests with labels as an easier way to provide business metrics
- Metro: Add log-collector utility (`metro`), it will be moved to its own repository. Attaching it here fro development purposes. Try it with the following command `LOG_LEVEL=info npm t | node metro`
- Metro: Creates `metrics-collector.js` a stream to update Prometheus' counters and histograms and exposes them via Express' app (`:9145/metrics`). Use the ones defined in `grok_exporter`
Bug Fixes:
- While instantiating a map, set the `cache buster` equal to `0` when there are no affected tables in the MapConfig. Thus `layergroupid` has the same structure always:
- `${map_id}:${cache_buster}` for anonymous map

191
app.js
View File

@@ -1,28 +1,17 @@
'use strict';
var http = require('http');
var https = require('https');
var path = require('path');
var fs = require('fs');
var _ = require('underscore');
var semver = require('semver');
const http = require('http');
const https = require('https');
const path = require('path');
const semver = require('semver');
// TODO: research it it's still needed
const setICUEnvVariable = require('./lib/utils/icu-data-env-setter');
var log = console.log.bind(console);
var logError = console.error.bind(console);
var nodejsVersion = process.versions.node;
const { engines } = require('./package.json');
if (!semver.satisfies(nodejsVersion, engines.node)) {
logError(`Node version ${nodejsVersion} is not supported, please use Node.js ${engines.node}.`);
process.exit(1);
}
// This function should be called before the require('yargs').
setICUEnvVariable();
var argv = require('yargs')
const argv = require('yargs')
.usage('Usage: node $0 <environment> [options]')
.help('h')
.example(
@@ -35,96 +24,65 @@ var argv = require('yargs')
.describe('c', 'Load configuration from path')
.argv;
var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
if (!fs.existsSync(configurationFile)) {
logError('Configuration file "%s" does not exist', configurationFile);
process.exit(1);
}
const environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
const configurationFile = path.resolve(argv.config || `./config/environments/${environmentArg}.js`);
global.environment = require(configurationFile);
var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.environment.environment;
process.env.NODE_ENV = ENVIRONMENT;
process.env.NODE_ENV = argv._[0] || process.env.NODE_ENV || global.environment.environment;
var availableEnvironments = {
production: true,
staging: true,
development: true
};
// sanity check
if (!availableEnvironments[ENVIRONMENT]) {
logError('node app.js [environment]');
logError('environments: %s', Object.keys(availableEnvironments).join(', '));
process.exit(1);
}
process.env.NODE_ENV = ENVIRONMENT;
if (global.environment.uv_threadpool_size) {
process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
}
// set global HTTP and HTTPS agent default configurations
// ref https://nodejs.org/api/http.html#http_new_agent_options
var agentOptions = _.defaults(global.environment.httpAgent || {}, {
const agentOptions = Object.assign({
keepAlive: false,
keepAliveMsecs: 1000,
maxSockets: Infinity,
maxFreeSockets: 256
});
}, global.environment.httpAgent || {});
http.globalAgent = new http.Agent(agentOptions);
https.globalAgent = new https.Agent(agentOptions);
global.log4js = require('log4js');
var log4jsConfig = {
appenders: [],
replaceConsole: true
};
if (global.environment.log_filename) {
var logFilename = path.resolve(global.environment.log_filename);
var logDirectory = path.dirname(logFilename);
if (!fs.existsSync(logDirectory)) {
logError('Log filename directory does not exist: ' + logDirectory);
process.exit(1);
}
log('Logs will be written to ' + logFilename);
log4jsConfig.appenders.push(
{ type: 'file', absolute: true, filename: logFilename }
);
} else {
log4jsConfig.appenders.push(
{ type: 'console', layout: { type: 'basic' } }
);
}
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
// Include cartodb_windshaft only _after_ the "global" variable is set
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
var cartodbWindshaft = require('./lib/server');
var serverOptions = require('./lib/server-options');
const createServer = require('./lib/server');
const serverOptions = require('./lib/server-options');
const { logger } = serverOptions;
var server = cartodbWindshaft(serverOptions);
const availableEnvironments = {
production: true,
staging: true,
development: true
};
// Maximum number of connections for one process
// 128 is a good number if you have up to 1024 filedescriptors
// 4 is good if you have max 32 filedescriptors
// 1 is good if you have max 16 filedescriptors
var backlog = global.environment.maxConnections || 128;
if (!availableEnvironments[process.env.NODE_ENV]) {
logger.fatal(new Error(`Invalid environment argument, valid ones: ${Object.keys(availableEnvironments).join(', ')}`));
process.exit(1);
}
var listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, backlog);
const { engines } = require('./package.json');
if (!semver.satisfies(process.versions.node, engines.node)) {
logger.fatal(new Error(`Node version ${process.versions.node} is not supported, please use Node.js ${engines.node}.`));
process.exit(1);
}
var version = require('./package').version;
const server = createServer(serverOptions);
// Specify the maximum length of the queue of pending connections for the HTTP server.
// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
// The default value of this parameter is 511 (not 512).
// See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
const backlog = global.environment.maxConnections || 128;
const listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, backlog);
const { version, name } = require('./package');
listener.on('listening', function () {
log('Using Node.js %s', process.version);
log('Using configuration file "%s"', configurationFile);
log(
'Windshaft tileserver %s started on %s:%s PID=%d (%s)',
version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
);
const { address, port } = listener.address();
logger.info({ 'Node.js': process.version, pid: process.pid, environment: process.env.NODE_ENV, [name]: version, address, port, config: configurationFile }, `${name} initialized successfully`);
});
function getCPUUsage (oldUsage) {
@@ -159,22 +117,14 @@ setInterval(function cpuUsageMetrics () {
});
previousCPUUsage = CPUUsage;
}, 5000);
}, 5000).unref();
setInterval(function () {
var memoryUsage = process.memoryUsage();
Object.keys(memoryUsage).forEach(function (k) {
global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
});
}, 5000);
process.on('SIGHUP', function () {
global.log4js.clearAndShutdownAppenders(function () {
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
log('Log files reloaded');
});
});
}, 5000).unref();
if (global.gc) {
var gcInterval = Number.isFinite(global.environment.gc_interval)
@@ -184,7 +134,7 @@ if (global.gc) {
if (gcInterval > 0) {
setInterval(function gcForcedCycle () {
global.gc();
}, gcInterval);
}, gcInterval).unref();
}
}
@@ -227,41 +177,36 @@ function getGCTypeValue (type) {
return value;
}
addHandlers(listener, global.logger, 45000);
const exitProcess = logger.finish((err, finalLogger, listener, signal, killTimeout) => {
scheduleForcedExit(killTimeout, finalLogger);
function addHandlers (listener, logger, killTimeout) {
process.on('uncaughtException', exitProcess(listener, logger, killTimeout));
process.on('unhandledRejection', exitProcess(listener, logger, killTimeout));
process.on('ENOMEM', exitProcess(listener, logger, killTimeout));
process.on('SIGINT', exitProcess(listener, logger, killTimeout));
process.on('SIGTERM', exitProcess(listener, logger, killTimeout));
finalLogger.info(`Process has received signal: ${signal}`);
let code = 0;
if (err) {
code = 1;
finalLogger.fatal(err);
}
finalLogger.info(`Process is going to exit with code: ${code}`);
listener.close(() => process.exit(code));
});
function addHandlers (listener, killTimeout) {
process.on('uncaughtException', (err) => exitProcess(err, listener, 'uncaughtException', killTimeout));
process.on('unhandledRejection', (err) => exitProcess(err, listener, 'unhandledRejection', killTimeout));
process.on('ENOMEM', (err) => exitProcess(err, listener, 'ENOMEM', killTimeout));
process.on('SIGINT', () => exitProcess(null, listener, 'SIGINT', killTimeout));
process.on('SIGTERM', () => exitProcess(null, listener, 'SIGINT', killTimeout));
}
function exitProcess (listener, logger, killTimeout) {
return function exitProcessFn (signal) {
scheduleForcedExit(killTimeout, logger);
addHandlers(listener, 45000);
let code = 0;
if (!['SIGINT', 'SIGTERM'].includes(signal)) {
const err = signal instanceof Error ? signal : new Error(signal);
signal = undefined;
code = 1;
logger.fatal(err);
} else {
logger.info(`Process has received signal: ${signal}`);
}
logger.info(`Process is going to exit with code: ${code}`);
listener.close(() => global.log4js.shutdown(() => process.exit(code)));
};
}
function scheduleForcedExit (killTimeout, logger) {
function scheduleForcedExit (killTimeout, finalLogger) {
// Schedule exit if there is still ongoing work to deal with
const killTimer = setTimeout(() => {
logger.info('Process didn\'t close on time. Force exit');
finalLogger.info('Process didn\'t close on time. Force exit');
process.exit(1);
}, killTimeout);

19
config/environments/development.js.example
View File

@@ -67,9 +67,10 @@ var config = {
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
}
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
// Specify the maximum length of the queue of pending connections for the HTTP server.
// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
// The default value of this parameter is 511 (not 512).
// See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
,maxConnections:128
// Maximum number of templates per user. Unlimited by default.
,maxUserTemplates:1024
@@ -82,12 +83,6 @@ var config = {
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: undefined
,log_windshaft: true
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -262,12 +257,6 @@ var config = {
// the template to use for adding the host header in the batch api requests
hostHeaderTemplate: '{{=it.username}}.localhost.lan'
},
logger: {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: undefined
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {

19
config/environments/production.js.example
View File

@@ -67,9 +67,10 @@ var config = {
http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
}
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
// Specify the maximum length of the queue of pending connections for the HTTP server.
// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
// The default value of this parameter is 511 (not 512).
// See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
,maxConnections:128
// Maximum number of templates per user. Unlimited by default.
,maxUserTemplates:1024
@@ -82,12 +83,6 @@ var config = {
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: 'logs/node-windshaft.log'
,log_windshaft: true
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -262,12 +257,6 @@ var config = {
// the template to use for adding the host header in the batch api requests
hostHeaderTemplate: '{{=it.username}}.localhost.lan'
},
logger: {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'logs/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {

18
config/environments/staging.js.example
View File

@@ -67,9 +67,9 @@ var config = {
http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
}
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
// Specify the maximum length of the queue of pending connections for the HTTP server.
// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
// The default value of this parameter is 511 (not 512).
,maxConnections:128
// Maximum number of templates per user. Unlimited by default.
,maxUserTemplates:1024
@@ -82,12 +82,6 @@ var config = {
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: 'logs/node-windshaft.log'
,log_windshaft: true
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -262,12 +256,6 @@ var config = {
// the template to use for adding the host header in the batch api requests
hostHeaderTemplate: '{{=it.username}}.localhost.lan'
},
logger: {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'logs/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {

19
config/environments/test.js.example
View File

@@ -67,9 +67,10 @@ var config = {
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
}
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
// Specify the maximum length of the queue of pending connections for the HTTP server.
// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
// The default value of this parameter is 511 (not 512).
// See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
,maxConnections:128
// Maximum number of templates per user. Unlimited by default.
,maxUserTemplates:1024
@@ -82,12 +83,6 @@ var config = {
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: false
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: '/tmp/node-windshaft.log'
,log_windshaft: true
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@@ -264,12 +259,6 @@ var config = {
// the template to use for adding the host header in the batch api requests
hostHeaderTemplate: '{{=it.username}}.localhost.lan'
},
logger: {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: '/tmp/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {

45
lib/api/api-router.js
View File

@@ -47,10 +47,10 @@ const LayergroupMetadata = require('../utils/layergroup-metadata');
const RendererStatsReporter = require('../stats/reporter/renderer');
const initializeStatusCode = require('./middlewares/initialize-status-code');
const logger = require('./middlewares/logger');
const initLogger = require('./middlewares/logger');
const bodyParser = require('body-parser');
const servedByHostHeader = require('./middlewares/served-by-host-header');
const stats = require('./middlewares/stats');
const profiler = require('./middlewares/profiler');
const lzmaMiddleware = require('./middlewares/lzma');
const cors = require('./middlewares/cors');
const user = require('./middlewares/user');
@@ -83,15 +83,11 @@ module.exports = class ApiRouter {
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
});
const windshaftLogger = environmentOptions.log_windshaft && global.log4js
? global.log4js.getLogger('[windshaft]')
: null;
const { rendererCache, tileBackend, attributesBackend, previewBackend, mapBackend, mapStore } = windshaftFactory({
rendererOptions: serverOptions,
redisPool,
onTileErrorStrategy: getOnTileErrorStrategy({ enabled: environmentOptions.enabledFeatures.onTileErrorStrategy }),
logger: windshaftLogger
logger: this.serverOptions.logger
});
const rendererStatsReporter = new RendererStatsReporter(rendererCache, serverOptions.renderCache.statsInterval);
@@ -102,7 +98,7 @@ module.exports = class ApiRouter {
const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions);
const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache });
const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache, logger: this.serverOptions.logger });
const analysisStatusBackend = new AnalysisStatusBackend();
const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
@@ -203,20 +199,21 @@ module.exports = class ApiRouter {
const apiRouter = router({ mergeParams: true });
const { paths, middlewares = [] } = route;
apiRouter.use(initLogger({ logger: this.serverOptions.logger }));
apiRouter.use(profiler({
enabled: this.serverOptions.useProfiler,
statsClient: global.statsClient
}));
apiRouter.use(user(this.metadataBackend));
middlewares.forEach(middleware => apiRouter.use(middleware()));
apiRouter.use(logger(this.serverOptions));
apiRouter.use(initializeStatusCode());
apiRouter.use(bodyParser.json());
apiRouter.use(servedByHostHeader());
apiRouter.use(clientHeader());
apiRouter.use(stats({
enabled: this.serverOptions.useProfiler,
statsClient: global.statsClient
}));
apiRouter.use(lzmaMiddleware());
apiRouter.use(cors());
apiRouter.use(user(this.metadataBackend));
this.templateRouter.route(apiRouter, route.template);
this.mapRouter.route(apiRouter, route.map);
@@ -230,25 +227,19 @@ module.exports = class ApiRouter {
}
};
function createTemplateMaps ({ redisPool, surrogateKeysCache }) {
function createTemplateMaps ({ redisPool, surrogateKeysCache, logger }) {
const templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
function invalidateNamedMap (owner, templateName) {
var startTime = Date.now();
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function (err) {
var logMessage = JSON.stringify({
username: owner,
type: 'named_map_invalidation',
elapsed: Date.now() - startTime,
error: err ? JSON.stringify(err.message) : undefined
});
function invalidateNamedMap (user, templateName) {
const startTime = Date.now();
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(user, templateName), (err) => {
if (err) {
global.logger.warn(logMessage);
} else {
global.logger.info(logMessage);
return logger.error(err, `Named map (${templateName}) invalidation failed, user: ${user}`);
}
logger.info({ user, type: 'named_map_invalidation', elapsed: Date.now() - startTime }, `Named map (${templateName}) invalidation success, user: ${user}`);
});
}

2
lib/api/map/analyses-catalog-controller.js
View File

@@ -1,6 +1,7 @@
'use strict';
const PSQL = require('cartodb-psql');
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const authorize = require('../middlewares/authorize');
@@ -23,6 +24,7 @@ module.exports = class AnalysesController {
middlewares () {
return [
tag({ tags: ['analysis', 'catalog'] }),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),

2
lib/api/map/analysis-layergroup-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@@ -23,6 +24,7 @@ module.exports = class AnalysisLayergroupController {
middlewares () {
return [
tag({ tags: ['analysis', 'node'] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),

16
lib/api/map/anonymous-map-controller.js
View File

@@ -3,11 +3,11 @@
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const initProfiler = require('../middlewares/init-profiler');
const checkJsonContentType = require('../middlewares/check-json-content-type');
const incrementMapViewCount = require('../middlewares/increment-map-view-count');
const augmentLayergroupData = require('../middlewares/augment-layergroup-data');
@@ -76,7 +76,6 @@ module.exports = class AnonymousMapController {
}
middlewares () {
const isTemplateInstantiation = false;
const useTemplateHash = false;
const includeQuery = true;
const label = 'ANONYMOUS LAYERGROUP';
@@ -92,10 +91,10 @@ module.exports = class AnonymousMapController {
};
return [
tag({ tags: ['map', 'anonymous'] }),
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,
logger: global.logger,
tags: metricsTags
}),
credentials(),
@@ -103,7 +102,6 @@ module.exports = class AnonymousMapController {
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS),
cleanUpQueryParams(['aggregation']),
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
@@ -144,7 +142,6 @@ function checkCreateLayergroup () {
}
}
req.profiler.done('checkCreateLayergroup');
return next();
};
}
@@ -153,6 +150,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
return function prepareAdapterMapConfigMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { logger } = res.locals;
const { user, api_key: apiKey } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
@@ -160,6 +158,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
const context = {
analysisConfiguration: {
user,
logger,
db: {
host: dbhost,
port: dbport,
@@ -178,12 +177,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
requestMapConfig,
params,
context,
(err, requestMapConfig, stats = { overviewsAddedToMapconfig: false }) => {
req.profiler.done('anonymous.getMapConfig');
stats.mapType = 'anonymous';
req.profiler.add(stats);
(err, requestMapConfig) => {
if (err) {
return next(err);
}

4
lib/api/map/attributes-layergroup-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@@ -38,6 +39,7 @@ module.exports = class AttributesLayergroupController {
middlewares () {
return [
tag({ tags: ['attributes'] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),
@@ -61,8 +63,6 @@ module.exports = class AttributesLayergroupController {
function getFeatureAttributes (attributesBackend) {
return function getFeatureAttributesMiddleware (req, res, next) {
req.profiler.start('windshaft.maplayer_attribute');
const { mapConfigProvider } = res.locals;
const { token } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;

4
lib/api/map/clustered-features-layergroup-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@@ -38,6 +39,7 @@ module.exports = class AggregatedFeaturesLayergroupController {
middlewares () {
return [
tag({ tags: ['cluster'] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),
@@ -62,8 +64,6 @@ module.exports = class AggregatedFeaturesLayergroupController {
function getClusteredFeatures (clusterBackend) {
return function getFeatureAttributesMiddleware (req, res, next) {
req.profiler.start('windshaft.maplayer_cluster_features');
const { mapConfigProvider } = res.locals;
const { user, token } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;

2
lib/api/map/dataview-layergroup-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@@ -76,6 +77,7 @@ module.exports = class DataviewLayergroupController {
middlewares ({ action, rateLimitGroup }) {
return [
tag({ tags: ['dataview', action] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),

4
lib/api/map/preview-layergroup-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@@ -61,6 +62,7 @@ module.exports = class PreviewLayergroupController {
}
return [
tag({ tags: ['static', 'tile'] }),
layergroupToken(),
validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(),
credentials(),
@@ -100,7 +102,6 @@ function getPreviewImageByCenter (previewBackend) {
const options = { mapConfigProvider, format, width, height, zoom, center };
previewBackend.getImage(options, (err, image, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
@@ -133,7 +134,6 @@ function getPreviewImageByBoundingBox (previewBackend) {
const options = { mapConfigProvider, format, width, height, bbox };
previewBackend.getImage(options, (err, image, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {

26
lib/api/map/preview-template-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
@@ -67,10 +68,10 @@ module.exports = class PreviewTemplateController {
};
return [
tag({ tags: ['named', 'static', 'tile'] }),
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,
logger: global.logger,
tags: metricsTags
}),
credentials(),
@@ -293,7 +294,7 @@ function getImage ({ previewBackend, label }) {
if (zoom !== undefined && center) {
const options = { mapConfigProvider, format, width, height, zoom, center };
return previewBackend.getImage(options, (err, image, stats) => {
return previewBackend.getImage(options, (err, image, stats = {}) => {
req.profiler.add(stats);
if (err) {
@@ -310,9 +311,8 @@ function getImage ({ previewBackend, label }) {
const options = { mapConfigProvider, format, width, height, bbox };
previewBackend.getImage(options, (err, image, stats) => {
previewBackend.getImage(options, (err, image, stats = {}) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
@@ -337,29 +337,25 @@ function setContentTypeHeader () {
};
}
function incrementMapViewsError (ctx) {
return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
}
function incrementMapViews ({ metadataBackend }) {
return function incrementMapViewsMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { user, mapConfigProvider, logger } = res.locals;
mapConfigProvider.getMapConfig((err, mapConfig) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
err.message = `Failed to increment mapview count for user '${user}'. ${err.message}`;
logger.warn({ error: err });
return next();
}
const statTag = mapConfig.obj().stat_tag;
res.locals.mapConfig = mapConfig;
if (statTag) {
res.set('Carto-Stat-Tag', `${statTag}`);
}
const statTag = mapConfig.obj().stat_tag;
metadataBackend.incMapviewCount(user, statTag, (err) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
err.message = `Failed to increment mapview count for user '${user}'. ${err.message}`;
logger.warn({ error: err });
}
next();

4
lib/api/map/tile-layergroup-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@@ -59,6 +60,7 @@ module.exports = class TileLayergroupController {
middlewares () {
return [
tag({ tags: ['tile'] }),
layergroupToken(),
coordinates(),
credentials(),
@@ -96,8 +98,6 @@ function getStatusCode (tile, format) {
function getTile (tileBackend) {
return function getTileMiddleware (req, res, next) {
req.profiler.start(`windshaft.${req.params.layer ? 'maplayer_tile' : 'map_tile'}`);
const { mapConfigProvider } = res.locals;
const { token } = res.locals;
const { layer, z, x, y, format } = req.params;

2
lib/api/middlewares/authorize.js
View File

@@ -3,8 +3,6 @@
module.exports = function authorize (authBackend) {
return function authorizeMiddleware (req, res, next) {
authBackend.authorize(req, res, (err, authorized) => {
req.profiler.done('authorize');
if (err) {
return next(err);
}

5
lib/api/middlewares/cache-channel-header.js
View File

@@ -6,11 +6,12 @@ module.exports = function setCacheChannelHeader () {
return next();
}
const { mapConfigProvider } = res.locals;
const { mapConfigProvider, logger } = res.locals;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Cache Channel Header:', err);
err.message = `Error generating Cache Channel Header. ${err.message}`;
logger.warn({ error: err });
return next();
}

5
lib/api/middlewares/cache-control-header.js
View File

@@ -40,11 +40,12 @@ module.exports = function setCacheControlHeader ({
return next();
}
const { mapConfigProvider = { getAffectedTables: callback => callback() } } = res.locals;
const { mapConfigProvider = { getAffectedTables: callback => callback() }, logger } = res.locals;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Cache Control Header:', err);
err.message = `Error generating Cache Control Header. ${err.message}`;
logger.warn({ error: err });
return next();
}

2
lib/api/middlewares/check-json-content-type.js
View File

@@ -6,8 +6,6 @@ module.exports = function checkJsonContentType () {
return next(new Error('POST data must be of type application/json'));
}
req.profiler.done('checkJsonContentTypeMiddleware');
next();
};
};

2
lib/api/middlewares/db-conn-setup.js
View File

@@ -7,8 +7,6 @@ module.exports = function dbConnSetup (pgConnection) {
const { user } = res.locals;
pgConnection.setDBConn(user, res.locals, (err) => {
req.profiler.done('dbConnSetup');
if (err) {
if (err.message && err.message.indexOf('name not found') !== -1) {
err.http_status = 404;

101
lib/api/middlewares/error-middleware.js
View File

@@ -1,42 +1,31 @@
'use strict';
const _ = require('underscore');
const debug = require('debug')('windshaft:cartodb:error-middleware');
const setCommonHeaders = require('../../utils/common-headers');
module.exports = function errorMiddleware (/* options */) {
return function error (err, req, res, next) {
var allErrors = Array.isArray(err) ? err : [err];
const { logger } = res.locals;
const errors = populateLimitErrors(Array.isArray(err) ? err : [err]);
allErrors = populateLimitErrors(allErrors);
logger.error({ error: errors });
const label = err.label || 'UNKNOWN';
err = allErrors[0] || new Error(label);
allErrors[0] = err;
setCommonHeaders(req, res, () => {
const errorResponseBody = {
errors: errors.map(errorMessage),
errors_with_context: errors.map(errorMessageWithContext)
};
var statusCode = findStatusCode(err);
// If a callback was requested, force status to 200
res.status(req.query.callback ? 200 : findStatusCode(errors[0]));
setErrorHeader(allErrors, statusCode, res);
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
if (req.query && req.query.callback) {
res.jsonp(errorResponseBody);
} else {
res.json(errorResponseBody);
}
// If a callback was requested, force status to 200
if (req.query && req.query.callback) {
statusCode = 200;
}
var errorResponseBody = {
errors: allErrors.map(errorMessage),
errors_with_context: allErrors.map(errorMessageWithContext)
};
res.status(statusCode);
if (req.query && req.query.callback) {
res.jsonp(errorResponseBody);
} else {
res.json(errorResponseBody);
}
return next();
return next();
});
};
};
@@ -135,7 +124,7 @@ function statusFromErrorMessage (errMsg) {
function errorMessage (err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var message = (_.isString(err) ? err : err.message) || 'Unknown error';
var message = (typeof err === 'string' ? err : err.message) || 'Unknown error';
return stripConnectionInfo(message);
}
@@ -165,7 +154,7 @@ function shouldBeExposed (prop) {
function errorMessageWithContext (err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var message = (_.isString(err) ? err : err.message) || 'Unknown error';
var message = (typeof err === 'string' ? err : err.message) || 'Unknown error';
var error = {
type: err.type || 'unknown',
@@ -181,53 +170,3 @@ function errorMessageWithContext (err) {
return error;
}
function setErrorHeader (errors, statusCode, res) {
const errorsCopy = errors.slice(0);
const mainError = errorsCopy.shift();
const errorsLog = {
mainError: {
statusCode: statusCode || 200,
message: mainError.message,
name: mainError.name,
label: mainError.label,
type: mainError.type,
subtype: mainError.subtype
}
};
errorsLog.moreErrors = errorsCopy.map(error => {
return {
message: error.message,
name: error.name,
label: error.label,
type: error.type,
subtype: error.subtype
};
});
res.set('X-Tiler-Errors', stringifyForLogs(errorsLog));
}
/**
* Remove problematic nested characters
* from object for logs RegEx
*
* @param {Object} object
*/
function stringifyForLogs (object) {
Object.keys(object).map(key => {
if (typeof object[key] === 'string') {
object[key] = object[key].replace(/[^a-zA-Z0-9]/g, ' ');
} else if (typeof object[key] === 'object') {
stringifyForLogs(object[key]);
} else if (object[key] instanceof Array) {
for (const element of object[key]) {
stringifyForLogs(element);
}
}
});
return JSON.stringify(object);
}

12
lib/api/middlewares/increment-map-view-count.js
View File

@@ -2,19 +2,13 @@
module.exports = function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware (req, res, next) {
const { mapConfig, user } = res.locals;
const { mapConfig, user, logger } = res.locals;
const statTag = mapConfig.obj().stat_tag;
if (statTag) {
res.set('Carto-Stat-Tag', `${statTag}`);
}
// Error won't blow up, just be logged.
metadataBackend.incMapviewCount(user, statTag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`);
err.message = `Failed to increment mapview count for user '${user}'. ${err.message}`;
logger.warn({ error: err });
}
next();

11
lib/api/middlewares/init-profiler.js
View File

@@ -1,11 +0,0 @@
'use strict';
module.exports = function initProfiler (isTemplateInstantiation) {
const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
return function initProfilerMiddleware (req, res, next) {
req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`);
req.profiler.done(`${operation}.initProfilerMiddleware`);
next();
};
};

5
lib/api/middlewares/last-modified-header.js
View File

@@ -6,7 +6,7 @@ module.exports = function setLastModifiedHeader () {
return next();
}
const { mapConfigProvider, cache_buster: cacheBuster } = res.locals;
const { mapConfigProvider, cache_buster: cacheBuster, logger } = res.locals;
if (cacheBuster) {
const cacheBusterTimestamp = parseInt(cacheBuster, 10);
@@ -21,7 +21,8 @@ module.exports = function setLastModifiedHeader () {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Last Modified Header:', err);
err.message = `Error generating Last Modified Header. ${err.message}`;
logger.warn({ error: err });
return next();
}

30
lib/api/middlewares/logger.js
View File

@@ -1,24 +1,16 @@
'use strict';
module.exports = function logger (options) {
if (!global.log4js || !options.log_format) {
return function dummyLoggerMiddleware (req, res, next) {
next();
};
}
const uuid = require('uuid');
const opts = {
level: 'info',
// Allowing for unbuffered logging is mainly
// used to avoid hanging during unit testing.
// TODO: provide an explicit teardown function instead,
// releasing any event handler or timer set by
// this component.
buffer: !options.unbuffered_logging,
// optional log format
format: options.log_format
module.exports = function initLogger ({ logger }) {
return function initLoggerMiddleware (req, res, next) {
const id = req.get('X-Request-Id') || uuid.v4();
res.locals.logger = logger.child({ id });
res.locals.logger.info({ request: req });
res.on('finish', () => res.locals.logger.info({ response: res }));
res.on('close', () => res.locals.logger.info({ end: true }));
next();
};
const logger = global.log4js.getLogger();
return global.log4js.connectLogger(logger, opts);
};

2
lib/api/middlewares/lzma.js
View File

@@ -24,8 +24,6 @@ module.exports = function lzma () {
delete req.query.lzma;
Object.assign(req.query, JSON.parse(result));
req.profiler.done('lzma');
next();
} catch (err) {
next(new Error('Error parsing lzma as JSON: ' + err));

1
lib/api/middlewares/map-error.js
View File

@@ -4,7 +4,6 @@ module.exports = function mapError (options) {
const { addContext = false, label = 'MAPS CONTROLLER' } = options;
return function mapErrorMiddleware (err, req, res, next) {
req.profiler.done('error');
const { mapConfig } = res.locals;
if (addContext) {

17
lib/api/middlewares/metrics.js
View File

@@ -3,7 +3,7 @@
const EVENT_VERSION = '1';
const MAX_LENGTH = 100;
module.exports = function metrics ({ enabled, tags, metricsBackend, logger }) {
module.exports = function metrics ({ enabled, tags, metricsBackend }) {
if (!enabled) {
return function metricsDisabledMiddleware (req, res, next) {
next();
@@ -15,11 +15,15 @@ module.exports = function metrics ({ enabled, tags, metricsBackend, logger }) {
}
return function metricsMiddleware (req, res, next) {
// FIXME: use parent logger as we don't want bind the error to the request
// but we still want to know if an error is thrown
const { logger } = res.locals;
res.on('finish', () => {
const { event, attributes } = getEventData(req, res, tags);
metricsBackend.send(event, attributes)
.catch((error) => logger.error(`Failed to publish event "${event}": ${error.message}`));
.catch((err) => logger.error(err, `Failed to publish event "${event}"`));
});
return next();
@@ -51,7 +55,7 @@ function getEventData (req, res, tags) {
template_hash: getTemplateHash({ res }),
stat_tag: getStatTag({ res }),
response_code: res.statusCode.toString(),
response_time: getResponseTime(res),
response_time: getResponseTime(req),
source_domain: req.hostname,
event_version: EVENT_VERSION
}, tags.attributes, extra);
@@ -121,13 +125,12 @@ function getStatTag ({ res }) {
}
}
// FIXME: 'X-Tiler-Profiler' might not be accurate enough
function getResponseTime (res) {
const profiler = res.get('X-Tiler-Profiler');
// FIXME: 'Profiler' might not be accurate enough
function getResponseTime (req) {
let stats;
try {
stats = JSON.parse(profiler);
stats = req.profiler.toJSON();
} catch (e) {
return undefined;
}

15
lib/api/middlewares/stats.js → lib/api/middlewares/profiler.js
View File

@@ -2,20 +2,27 @@
const Profiler = require('../../stats/profiler-proxy');
const debug = require('debug')('windshaft:cartodb:stats');
const onHeaders = require('on-headers');
const { name: prefix } = require('../../../package.json');
module.exports = function stats (options) {
module.exports = function profiler (options) {
const { enabled = true, statsClient } = options;
return function statsMiddleware (req, res, next) {
return function profilerMiddleware (req, res, next) {
const { logger } = res.locals;
// TODO: stop using profiler and log stats instead of adding them to the profiler
req.profiler = new Profiler({
statsd_client: statsClient,
profile: enabled
});
onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString()));
req.profiler.start(prefix);
res.on('finish', () => {
req.profiler.done('response');
req.profiler.end();
logger.info({ stats: req.profiler.toJSON() });
try {
// May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166
req.profiler.sendStats();

28
lib/api/middlewares/send-response.js
View File

@@ -1,22 +1,24 @@
'use strict';
const setCommonHeaders = require('../../utils/common-headers');
module.exports = function sendResponse () {
return function sendResponseMiddleware (req, res, next) {
req.profiler.done('res');
setCommonHeaders(req, res, () => {
res.status(res.statusCode);
res.status(res.statusCode);
if (Buffer.isBuffer(res.body)) {
res.send(res.body);
return next();
}
if (Buffer.isBuffer(res.body)) {
res.send(res.body);
if (req.query.callback) {
res.jsonp(res.body);
return next();
}
res.json(res.body);
return next();
}
if (req.query.callback) {
res.jsonp(res.body);
return next();
}
res.json(res.body);
return next();
});
};
};

5
lib/api/middlewares/surrogate-key-header.js
View File

@@ -5,7 +5,7 @@ const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named
module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { user, mapConfigProvider, logger } = res.locals;
if (mapConfigProvider instanceof NamedMapMapConfigProvider) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName()));
@@ -17,7 +17,8 @@ module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Surrogate Key Header:', err);
err.message = `Erros generating Surrogate Key Header. ${err.message}`;
logger.warn({ error: err });
return next();
}

15
lib/api/middlewares/tag.js Normal file
View File

@@ -0,0 +1,15 @@
'use strict';
module.exports = function tag ({ tags }) {
if (!Array.isArray(tags) || !tags.every((tag) => typeof tag === 'string')) {
throw new Error('Required "tags" option must be a valid Array: [string, string, ...]');
}
return function tagMiddleware (req, res, next) {
const { logger } = res.locals;
res.locals.tags = tags;
res.on('finish', () => logger.info({ tags: res.locals.tags }));
next();
};
};

10
lib/api/middlewares/user.js
View File

@@ -6,7 +6,13 @@ module.exports = function user (metadataBackend) {
const cdbRequest = new CdbRequest();
return function userMiddleware (req, res, next) {
res.locals.user = getUserNameFromRequest(req, cdbRequest);
const { logger } = res.locals;
try {
res.locals.user = getUserNameFromRequest(req, cdbRequest);
logger.info({ user: res.locals.user });
} catch (err) {
return next(err);
}
metadataBackend.getUserId(res.locals.user, (err, userId) => {
if (err || !userId) {
@@ -14,7 +20,7 @@ module.exports = function user (metadataBackend) {
}
res.locals.userId = userId;
res.set('Carto-User-Id', `${userId}`);
return next();
});
};

8
lib/api/template/admin-template-controller.js
View File

@@ -1,6 +1,7 @@
'use strict';
const { templateName } = require('../../backends/template-maps');
const tag = require('../middlewares/tag');
const credentials = require('../middlewares/credentials');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
@@ -76,6 +77,7 @@ module.exports = class AdminTemplateController {
}
return [
tag({ tags: ['named', 'admin', action] }),
credentials(),
authorizedByAPIKey({ authBackend: this.authBackend, action, label }),
rateLimit(this.userLimitsBackend, rateLimitGroup),
@@ -166,8 +168,6 @@ function updateTemplate ({ templateMaps }) {
function retrieveTemplate ({ templateMaps }) {
return function retrieveTemplateMiddleware (req, res, next) {
req.profiler.start('windshaft-cartodb.get_template');
const { user } = res.locals;
const templateId = templateName(req.params.template_id);
@@ -195,8 +195,6 @@ function retrieveTemplate ({ templateMaps }) {
function destroyTemplate ({ templateMaps }) {
return function destroyTemplateMiddleware (req, res, next) {
req.profiler.start('windshaft-cartodb.delete_template');
const { user } = res.locals;
const templateId = templateName(req.params.template_id);
@@ -215,8 +213,6 @@ function destroyTemplate ({ templateMaps }) {
function listTemplates ({ templateMaps }) {
return function listTemplatesMiddleware (req, res, next) {
req.profiler.start('windshaft-cartodb.get_template_list');
const { user } = res.locals;
templateMaps.listTemplates(user, (err, templateIds) => {

13
lib/api/template/named-template-controller.js
View File

@@ -1,10 +1,10 @@
'use strict';
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const initProfiler = require('../middlewares/init-profiler');
const checkJsonContentType = require('../middlewares/check-json-content-type');
const incrementMapViewCount = require('../middlewares/increment-map-view-count');
const augmentLayergroupData = require('../middlewares/augment-layergroup-data');
@@ -74,7 +74,6 @@ module.exports = class NamedMapController {
}
middlewares () {
const isTemplateInstantiation = true;
const useTemplateHash = true;
const includeQuery = false;
const label = 'NAMED MAP LAYERGROUP';
@@ -90,10 +89,10 @@ module.exports = class NamedMapController {
};
return [
tag({ tags: ['map', 'named'] }),
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,
logger: global.logger,
tags: metricsTags
}),
credentials(),
@@ -101,7 +100,6 @@ module.exports = class NamedMapController {
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED),
cleanUpQueryParams(['aggregation']),
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
checkInstantiteLayergroup(),
getTemplate(
@@ -151,8 +149,6 @@ function checkInstantiteLayergroup () {
}
}
req.profiler.done('checkInstantiteLayergroup');
return next();
};
}
@@ -187,10 +183,9 @@ function getTemplate (
params
);
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams, context, stats = {}) => {
req.profiler.done('named.getMapConfig');
mapConfigProvider.logger = res.locals.logger;
stats.mapType = 'named';
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams, context, stats = {}) => {
req.profiler.add(stats);
if (err) {

5
lib/api/template/tile-template-controller.js
View File

@@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@@ -37,6 +38,7 @@ module.exports = class TileTemplateController {
middlewares () {
return [
tag({ tags: ['tile', 'named'] }),
coordinates(),
credentials(),
authorize(this.authBackend),
@@ -67,9 +69,8 @@ function getTile ({ tileBackend, label }) {
const { layer, z, x, y, format } = req.params;
const params = { layer, z, x, y, format };
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => {
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;

22
lib/backends/analysis.js
View File

@@ -2,7 +2,6 @@
var _ = require('underscore');
var camshaft = require('camshaft');
var fs = require('fs');
var REDIS_LIMITS = {
DB: 5,
@@ -14,7 +13,6 @@ function AnalysisBackend (metadataBackend, options) {
this.options = options || {};
this.options.limits = this.options.limits || {};
this.setBatchConfig(this.options.batch);
this.setLoggerConfig(this.options.logger);
}
module.exports = AnalysisBackend;
@@ -27,31 +25,11 @@ AnalysisBackend.prototype.setBatchConfig = function (options) {
this.batchConfig = batchConfig;
};
AnalysisBackend.prototype.setLoggerConfig = function (options) {
this.loggerConfig = options || {};
if (this.loggerConfig.filename) {
this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
process.on('SIGHUP', function () {
if (this.stream) {
this.stream.destroy();
}
this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
}.bind(this));
}
};
AnalysisBackend.prototype.create = function (analysisConfiguration, analysisDefinition, callback) {
analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
analysisConfiguration.logger = {
stream: this.stream ? this.stream : process.stdout
};
this.getAnalysesLimits(analysisConfiguration.user, function (err, limits) {
if (err) {}
analysisConfiguration.limits = limits || {};

6
lib/backends/auth.js
View File

@@ -133,8 +133,6 @@ AuthBackend.prototype.authorize = function (req, res, callback) {
if (isAuthorizedByApikey) {
return this.pgConnection.setDBAuth(user, res.locals, 'regular', function (err) {
req.profiler.done('setDBAuth');
if (err) {
return callback(err);
}
@@ -150,8 +148,6 @@ AuthBackend.prototype.authorize = function (req, res, callback) {
if (isAuthorizedBySigner) {
return this.pgConnection.setDBAuth(user, res.locals, 'master', function (err) {
req.profiler.done('setDBAuth');
if (err) {
return callback(err);
}
@@ -163,8 +159,6 @@ AuthBackend.prototype.authorize = function (req, res, callback) {
// if no signer name was given, use default api key
if (!res.locals.signer) {
return this.pgConnection.setDBAuth(user, res.locals, 'default', function (err) {
req.profiler.done('setDBAuth');
if (err) {
return callback(err);
}

38
lib/models/cdb-request.js
View File

@@ -1,26 +1,24 @@
'use strict';
function CdbRequest () {
this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host ||
'^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
);
}
module.exports = class CdbRequest {
constructor () {
// would extract "strk" from "strk.cartodb.com"
this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host || '^([^\\.]+)\\.');
}
module.exports = CdbRequest;
userByReq (req) {
const host = req.headers.host || '';
CdbRequest.prototype.userByReq = function (req) {
var host = req.headers.host || '';
if (req.params.user) {
return req.params.user;
if (req.params.user) {
return req.params.user;
}
const mat = host.match(this.RE_USER_FROM_HOST);
if (!mat || mat.length !== 2) {
throw new Error(`No username found in hostname '${host}'`);
}
return mat[1];
}
var mat = host.match(this.RE_USER_FROM_HOST);
if (!mat) {
global.logger.error("Pattern '%s' does not match hostname '%s'", this.RE_USER_FROM_HOST, host);
return;
}
if (mat.length !== 2) {
global.logger.error("Pattern '%s' gave unexpected matches against '%s': %s", this.RE_USER_FROM_HOST, host, mat);
return;
}
return mat[1];
};

17
lib/models/dataview/base.js
View File

@@ -23,7 +23,7 @@ function getPGTypeName (pgType) {
module.exports = class BaseDataview {
getResult (psql, override, callback) {
this.sql(psql, override, (err, query, flags = null) => {
this.sql(psql, override, (err, query) => {
if (err) {
return callback(err);
}
@@ -36,20 +36,7 @@ module.exports = class BaseDataview {
result = this.format(result, override);
result.type = this.getType();
// Overviews logging
const stats = {};
if (flags && flags.usesOverviews !== undefined) {
stats.usesOverviews = flags.usesOverviews;
} else {
stats.usesOverviews = false;
}
if (this.getType) {
stats.dataviewType = this.getType();
}
return callback(null, result, stats);
return callback(null, result);
}, true); // use read-only transaction
});
}

2
lib/models/dataview/overviews/aggregation.js
View File

@@ -213,7 +213,7 @@ Aggregation.prototype.sql = function (psql, override, callback) {
debug(aggregationSql);
return callback(null, aggregationSql, { usesOverviews: true });
return callback(null, aggregationSql);
};
var aggregationFnQueryTpl = {

2
lib/models/dataview/overviews/formula.js
View File

@@ -76,5 +76,5 @@ Formula.prototype.sql = function (psql, override, callback) {
debug(formulaSql);
return callback(null, formulaSql, { usesOverviews: true });
return callback(null, formulaSql);
};

2
lib/models/dataview/overviews/histogram.js
View File

@@ -179,7 +179,7 @@ Histogram.prototype.sql = function (psql, override, callback) {
var histogramSql = this._buildQuery(override);
return callback(null, histogramSql, { usesOverviews: true });
return callback(null, histogramSql);
};
Histogram.prototype._buildQuery = function (override) {

24
lib/models/mapconfig/adapter/mapconfig-overviews-adapter.js
View File

@@ -25,58 +25,50 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function (user, requestMapCon
layers.forEach(layer => augmentLayersQueue.defer(this._augmentLayer.bind(this), user, layer, analysesResults));
augmentLayersQueue.awaitAll(function layersAugmentQueueFinish (err, results) {
augmentLayersQueue.awaitAll(function layersAugmentQueueFinish (err, layers) {
if (err) {
return callback(err);
}
const layers = results.map(result => result.layer);
const overviewsAddedToMapconfig = results.some(result => result.overviewsAddedToMapconfig);
if (!layers || layers.length === 0) {
return callback(new Error('Missing layers array from layergroup config'));
}
requestMapConfig.layers = layers;
const stats = { overviewsAddedToMapconfig };
return callback(null, requestMapConfig, stats);
return callback(null, requestMapConfig);
});
};
MapConfigOverviewsAdapter.prototype._augmentLayer = function (user, layer, analysesResults, callback) {
let overviewsAddedToMapconfig = false;
if (layer.type !== 'mapnik' && layer.type !== 'cartodb') {
return callback(null, { layer, overviewsAddedToMapconfig });
return callback(null, layer);
}
this.overviewsMetadataBackend.getOverviewsMetadata(user, layer.options.sql, (err, metadata) => {
if (err) {
return callback(err, { layer, overviewsAddedToMapconfig });
return callback(err);
}
if (_.isEmpty(metadata)) {
return callback(null, { layer, overviewsAddedToMapconfig });
return callback(null, layer);
}
var filters = getFilters(analysesResults, layer);
overviewsAddedToMapconfig = true;
if (!filters) {
layer.options = Object.assign({}, layer.options, getQueryRewriteData(layer, analysesResults, {
overviews: metadata
}));
return callback(null, { layer, overviewsAddedToMapconfig });
return callback(null, layer);
}
var unfilteredQuery = getUnfilteredQuery(analysesResults, layer);
this.filterStatsBackend.getFilterStats(user, unfilteredQuery, filters, function (err, stats) {
if (err) {
return callback(null, { layer, overviewsAddedToMapconfig });
return callback(null, layer);
}
layer.options = Object.assign({}, layer.options, getQueryRewriteData(layer, analysesResults, {
@@ -84,7 +76,7 @@ MapConfigOverviewsAdapter.prototype._augmentLayer = function (user, layer, analy
filter_stats: stats
}));
return callback(null, { layer, overviewsAddedToMapconfig });
return callback(null, layer);
});
});
};

1
lib/models/mapconfig/provider/named-map-provider.js
View File

@@ -129,6 +129,7 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
context.analysisConfiguration = {
user: this.user,
logger: this.logger,
db: {
host: rendererParams.dbhost,
port: rendererParams.dbport,

11
lib/server-options.js
View File

@@ -3,6 +3,7 @@
const fqdn = require('@carto/fqdn-sync');
var _ = require('underscore');
var OverviewsQueryRewriter = require('./utils/overviews-query-rewriter');
const Logger = require('./utils/logger');
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
@@ -52,9 +53,6 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
hostHeaderTemplate: '{{=it.username}}.localhost.lan'
},
logger: {
filename: undefined
},
limits: {}
});
@@ -118,9 +116,6 @@ module.exports = {
endpoint: analysisConfig.batch.endpoint,
hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate
},
logger: {
filename: analysisConfig.logger.filename
},
limits: analysisConfig.limits
},
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
@@ -133,7 +128,7 @@ module.exports = {
varnish_purge_enabled: global.environment.varnish.purge_enabled,
fastly: global.environment.fastly || {},
cache_enabled: global.environment.cache_enabled,
log_format: global.environment.log_format,
useProfiler: global.environment.useProfiler,
pubSubMetrics: Object.assign({ enabled: false }, global.environment.pubSubMetrics)
pubSubMetrics: Object.assign({ enabled: false }, global.environment.pubSubMetrics),
logger: new Logger()
};

4
lib/stats/profiler-proxy.js
View File

@@ -52,4 +52,8 @@ ProfilerProxy.prototype.toJSONString = function () {
return this.profile ? this.profiler.toJSONString() : '{}';
};
ProfilerProxy.prototype.toJSON = function () {
return this.profile ? JSON.parse(this.profiler.toJSONString()) : {};
};
module.exports = ProfilerProxy;

127
lib/utils/common-headers.js Normal file
View File

@@ -0,0 +1,127 @@
'use strict';
module.exports = function setCommonHeaders (req, res, callback) {
const { logger } = res.locals;
res.set('X-Request-Id', logger.bindings().id);
// TODO: x-layergroupid header??
const user = getUser({ res });
if (user) {
res.set('Carto-User', user);
}
const userId = getUserId({ res });
if (userId) {
res.set('Carto-User-Id', `${userId}`);
}
const mapId = getMapId({ res });
if (mapId) {
res.set('Carto-Map-Id', mapId);
}
const cacheBuster = getCacheBuster({ res });
if (cacheBuster) {
res.set('Carto-Cache-Buster', cacheBuster);
}
const templateHash = getTemplateHash({ res });
if (templateHash) {
res.set('Carto-Template-Hash', templateHash);
}
getStatTag({ res }, (err, statTag) => {
if (err) {
err.message = `Error generating Stat Tag header: ${err.message}`;
logger.warn({ error: err });
}
if (statTag) {
res.set('Carto-Stat-Tag', statTag);
}
callback();
});
};
function getUser ({ res }) {
if (res.locals.user) {
return res.locals.user;
}
}
function getUserId ({ res }) {
if (res.locals.userId) {
return res.locals.userId;
}
}
function getMapId ({ res }) {
if (res.locals.token) {
return res.locals.token;
}
if (res.locals.mapConfig) {
return res.locals.mapConfig.id();
}
if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) {
return res.locals.mapConfigProvider.mapConfig.id();
}
}
function getCacheBuster ({ res }) {
if (res.locals.cache_buster !== undefined) {
return `${res.locals.cache_buster}`;
}
if (res.locals.mapConfigProvider) {
return `${res.locals.mapConfigProvider.getCacheBuster()}`;
}
}
function getTemplateHash ({ res }) {
const { logger } = res.locals;
if (res.locals.templateHash) {
return res.locals.templateHash;
}
if (res.locals.mapConfigProvider && typeof res.locals.mapConfigProvider.getTemplateHash === 'function') {
let templateHash;
try {
templateHash = res.locals.mapConfigProvider.getTemplateHash().substring(0, 8);
} catch (err) {
err.message = `Error generating Stat Tag header: ${err.message}`;
logger.warn({ error: err });
}
return templateHash;
}
}
function getStatTag ({ res }, callback) {
if (res.locals.mapConfig) {
return callback(null, res.locals.mapConfig.obj().stat_tag);
}
if (!res.locals.mapConfigProvider) {
return callback();
}
res.locals.mapConfigProvider.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
return callback(null, mapConfig.obj().stat_tag);
});
}

60
lib/utils/logger.js Normal file
View File

@@ -0,0 +1,60 @@
'use strict';
const pino = require('pino');
const { req: requestSerializer, res: responseSerializer, err, wrapErrorSerializer } = pino.stdSerializers;
const DEV_ENVS = ['test', 'development'];
module.exports = class Logger {
constructor () {
const { LOG_LEVEL, NODE_ENV } = process.env;
const logLevelFromNodeEnv = NODE_ENV === 'test' ? 'fatal' : 'info';
const errorSerializer = DEV_ENVS.includes(NODE_ENV) ? err : wrapErrorSerializer(err => {
err.stack = err.stack.split('\n').slice(0, 3).join('\n');
return err;
});
const options = {
base: null, // Do not bind hostname, pid and friends by default
level: LOG_LEVEL || logLevelFromNodeEnv,
serializers: {
request: requestSerializer,
response: responseSerializer,
error: (error) => Array.isArray(error) ? error.map((err) => errorSerializer(err)) : [errorSerializer(error)]
}
};
const dest = pino.destination({ sync: false }); // stdout
this._logger = pino(options, dest);
}
trace (...args) {
this._logger.trace(...args);
}
debug (...args) {
this._logger.debug(...args);
}
info (...args) {
this._logger.info(...args);
}
warn (...args) {
this._logger.warn(...args);
}
error (...args) {
this._logger.error(...args);
}
fatal (...args) {
this._logger.fatal(...args);
}
child (...args) {
return this._logger.child(...args);
}
finish (callback) {
return pino.final(this._logger, callback);
}
};

11
metro/index.js Normal file
View File

@@ -0,0 +1,11 @@
'use strict';
const split = require('split2');
const logCollector = require('./log-collector');
const metricsCollector = require('./metrics-collector');
process.stdin
.pipe(split())
.pipe(logCollector())
.pipe(metricsCollector())
.pipe(process.stdout);

78
metro/log-collector.js Normal file
View File

@@ -0,0 +1,78 @@
'use strict'
const split = require('split2');
const assingDeep = require('assign-deep');
const { Transform } = require('stream');
const DEV_ENVS = ['test', 'development'];
const logs = new Map();
const LEVELS = {
10: 'trace',
20: 'debug',
30: 'info',
40: 'warn',
50: 'error',
60: 'fatal'
}
module.exports = function logCollector () {
return new Transform({
transform (chunk, enc, callback) {
let entry;
try {
entry = JSON.parse(chunk);
const { level, time } = entry;
if (level === undefined && time === undefined) {
throw new Error('Entry log is not valid');
}
} catch (e) {
if (DEV_ENVS.includes(process.env.NODE_ENV)) {
this.push(chunk + '\n');
}
return callback();
}
const { id, end } = entry;
if (id === undefined) {
entry.level = LEVELS[entry.level];
this.push(`${JSON.stringify(entry)}\n`);
return callback();
}
if (end === true) {
const accEntry = logs.get(id);
accEntry.level = LEVELS[accEntry.level];
accEntry.time = entry.time;
this.push(`${JSON.stringify(accEntry)}\n`);
logs.delete(id);
return callback();
}
if (logs.has(id)) {
const accEntry = logs.get(id);
if (accEntry.level > entry.level) {
delete entry.level
}
let error;
if (hasProperty(accEntry, 'error') && hasProperty(entry, 'error')) {
logs.set(id, assingDeep({}, accEntry, entry, { error: accEntry.error.concat(entry.error) }));
} else {
logs.set(id, assingDeep({}, accEntry, entry));
}
} else {
logs.set(id, entry);
}
callback();
}
})
}
function hasProperty(obj, prop) {
return Object.prototype.hasOwnProperty.call(obj, prop)
}

121
metro/metrics-collector.js Normal file
View File

@@ -0,0 +1,121 @@
'use strict'
const http = require('http');
const { Counter, Histogram, register } = require('prom-client');
const split = require('split2');
const { Transform } = require('stream');
const DEV_ENVS = ['test', 'development'];
const requestCounter = new Counter({
name: 'maps_api_requests_total',
help: 'MAPS API requests total'
});
const requestOkCounter = new Counter({
name: 'maps_api_requests_ok_total',
help: 'MAPS API requests ok total'
});
const requestErrorCounter = new Counter({
name: 'maps_api_requests_errors_total',
help: 'MAPS API requests errors total'
});
const responseTimeHistogram = new Histogram({
name: 'maps_api_response_time_total',
help: 'MAPS API response time total'
});
const userRequestCounter = new Counter({
name: 'maps_api_requests',
help: 'MAPS API requests per user',
labelNames: ['user', 'http_code']
});
const userRequestOkCounter = new Counter({
name: 'maps_api_requests_ok',
help: 'MAPS API requests per user with success HTTP code',
labelNames: ['user', 'http_code']
});
const userRequestErrorCounter = new Counter({
name: 'maps_api_requests_errors',
help: 'MAPS API requests per user with error HTTP code',
labelNames: ['user', 'http_code']
});
const userResponseTimeHistogram = new Histogram({
name: 'maps_api_response_time',
help: 'MAPS API response time total',
labelNames: ['user']
});
module.exports = function metricsCollector () {
return new Transform({
transform (chunk, enc, callback) {
let entry;
try {
entry = JSON.parse(chunk);
const { level, time } = entry;
if (level === undefined && time === undefined) {
throw new Error('Entry log is not valid');
}
} catch (e) {
if (DEV_ENVS.includes(process.env.NODE_ENV)) {
this.push(chunk);
}
return callback();
}
const { request, response, stats } = entry;
if (request === undefined || response === undefined || stats === undefined) {
this.push(chunk);
return callback();
}
const { statusCode, headers } = response;
const { 'carto-user': user } = headers;
requestCounter.inc();
if (statusCode !== undefined && user !== undefined) {
userRequestCounter.labels(user, `${statusCode}`).inc();
}
if (statusCode >= 200 && statusCode < 400) {
requestOkCounter.inc();
if (user !== undefined) {
userRequestOkCounter.labels(user, `${statusCode}`).inc();
}
} else if (statusCode >= 400) {
requestErrorCounter.inc();
if (user !== undefined) {
userRequestErrorCounter.labels(user, `${statusCode}`).inc();
}
}
const { response: responseTime } = stats;
if (Number.isFinite(responseTime)) {
responseTimeHistogram.observe(responseTime);
userResponseTimeHistogram.labels(user).observe(responseTime);
}
this.push(chunk);
callback();
}
})
}
const port = process.env.PORT || 9145;
http
.createServer((req, res) => {
res.writeHead(200, { 'Content-Type': register.contentType });
res.end(register.metrics());
})
.listen(port)
.unref();

435
package-lock.json generated
View File

@@ -272,6 +272,12 @@
"protobufjs": "^6.8.6"
}
},
"@hapi/bourne": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-2.0.0.tgz",
"integrity": "sha512-WEezM1FWztfbzqIUbsDzFRVMxSoLy3HugVcux6KDDtTqzPsLE8NDRHfXvev66aH1i2oOKKar3/XDjbvh/OUBdg==",
"dev": true
},
"@mapbox/sphericalmercator": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@mapbox/sphericalmercator/-/sphericalmercator-1.1.0.tgz",
@@ -533,6 +539,61 @@
"sprintf-js": "~1.0.2"
}
},
"args": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/args/-/args-5.0.1.tgz",
"integrity": "sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==",
"dev": true,
"requires": {
"camelcase": "5.0.0",
"chalk": "2.4.2",
"leven": "2.1.0",
"mri": "1.1.4"
},
"dependencies": {
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dev": true,
"requires": {
"color-convert": "^1.9.0"
}
},
"camelcase": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz",
"integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==",
"dev": true
},
"chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"dev": true,
"requires": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
}
},
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
"dev": true
},
"supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"dev": true,
"requires": {
"has-flag": "^3.0.0"
}
}
}
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -572,6 +633,19 @@
"integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
"dev": true
},
"assign-deep": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/assign-deep/-/assign-deep-1.0.1.tgz",
"integrity": "sha512-CSXAX79mibneEYfqLT5FEmkqR5WXF+xDRjgQQuVf6wSCXCYU8/vHttPidNar7wJ5BFmKAo8Wei0rCtzb+M/yeA==",
"requires": {
"assign-symbols": "^2.0.2"
}
},
"assign-symbols": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-2.0.2.tgz",
"integrity": "sha512-9sBQUQZMKFKcO/C3Bo6Rx4CQany0R0UeVcefNGRRdW2vbmaMOhV1sbmlXcQLcD56juLXbSGTBm0GGuvmrAF8pA=="
},
"astral-regex": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz",
@@ -593,6 +667,11 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"atomic-sleep": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@@ -640,6 +719,11 @@
"integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
"dev": true
},
"bintrees": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz",
"integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ="
},
"body-parser": {
"version": "1.18.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz",
@@ -701,17 +785,6 @@
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
"integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg="
},
"bunyan": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz",
"integrity": "sha1-aMakpQLVYgvJ9y1nNoEMGxiYCX8=",
"requires": {
"dtrace-provider": "~0.6",
"moment": "^2.10.6",
"mv": "~2",
"safe-json-stringify": "~1"
}
},
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
@@ -741,16 +814,16 @@
"integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo="
},
"camshaft": {
"version": "0.65.3",
"resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.65.3.tgz",
"integrity": "sha512-URr3gWV1QtUATYXQzCTTc7Z1sjlc4ZDTdm+qsOhrOll5ieU1BXkO8puIe8Ts7pwMHH2cLgEy1Ddh+a/V8QMt7A==",
"version": "0.66.0",
"resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.66.0.tgz",
"integrity": "sha512-25WFuUEiPD9Tw0XWSno/m7qJ7QqaEy9t/WbcXt+j+zMTOxvQ4RqpvCAhmUEbLgTLRyTCTvY8nqPtu4PaVJuIyw==",
"requires": {
"async": "^1.5.2",
"bunyan": "1.8.1",
"cartodb-psql": "0.14.0",
"cartodb-query-tables": "^0.6.1",
"debug": "^3.1.0",
"dot": "^1.0.3",
"pino": "^6.3.1",
"request": "^2.85.0"
},
"dependencies": {
@@ -1102,6 +1175,12 @@
"assert-plus": "^1.0.0"
}
},
"dateformat": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
"integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==",
"dev": true
},
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
@@ -1236,15 +1315,6 @@
"integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=",
"dev": true
},
"dtrace-provider": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz",
"integrity": "sha1-CweNVReTfYcxAUUtkUZzdVe3XlE=",
"optional": true,
"requires": {
"nan": "^2.0.8"
}
},
"duplexify": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@@ -2022,6 +2092,16 @@
"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
"dev": true
},
"fast-redact": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-2.0.0.tgz",
"integrity": "sha512-zxpkULI9W9MNTK2sJ3BpPQrTEXFNESd2X6O1tXMFpK/XM0G5c5Rll2EVYZH2TqI3xRGK/VaJ+eEOt7pnENJpeA=="
},
"fast-safe-stringify": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz",
"integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA=="
},
"fast-text-encoding": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz",
@@ -2216,6 +2296,11 @@
}
}
},
"flatstr": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz",
"integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw=="
},
"flatted": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
@@ -3406,11 +3491,6 @@
"resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz",
"integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI="
},
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -3556,6 +3636,18 @@
"html-escaper": "^2.0.0"
}
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=",
"dev": true
},
"joycon": {
"version": "2.2.5",
"resolved": "https://registry.npmjs.org/joycon/-/joycon-2.2.5.tgz",
"integrity": "sha512-YqvUxoOcVPnCp0VU1/56f+iKSdvIRJYPznH22BdXV3xMk75SFXhWeJkZ8C9XxUWt1b5x2X1SxuFygW1U0FmkEQ==",
"dev": true
},
"js-base64": {
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.5.1.tgz",
@@ -3665,6 +3757,12 @@
"invert-kv": "^1.0.0"
}
},
"leven": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz",
"integrity": "sha1-wuep93IJTe6dNCAq6KzORoeHVYA=",
"dev": true
},
"levn": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
@@ -3787,33 +3885,6 @@
}
}
},
"log4js": {
"version": "github:cartodb/log4js-node#145d5f91e35e7fb14a6278cbf7a711ced6603727",
"from": "github:cartodb/log4js-node#cdb",
"requires": {
"async": "~0.2.0",
"readable-stream": "~1.0.2",
"semver": "~4.3.3",
"underscore": "1.8.2"
},
"dependencies": {
"async": {
"version": "0.2.10",
"resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
"integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E="
},
"semver": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
"integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto="
},
"underscore": {
"version": "1.8.2",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.2.tgz",
"integrity": "sha1-ZN8utZCJnelQeC83NRkLpC6/MR0="
}
}
},
"long": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
@@ -4261,7 +4332,14 @@
"moment": {
"version": "2.22.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz",
"integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ=="
"integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ==",
"dev": true
},
"mri": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.1.4.tgz",
"integrity": "sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==",
"dev": true
},
"ms": {
"version": "2.0.0",
@@ -4274,17 +4352,6 @@
"integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=",
"dev": true
},
"mv": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
"integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
"optional": true,
"requires": {
"mkdirp": "~0.5.1",
"ncp": "~2.0.0",
"rimraf": "~2.4.0"
}
},
"nan": {
"version": "2.14.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
@@ -4296,12 +4363,6 @@
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
"dev": true
},
"ncp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
"integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
"optional": true
},
"needle": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/needle/-/needle-2.3.3.tgz",
@@ -4674,6 +4735,12 @@
"ansi-regex": "^4.1.0"
}
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"dev": true
},
"which-module": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
@@ -4872,11 +4939,6 @@
"ee-first": "1.1.1"
}
},
"on-headers": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz",
"integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -5154,6 +5216,101 @@
"pinkie": "^2.0.0"
}
},
"pino": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/pino/-/pino-6.3.1.tgz",
"integrity": "sha512-RgT010a5FfnxJ2AwB0TqcEuM+gNsnd08PZnCob98JSTLldLF0GMFJ/Z1VE/rdl5yJCqcoLwftmZSwSFY4/Hc2g==",
"requires": {
"fast-redact": "^2.0.0",
"fast-safe-stringify": "^2.0.7",
"flatstr": "^1.0.12",
"pino-std-serializers": "^2.4.2",
"quick-format-unescaped": "^4.0.1",
"sonic-boom": "^1.0.0"
}
},
"pino-pretty": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-4.0.0.tgz",
"integrity": "sha512-YLy/n3dMXYWOodSm530gelkSAJGmEp29L9pqiycInlIae5FEJPWAkMRO3JFMbIFtjD2Ve4SH2aBcz2aRreGpBQ==",
"dev": true,
"requires": {
"@hapi/bourne": "^2.0.0",
"args": "^5.0.1",
"chalk": "^3.0.0",
"dateformat": "^3.0.3",
"fast-safe-stringify": "^2.0.7",
"jmespath": "^0.15.0",
"joycon": "^2.2.5",
"pump": "^3.0.0",
"readable-stream": "^3.6.0",
"split2": "^3.1.1",
"strip-json-comments": "^3.0.1"
},
"dependencies": {
"ansi-styles": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
"integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
"dev": true,
"requires": {
"@types/color-name": "^1.1.1",
"color-convert": "^2.0.1"
}
},
"chalk": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
"integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
"dev": true,
"requires": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true
},
"strip-json-comments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.0.tgz",
"integrity": "sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==",
"dev": true
},
"supports-color": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz",
"integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==",
"dev": true,
"requires": {
"has-flag": "^4.0.0"
}
}
}
},
"pino-std-serializers": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.4.2.tgz",
"integrity": "sha512-WaL504dO8eGs+vrK+j4BuQQq6GLKeCCcHaMB2ItygzVURcL1CycwNEUHTD/lHFHs/NL5qAz2UKrjYWXKSf4aMQ=="
},
"pkg-dir": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz",
@@ -5258,6 +5415,14 @@
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"dev": true
},
"prom-client": {
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/prom-client/-/prom-client-12.0.0.tgz",
"integrity": "sha512-JbzzHnw0VDwCvoqf8y1WDtq4wSBAbthMB1pcVI/0lzdqHGJI3KBJDXle70XK+c7Iv93Gihqo0a5LlOn+g8+DrQ==",
"requires": {
"tdigest": "^0.1.1"
}
},
"propagate": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/propagate/-/propagate-1.0.0.tgz",
@@ -5305,6 +5470,16 @@
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
},
"pump": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
@@ -5320,6 +5495,11 @@
"resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.1.0.tgz",
"integrity": "sha1-1fiOMv0B/uT22UbSHRer0WrGXzU="
},
"quick-format-unescaped": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz",
"integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A=="
},
"range-parser": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
@@ -5374,14 +5554,13 @@
}
},
"readable-stream": {
"version": "1.0.34",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
},
"readdirp": {
@@ -5495,6 +5674,13 @@
"tough-cookie": "~2.3.3",
"tunnel-agent": "^0.6.0",
"uuid": "^3.1.0"
},
"dependencies": {
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
}
}
},
"require-directory": {
@@ -5555,30 +5741,6 @@
}
}
},
"rimraf": {
"version": "2.4.5",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
"integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
"optional": true,
"requires": {
"glob": "^6.0.1"
},
"dependencies": {
"glob": {
"version": "6.0.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
"integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
"optional": true,
"requires": {
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "2 || 3",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
}
}
},
"run-async": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz",
@@ -5602,12 +5764,6 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
},
"safe-json-stringify": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz",
"integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==",
"optional": true
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
@@ -5747,6 +5903,15 @@
}
}
},
"sonic-boom": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.0.1.tgz",
"integrity": "sha512-o9tx+bonVEXSaPtptyXQXpP8l6UV9Bi3im2geZskvWw2a/o/hrbWI7EBbbv+rOx6Hubnzun9GgH4WfbgEA3MFQ==",
"requires": {
"atomic-sleep": "^1.0.0",
"flatstr": "^1.0.12"
}
},
"source-map": {
"version": "0.5.7",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@@ -5827,6 +5992,14 @@
"through": "2"
}
},
"split2": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/split2/-/split2-3.1.1.tgz",
"integrity": "sha512-emNzr1s7ruq4N+1993yht631/JH+jaj0NYBosuKmLcq+JkGQ9MmTw1RB1fGaTCzUuseRIClrlSLHRNYGwWQ58Q==",
"requires": {
"readable-stream": "^3.0.0"
}
},
"sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@@ -6100,9 +6273,19 @@
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"requires": {
"safe-buffer": "~5.2.0"
},
"dependencies": {
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
}
}
},
"strip-ansi": {
"version": "3.0.1",
@@ -6229,6 +6412,14 @@
}
}
},
"tdigest": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz",
"integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=",
"requires": {
"bintrees": "1.0.1"
}
},
"test-exclude": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz",
@@ -6581,9 +6772,9 @@
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.1.0.tgz",
"integrity": "sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg=="
},
"v8-compile-cache": {
"version": "2.1.0",

10
package.json
View File

@@ -36,9 +36,10 @@
"dependencies": {
"@carto/fqdn-sync": "0.2.2",
"@google-cloud/pubsub": "1.5.0",
"assign-deep": "^1.0.1",
"basic-auth": "2.0.0",
"body-parser": "1.18.3",
"camshaft": "^0.65.3",
"camshaft": "^0.66.0",
"cartodb-psql": "0.14.0",
"cartodb-query-tables": "^0.7.0",
"cartodb-redis": "^3.0.0",
@@ -48,18 +49,20 @@
"fastly-purge": "1.0.1",
"gc-stats": "^1.4.0",
"glob": "7.1.2",
"log4js": "github:cartodb/log4js-node#cdb",
"lru-cache": "4.1.3",
"lzma": "2.3.2",
"node-statsd": "0.1.1",
"on-headers": "1.0.1",
"pino": "^6.3.1",
"prom-client": "^12.0.0",
"queue-async": "1.1.0",
"redis-mpool": "^0.8.0",
"request": "2.87.0",
"semver": "5.5.0",
"split2": "^3.1.1",
"step-profiler": "0.3.0",
"turbo-carto": "0.21.2",
"underscore": "1.6.0",
"uuid": "^8.1.0",
"windshaft": "^7.0.1",
"yargs": "^15.3.1"
},
@@ -75,6 +78,7 @@
"moment": "2.22.1",
"nock": "9.2.6",
"nyc": "^14.1.1",
"pino-pretty": "^4.0.0",
"redis": "2.8.0",
"step": "1.0.0",
"strftime": "0.10.0"

4
test/acceptance/analysis/named-maps-test.js
View File

@@ -4,7 +4,7 @@ var assert = require('../../support/assert');
var helper = require('../../support/test-helper');
var CartodbWindshaft = require('../../../lib/server');
const createServer = require('../../../lib/server');
var serverOptions = require('../../../lib/server-options');
var TestClient = require('../../support/test-client');
@@ -14,7 +14,7 @@ describe('named-maps analysis', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var IMAGE_TOLERANCE_PER_MIL = 20;

4
test/acceptance/auth/authorization-basic-use-cases-test.js
View File

@@ -2,7 +2,7 @@
const assert = require('../../support/assert');
const testHelper = require('../../support/test-helper');
const CartodbWindshaft = require('../../../lib/server');
const createServer = require('../../../lib/server');
const serverOptions = require('../../../lib/server-options');
var LayergroupToken = require('../../../lib/models/layergroup-token');
@@ -47,7 +47,7 @@ describe('Basic authorization use cases', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
beforeEach(function () {

4
test/acceptance/cache/cache-headers-test.js vendored
View File

@@ -5,7 +5,7 @@ var testHelper = require('../../support/test-helper');
var assert = require('../../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../../lib/server');
const createServer = require('../../../lib/server');
var serverOptions = require('../../../lib/server-options');
var LayergroupToken = require('../../../lib/models/layergroup-token');
@@ -14,7 +14,7 @@ describe('get requests with cache headers', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
server.setMaxListeners(0);
});

4
test/acceptance/cache/surrogate-keys-invalidation-test.js vendored
View File

@@ -7,7 +7,7 @@ var step = require('step');
var FastlyPurge = require('fastly-purge');
var _ = require('underscore');
var NamedMapsCacheEntry = require('../../../lib/cache/model/named-maps-entry');
var CartodbWindshaft = require('../../../lib/server');
const createServer = require('../../../lib/server');
var nock = require('nock');
describe('templates surrogate keys', function () {
@@ -33,7 +33,7 @@ describe('templates surrogate keys', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
nock.disableNetConnect();
nock.enableNetConnect(/(127.0.0.1|cartocdn.com)/);
});

24
test/acceptance/dataviews/overviews-test.js
View File

@@ -55,7 +55,6 @@ describe('dataviews using tables without overviews', function () {
return done(err);
}
assert.deepStrictEqual(formulaResult, { operation: 'count', result: 7313, nulls: 0, type: 'formula' });
assert(getUsesOverviewsFromHeaders(headers) === false); // Overviews logging
testClient.drain(done);
});
@@ -269,8 +268,6 @@ describe('dataviews using tables with overviews', function () {
nulls: 0,
type: 'formula'
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging
testClient.drain(done);
});
@@ -290,8 +287,6 @@ describe('dataviews using tables with overviews', function () {
infinities: 0,
nans: 0
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging
testClient.drain(done);
});
@@ -311,8 +306,6 @@ describe('dataviews using tables with overviews', function () {
infinities: 0,
nans: 0
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging
testClient.drain(done);
});
@@ -387,8 +380,6 @@ describe('dataviews using tables with overviews', function () {
assert.ok(histogram);
assert.strictEqual(histogram.type, 'histogram');
assert.ok(Array.isArray(histogram.bins));
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
assert(getDataviewTypeFromHeaders(headers) === 'histogram'); // Overviews logging
testClient.drain(done);
});
@@ -480,7 +471,7 @@ describe('dataviews using tables with overviews', function () {
nans: 0,
type: 'formula'
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
testClient.drain(done);
});
});
@@ -499,7 +490,6 @@ describe('dataviews using tables with overviews', function () {
nans: 0,
type: 'formula'
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
testClient.drain(done);
});
@@ -519,7 +509,6 @@ describe('dataviews using tables with overviews', function () {
nulls: 0,
type: 'formula'
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
testClient.drain(done);
});
@@ -611,9 +600,6 @@ describe('dataviews using tables with overviews', function () {
type: 'aggregation'
});
assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
assert(getDataviewTypeFromHeaders(headers) === 'aggregation'); // Overviews logging
testClient.drain(done);
});
});
@@ -830,11 +816,3 @@ describe('dataviews using tables with overviews', function () {
});
});
});
function getUsesOverviewsFromHeaders (headers) {
return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).usesOverviews;
}
function getDataviewTypeFromHeaders (headers) {
return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).dataviewType;
}

4
test/acceptance/dynamic-styling-named-maps-test.js
View File

@@ -4,14 +4,14 @@ var assert = require('../support/assert');
var step = require('step');
var LayergroupToken = require('../../lib/models/layergroup-token');
var testHelper = require('../support/test-helper');
var CartodbWindshaft = require('../../lib/server');
var createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
describe('dynamic styling for named maps', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var keysToDelete;

44
test/acceptance/error-middleware-test.js
View File

@@ -1,44 +0,0 @@
'use strict';
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
describe('error middleware', function () {
it('should returns a errors header', function (done) {
const mapConfig = {
version: '1.6.0',
layers: [{
type: 'mapnik',
options: {}
}]
};
const errorHeader = {
mainError: {
statusCode: 400,
message: 'Missing cartocss for layer 0 options',
name: 'Error',
label: 'ANONYMOUS LAYERGROUP',
type: 'layer'
},
moreErrors: []
};
this.testClient = new TestClient(mapConfig, 1234);
const params = {
response: {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8',
'X-Tiler-Errors': JSON.stringify(errorHeader)
}
}
};
this.testClient.getLayergroup(params, (err) => {
assert.ifError(err);
done();
});
});
});

4
test/acceptance/errors-with-context-test.js
View File

@@ -1,14 +1,14 @@
'use strict';
var assert = require('../support/assert');
var CartodbWindshaft = require('../../lib/server');
var createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
describe('error with context', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var layerOK = {

10
test/acceptance/health-check-test.js
View File

@@ -5,7 +5,7 @@ require('../support/test-helper');
var fs = require('fs');
var assert = require('../support/assert');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
describe('health checks', function () {
@@ -41,7 +41,7 @@ describe('health checks', function () {
};
it('returns 200 and ok=true with enabled configuration', function (done) {
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, healthCheckRequest, RESPONSE_OK, function (res, err) {
assert.ok(!err);
@@ -62,7 +62,7 @@ describe('health checks', function () {
fs.readFile = function (filename, callback) {
callback(null, errorMessage);
};
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, healthCheckRequest, RESPONSE_FAIL, function (res, err) {
fs.readFile = readFileFn;
@@ -82,7 +82,7 @@ describe('health checks', function () {
fs.readFile = function (filename, callback) {
callback(null, '');
};
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, healthCheckRequest, RESPONSE_FAIL, function (res, err) {
fs.readFile = readFileFn;
@@ -100,7 +100,7 @@ describe('health checks', function () {
it('not err if disabled file does not exist', function (done) {
global.environment.disabled_file = '/tmp/ftreftrgtrccre';
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, healthCheckRequest, RESPONSE_OK, function (res, err) {
assert.ok(!err);

4
test/acceptance/multilayer-server-test.js
View File

@@ -10,14 +10,14 @@ var LayergroupToken = require('../../lib/models/layergroup-token');
var PgQueryRunner = require('../../lib/backends/pg-query-runner');
var QueryTables = require('cartodb-query-tables').queryTables;
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
describe('tests from old api translated to multilayer', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
server.setMaxListeners(0);
});

10
test/acceptance/multilayer-test.js
View File

@@ -19,7 +19,7 @@ var windshaftFixtures = path.join(__dirname, '/../../node_modules/windshaft/test
var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20;
var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25;
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var QueryTables = require('cartodb-query-tables').queryTables;
@@ -30,7 +30,7 @@ var QueryTables = require('cartodb-query-tables').queryTables;
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
server.setMaxListeners(0);
});
@@ -851,7 +851,7 @@ var QueryTables = require('cartodb-query-tables').queryTables;
function doRestartServer (err/*, res */) {
assert.ifError(err);
// hack simulating restart...
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
return null;
},
function doGet1 (err) {
@@ -1274,7 +1274,7 @@ var QueryTables = require('cartodb-query-tables').queryTables;
it('cache control for layergroup default value', function (done) {
global.environment.varnish.layergroupTtl = null;
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation,
function (res) {
@@ -1291,7 +1291,7 @@ var QueryTables = require('cartodb-query-tables').queryTables;
var layergroupTtl = 300;
global.environment.varnish.layergroupTtl = layergroupTtl;
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation,
function (res) {

4
test/acceptance/named-layers-test.js
View File

@@ -3,7 +3,7 @@
var testHelper = require('../support/test-helper');
var assert = require('../support/assert');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var LayergroupToken = require('../../lib/models/layergroup-token');
@@ -17,7 +17,7 @@ describe('named_layers', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
// configure redis pool instance to use in tests

4
test/acceptance/named-layers-visibility-test.js
View File

@@ -4,7 +4,7 @@ var step = require('step');
var testHelper = require('../support/test-helper');
var assert = require('../support/assert');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var RedisPool = require('redis-mpool');
@@ -17,7 +17,7 @@ describe('layers visibility for previews', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
// configure redis pool instance to use in tests

4
test/acceptance/named-maps-authentication-test.js
View File

@@ -6,7 +6,7 @@ var querystring = require('querystring');
var assert = require('../support/assert');
const mapnik = require('@carto/mapnik');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var TemplateMaps = require('../../lib/backends/template-maps');
var NamedMapsCacheEntry = require('../../lib/cache/model/named-maps-entry');
@@ -15,7 +15,7 @@ describe('named maps authentication', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
// configure redis pool instance to use in tests

4
test/acceptance/named-maps-cache-test.js
View File

@@ -5,14 +5,14 @@ require('../support/test-helper');
const helper = require('../support/test-helper');
var assert = require('../support/assert');
const mapnik = require('@carto/mapnik');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
describe('named maps provider cache', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var username = 'localhost';

6
test/acceptance/named-maps-static-view-test.js
View File

@@ -6,7 +6,7 @@ var RedisPool = require('redis-mpool');
var assert = require('../support/assert');
const mapnik = require('@carto/mapnik');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var TemplateMaps = require('../../lib/backends/template-maps');
@@ -86,7 +86,7 @@ describe('named maps static view', function () {
};
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
testHelper.deleteRedisKeys({ 'user:localhost:mapviews:global': 5 }, function () {
@@ -323,7 +323,7 @@ describe('named maps static view', function () {
};
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
assert.ifError(err);

4
test/acceptance/named-maps-stats-test.js
View File

@@ -6,7 +6,7 @@ var querystring = require('querystring');
var assert = require('../support/assert');
const mapnik = require('@carto/mapnik');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var TemplateMaps = require('../../lib/backends/template-maps');
var NamedMapsCacheEntry = require('../../lib/cache/model/named-maps-entry');
@@ -15,7 +15,7 @@ describe('named maps preview stats', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var redisPool = new RedisPool(global.environment.redis);

126
test/acceptance/overviews-metadata-named-maps-test.js
View File

@@ -3,7 +3,7 @@
var testHelper = require('../support/test-helper');
var assert = require('../support/assert');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var LayergroupToken = require('../../lib/models/layergroup-token');
@@ -18,7 +18,7 @@ describe('overviews metadata for named maps', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
// configure redis pool instance to use in tests
@@ -172,126 +172,4 @@ describe('overviews metadata for named maps', function () {
}
);
});
describe('Overviews Flags', function () {
it('Overviews used', function (done) {
step(
function postTemplate () {
var next = this;
assert.response(server, {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(template)
}, {}, function (res, err) {
next(err, res);
});
},
function instantiateTemplate (err) {
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/api/v1/map/named/' + templateId,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
}, {},
function (res, err) {
return next(err, res);
});
},
function checkFlags (err, res) {
assert.ifError(err);
var next = this;
var parsedBody = JSON.parse(res.body);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
const headers = JSON.parse(res.headers['x-tiler-profiler']);
assert.ok(headers.overviewsAddedToMapconfig);
assert.strictEqual(headers.mapType, 'named');
next();
},
function finish (err) {
done(err);
}
);
});
it('Overviews NOT used', function (done) {
const nonOverviewsTemplateId = 'non-overviews-template';
var nonOverviewsTemplate = {
version: '0.0.1',
name: nonOverviewsTemplateId,
auth: { method: 'open' },
layergroup: {
version: '1.0.0',
layers: [nonOverviewsLayer]
}
};
step(
function postTemplate () {
var next = this;
assert.response(server, {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(nonOverviewsTemplate)
}, {}, function (res, err) {
next(err, res);
});
},
function instantiateTemplate (err) {
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/api/v1/map/named/' + nonOverviewsTemplateId,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
}, {},
function (res, err) {
return next(err, res);
});
},
function checkFlags (err, res) {
assert.ifError(err);
var next = this;
var parsedBody = JSON.parse(res.body);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
const headers = JSON.parse(res.headers['x-tiler-profiler']);
assert.strictEqual(headers.overviewsAddedToMapconfig, false);
assert.strictEqual(headers.mapType, 'named');
next();
},
function finish (err) {
done(err);
}
);
});
});
});

83
test/acceptance/overviews-metadata-test.js
View File

@@ -3,7 +3,7 @@
var testHelper = require('../support/test-helper');
var assert = require('../support/assert');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var LayergroupToken = require('../../lib/models/layergroup-token');
@@ -18,7 +18,7 @@ describe('overviews metadata', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
// configure redis pool instance to use in tests
@@ -110,90 +110,13 @@ describe('overviews metadata', function () {
}
);
});
describe('Overviews Flags', function () {
it('Overviews used', function (done) {
var layergroup = {
version: '1.0.0',
layers: [overviewsLayer, nonOverviewsLayer]
};
var layergroupUrl = '/api/v1/map';
var expectedToken;
step(
function doPost () {
var next = this;
assert.response(server, {
url: layergroupUrl,
method: 'POST',
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function (res) {
assert.strictEqual(res.statusCode, 200, res.body);
const headers = JSON.parse(res.headers['x-tiler-profiler']);
assert.ok(headers.overviewsAddedToMapconfig);
assert.strictEqual(headers.mapType, 'anonymous');
const parsedBody = JSON.parse(res.body);
expectedToken = parsedBody.layergroupid;
next();
});
},
function finish (err) {
keysToDelete['map_cfg|' + LayergroupToken.parse(expectedToken).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done(err);
}
);
});
it('Overviews NOT used', function (done) {
var layergroup = {
version: '1.0.0',
layers: [nonOverviewsLayer]
};
var layergroupUrl = '/api/v1/map';
var expectedToken;
step(
function doPost () {
var next = this;
assert.response(server, {
url: layergroupUrl,
method: 'POST',
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function (res) {
assert.strictEqual(res.statusCode, 200, res.body);
const headers = JSON.parse(res.headers['x-tiler-profiler']);
assert.strictEqual(headers.overviewsAddedToMapconfig, false);
assert.strictEqual(headers.mapType, 'anonymous');
const parsedBody = JSON.parse(res.body);
expectedToken = parsedBody.layergroupid;
next();
});
},
function finish (err) {
keysToDelete['map_cfg|' + LayergroupToken.parse(expectedToken).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done(err);
}
);
});
});
});
describe('overviews metadata with filters', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
// configure redis pool instance to use in tests

4
test/acceptance/regressions-test.js
View File

@@ -5,7 +5,7 @@ var assert = require('../support/assert');
const helper = require('../support/test-helper');
var TestClient = require('../support/test-client');
const LayergroupToken = require('../../lib/models/layergroup-token');
const CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
const serverOptions = require('../../lib/server-options');
describe('regressions', function () {
@@ -44,7 +44,7 @@ describe('regressions', function () {
// See: https://github.com/CartoDB/Windshaft-cartodb/pull/956
it('"/user/localhost/api/v1/map" should create an anonymous map', function (done) {
const server = new CartodbWindshaft(serverOptions);
const server = createServer(serverOptions);
const layergroup = {
version: '1.7.0',
layers: [

6
test/acceptance/server-test.js
View File

@@ -6,14 +6,14 @@ var assert = require('../support/assert');
var querystring = require('querystring');
var step = require('step');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
describe('server', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
server.setMaxListeners(0);
});
@@ -45,7 +45,7 @@ describe('server old_api', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
server.setMaxListeners(0);
});

4
test/acceptance/templates-test.js
View File

@@ -21,7 +21,7 @@ var http = require('http');
var helper = require('../support/test-helper');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
var LayergroupToken = require('../../lib/models/layergroup-token');
@@ -30,7 +30,7 @@ describe('template_api', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
server.setMaxListeners(0);
// FIXME: we need a better way to reset cache while running tests
server.layergroupAffectedTablesCache.cache.reset();

4
test/acceptance/turbo-carto/named-maps-test.js
View File

@@ -4,7 +4,7 @@ var assert = require('../../support/assert');
var step = require('step');
var LayergroupToken = require('../../../lib/models/layergroup-token');
var testHelper = require('../../support/test-helper');
var CartodbWindshaft = require('../../../lib/server');
const createServer = require('../../../lib/server');
var serverOptions = require('../../../lib/server-options');
const mapnik = require('@carto/mapnik');
var IMAGE_TOLERANCE_PER_MIL = 10;
@@ -13,7 +13,7 @@ describe('turbo-carto for named maps', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var keysToDelete;

4
test/acceptance/widgets/named-maps-test.js
View File

@@ -8,7 +8,7 @@ var queue = require('queue-async');
var helper = require('../../support/test-helper');
var CartodbWindshaft = require('../../../lib/server');
const createServer = require('../../../lib/server');
var serverOptions = require('../../../lib/server-options');
var LayergroupToken = require('../../../lib/models/layergroup-token');
@@ -17,7 +17,7 @@ describe('named-maps widgets', function () {
var server;
before(function () {
server = new CartodbWindshaft(serverOptions);
server = createServer(serverOptions);
});
var username = 'localhost';

8
test/support/test-client.js
View File

@@ -12,7 +12,7 @@ var LayergroupToken = require('../../lib/models/layergroup-token');
var assert = require('./assert');
var helper = require('./test-helper');
var CartodbWindshaft = require('../../lib/server');
const createServer = require('../../lib/server');
var serverOptions = require('../../lib/server-options');
serverOptions.analysis.batch.inlineExecution = true;
@@ -30,7 +30,7 @@ function TestClient (config, apiKey, extraHeaders = {}, overrideServerOptions =
this.extraHeaders = extraHeaders;
this.keysToDelete = {};
this.serverOptions = Object.assign({}, serverOptions, overrideServerOptions);
this.server = new CartodbWindshaft(this.serverOptions);
this.server = createServer(this.serverOptions);
}
module.exports = TestClient;
@@ -1348,7 +1348,7 @@ TestClient.prototype.drain = function (callback) {
module.exports.getStaticMap = function getStaticMap (templateName, params, callback) {
var self = this;
self.server = new CartodbWindshaft(serverOptions);
self.server = createServer(serverOptions);
if (!callback) {
callback = params;
@@ -1378,7 +1378,7 @@ module.exports.getStaticMap = function getStaticMap (templateName, params, callb
};
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
var server = createServer(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
helper.deleteRedisKeys({ 'user:localhost:mapviews:global': 5 }, function () {

12
test/support/test-helper.js
View File

@@ -1,12 +1,5 @@
'use strict';
/**
* User: simon
* Date: 30/08/2011
* Time: 13:52
* Desc: Loads test specific variables
*/
var assert = require('assert');
var fs = require('fs');
var LZMA = require('lzma').LZMA;
@@ -14,7 +7,6 @@ var LZMA = require('lzma').LZMA;
var lzmaWorker = new LZMA();
var redis = require('redis');
var log4js = require('log4js');
const setICUEnvVariable = require('../../lib/utils/icu-data-env-setter');
// set environment specific variables
@@ -24,10 +16,6 @@ process.env.NODE_ENV = 'test';
setICUEnvVariable();
// don't output logs in test environment to reduce noise
log4js.configure({ appenders: [] });
global.logger = log4js.getLogger();
// Utility function to compress & encode LZMA
function lzmaCompressToBase64 (payload, mode, callback) {
lzmaWorker.compress(payload, mode,

20
test/unit/cdb-request-test.js
View File

@@ -5,7 +5,7 @@ var assert = require('assert');
var CdbRequest = require('../../lib/models/cdb-request');
describe('req2params', function () {
describe('username in host header (CdbRequest)', function () {
function createRequest (host, userParam) {
var req = {
params: {},
@@ -47,39 +47,33 @@ describe('req2params', function () {
assert.strictEqual(user, 'development');
});
it('returns undefined when it cannot extract username', function () {
it('returns throw when it cannot extract username', function () {
var userFromHostConfig = global.environment.user_from_host;
global.environment.user_from_host = null;
var cdbRequest = new CdbRequest();
var user = cdbRequest.userByReq(createRequest('localhost'));
assert.throws(() => cdbRequest.userByReq(createRequest('localhost')));
global.environment.user_from_host = userFromHostConfig;
assert.strictEqual(user, undefined);
});
it('should not fail for undefined host header', function () {
it('should throw for undefined host header', function () {
var userFromHostConfig = global.environment.user_from_host;
global.environment.user_from_host = null;
var cdbRequest = new CdbRequest();
var user = cdbRequest.userByReq(createRequest(undefined));
assert.throws(() => cdbRequest.userByReq(createRequest(undefined)));
global.environment.user_from_host = userFromHostConfig;
assert.strictEqual(user, undefined);
});
it('should not fail for null host header', function () {
it('should throw for null host header', function () {
var userFromHostConfig = global.environment.user_from_host;
global.environment.user_from_host = null;
var cdbRequest = new CdbRequest();
var user = cdbRequest.userByReq(createRequest(null));
assert.throws(() => cdbRequest.userByReq(createRequest(null)));
global.environment.user_from_host = userFromHostConfig;
assert.strictEqual(user, undefined);
});
});

167
test/unit/error-middleware-test.js
View File

@@ -20,171 +20,4 @@ describe('error-middleware', function () {
'Error status code for multiline/PSQL does not match'
);
});
it('should return a header with errors', function (done) {
const error = new Error('error test');
error.label = 'test label';
error.type = 'test type';
error.subtype = 'test subtype';
const errors = [error, error];
const req = {};
const res = {
headers: {},
set (key, value) {
this.headers[key] = value;
},
statusCode: 0,
status (status) {
this.statusCode = status;
},
json () {},
send () {}
};
const errorHeader = {
mainError: {
statusCode: 400,
message: error.message,
name: error.name,
label: error.label,
type: error.type,
subtype: error.subtype
},
moreErrors: [{
message: error.message,
name: error.name,
label: error.label,
type: error.type,
subtype: error.subtype
}]
};
const errorFn = errorMiddleware();
errorFn(errors, req, res, (err) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(res.headers, {
'X-Tiler-Errors': JSON.stringify(errorHeader)
});
return done();
});
});
it('JSONP should return a header with error status code', function (done) {
const error = new Error('error test');
error.label = 'test label';
error.type = 'test type';
error.subtype = 'test subtype';
const errors = [error, error];
const req = {
query: { callback: true }
};
const res = {
headers: {},
set (key, value) {
this.headers[key] = value;
},
statusCode: 0,
status (status) {
this.statusCode = status;
},
jsonp () {},
send () {}
};
const errorHeader = {
mainError: {
statusCode: 400,
message: error.message,
name: error.name,
label: error.label,
type: error.type,
subtype: error.subtype
},
moreErrors: [{
message: error.message,
name: error.name,
label: error.label,
type: error.type,
subtype: error.subtype
}]
};
const errorFn = errorMiddleware();
errorFn(errors, req, res, (err) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(res.headers, {
'X-Tiler-Errors': JSON.stringify(errorHeader)
});
return done();
});
});
it('should escape chars that broke logs regex', function (done) {
const badString = 'error: ( ) = " \" \' * $ & |'; // eslint-disable-line no-useless-escape
const escapedString = 'error ';
const error = new Error(badString);
error.label = badString;
error.type = badString;
error.subtype = badString;
const errors = [error, error];
const req = {};
const res = {
headers: {},
set (key, value) {
this.headers[key] = value;
},
statusCode: 0,
status (status) {
this.statusCode = status;
},
json () {},
send () {}
};
const errorHeader = {
mainError: {
statusCode: 400,
message: escapedString,
name: error.name,
label: escapedString,
type: escapedString,
subtype: escapedString
},
moreErrors: [{
message: escapedString,
name: error.name,
label: escapedString,
type: escapedString,
subtype: escapedString
}]
};
const errorFn = errorMiddleware();
errorFn(errors, req, res, (err) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(res.headers, {
'X-Tiler-Errors': JSON.stringify(errorHeader)
});
return done();
});
});
});