undefined might come as a string (#1196)

so a hack to check and cast it to a real undefined

Co-authored-by: Álvaro Manera <amanera@cartodb.com>

.github/workflows/main.yml

@@ -0,0 +1,58 @@
+name: continuous integration
+on: 
+  pull_request:
+    paths-ignore:
+      - 'LICENSE'
+      - 'README**'
+      - 'HOW_TO_RELEASE**'
+      - 'LOGGING**'
+    
+env:
+  GCLOUD_VERSION: '306.0.0'
+  ARTIFACTS_PROJECT_ID: cartodb-on-gcp-main-artifacts
+
+jobs:
+  build-test-docker:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 10
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+        token: ${{ secrets.CARTOFANTE_PERSONAL_TOKEN }}
+
+    - name: Build image
+      # we tag with "latest" but we don't push it on purpose. We use it as a base for the testing image
+      run: |
+          echo ${GITHUB_SHA::7}
+          echo ${GITHUB_REF##*/}
+          docker build -f private/Dockerfile -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/} -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}--${GITHUB_SHA::7} .
+          
+    - name: Build testing image
+      # here it uses the lastest from prev step to add the needed parts on top
+      run: |
+          docker build -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft-test:latest -f private/Dockerfile.test .
+
+    - name: Setup gcloud authentication
+      uses: google-github-actions/setup-gcloud@master
+      with:
+        version: ${{env.GCLOUD_VERSION}}
+        service_account_key: ${{ secrets.ARTIFACTS_GCLOUD_ACCOUNT_BASE64 }}
+
+    - name: Configure docker and pull images
+    # we pull images manually, as if done in next step using docker-compose it fails because missing openssl
+      run: |
+         gcloud auth configure-docker
+         docker pull gcr.io/cartodb-on-gcp-main-artifacts/postgres:latest
+         docker pull gcr.io/cartodb-on-gcp-main-artifacts/redis:latest
+    
+    - name: Run tests inside container
+      run: docker-compose -f private/ci/docker-compose.yml run windshaft-tests
+
+    - name: Upload image
+      run: |
+          docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}
+          docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7}
+          docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}--${GITHUB_SHA::7}
+    

.github/workflows/master.yml

@@ -0,0 +1,47 @@
+# in this workflow we don't run the tests. Only build image, tag (also latests) and upload. The tests are not run because they are run
+# on each pull request, and there is a branch protection that forces to have branch up to date before merging, so tests are always run 
+# with the latest code
+
+name: master build image
+on:
+  push:
+    branches:
+      - master
+
+env:
+  GCLOUD_VERSION: '306.0.0'
+  ARTIFACTS_PROJECT_ID: cartodb-on-gcp-main-artifacts
+
+jobs:
+  build-master:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 5
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+        token: ${{ secrets.CARTOFANTE_PERSONAL_TOKEN }}
+
+    - name: Build image
+      run: |
+          echo ${GITHUB_SHA::7}
+          echo ${GITHUB_REF##*/}
+          docker build -f private/Dockerfile -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/} -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7} .
+          
+    - name: Setup gcloud authentication
+      uses: google-github-actions/setup-gcloud@master
+      with:
+        version: ${{env.GCLOUD_VERSION}}
+        service_account_key: ${{ secrets.ARTIFACTS_GCLOUD_ACCOUNT_BASE64 }}
+
+    - name: Configure docker
+      run: |
+          gcloud auth configure-docker
+        
+    - name: Upload image
+      run: |
+          docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}
+          docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7}
+          docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:latest
+    

.gitignore

@@ -12,3 +12,5 @@ redis.pid
 coverage/
 .DS_Store
 .nyc_output
+build_resources/
+.dockerignore

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "private"]
+	path = private
+	url = git@github.com:CartoDB/Windshaft-cartodb-private.git
+	branch = master

.travis.yml

@@ -1,12 +0,0 @@
-language: generic
-sudo: required
-env:
-  matrix:
-    - NODE_VERSION=10.15.1
-      DOCKER_IMAGE=carto/nodejs-xenial-pg101:latest
-    - NODE_VERSION=10.15.1
-      DOCKER_IMAGE=carto/nodejs-xenial-pg1121:latest
-services:
-  - docker
-before_install: docker pull ${DOCKER_IMAGE}
-script: npm run test:docker

LOGGING.md

@@ -0,0 +1,21 @@
+# Logging structured traces
+
+In order to have meaningful and useful log traces, you should follow
+some general guidelines described in the [Project Guidelines](http://doc-internal.cartodb.net/platform/guidelines.html#structured-logging).
+
+In this project there is a specific logger in place that takes care of
+format and context of the traces for you. Take a look at [logger.js](https://github.com/CartoDB/Windshaft-cartodb/blob/cf82e1954e2244861e47fce0c2223ee466a5cd64/lib/utils/logger.js)
+(NOTE: that file will be moved soon to a common module).
+
+The logger is instantiated as part of the [app startup process](https://github.com/CartoDB/Windshaft-cartodb/blob/cf82e1954e2244861e47fce0c2223ee466a5cd64/app.js#L53),
+then passed to middlewares and other client classes.
+
+There are many examples of how to use the logger to generate traces
+throughout the code. Here are a few of them:
+
+```js
+lib/api/middlewares/logger.js:        res.locals.logger.info({ client_request: req }, 'Incoming request');
+lib/api/middlewares/logger.js:        res.on('finish', () => res.locals.logger.info({ server_response: res, status: res.statusCode }, 'Response sent'));
+lib/api/middlewares/profiler.js:            logger.info({ stats, duration: stats.response / 1000, duration_ms: stats.response }, 'Request profiling stats');
+lib/api/middlewares/tag.js:        res.on('finish', () => logger.info({ tags: res.locals.tags }, 'Request tagged'));
+```

NEWS.md

@@ -1,5 +1,58 @@
 # Changelog
 
+## 10.0.0
+Released 2020-mm-dd
+
+Breaking changes:
+- Log system revamp:
+  - Logs to stdout, disabled while testing
+  - Upgrade `camshaft` to version [`0.67.2`](https://github.com/CartoDB/camshaft/releases/tag/0.67.2)
+  - Use header `X-Request-Id`, or create a new `uuid` when no present, to identyfy log entries
+  - Be able to set log level from env variable `LOG_LEVEL`, useful while testing: `LOG_LEVEL=info npm test`; even more human-readable: `LOG_LEVEL=info npm t | ./node_modules/.bin/pino-pretty`
+  - Stop responding with `X-Tiler-Errors` header. Now errors are properly logged and will end up in ELK as usual.
+  - Stop responding with `X-Tiler-Profiler` header. Now profiling stats are properly logged and will end up in ELK as usual.
+  - Be able to reduce the footprint in the final log file depending on the environment
+  - Be able to pass the logger to the analysis creation (camshaft) while instantiating a named map with analysis.
+  - Be able to tag requests with labels as an easier way to provide business metrics
+  - Metro: Add log-collector utility (`metro`), it will be moved to its own repository. Attaching it here fro development purposes. Try it with the following command `LOG_LEVEL=info npm t | node metro`
+  - Metro: Creates `metrics-collector.js` a stream to update Prometheus' counters and histograms and exposes them via Express' app (`:9145/metrics`). Use the ones defined in `grok_exporter`
+
+Bug Fixes:
+- While instantiating a map, set the `cache buster` equal to `0` when there are no affected tables in the MapConfig. Thus `layergroupid` has the same structure always:
+  - `${map_id}:${cache_buster}` for anonymous map
+  - `${user}@${template_hash}@${map_id}:${cache_buster}` for named map
+
+## 9.0.0
+Released 2020-06-05
+
+Breaking changes:
+- Remove `/version` endpoint
+- Drop support for Node.js < 12
+
+Announcements:
+- Support Node.js 12
+- Upgrade `windshaft` to version [`7.0.1`](https://github.com/CartoDB/Windshaft/releases/tag/7.0.1)
+- Upgrade `camshaft` to version [`0.65.3`](https://github.com/CartoDB/camshaft/blob/0.65.3/CHANGELOG.md#0653):
+  - Fix noisy message logs while checking analyses' limits
+  - Fix CI setup, explicit use of PGPORT while creating the PostgreSQL cluster
+- Upgrade `cartodb-redis` to version [`3.0.0`](https://github.com/CartoDB/node-cartodb-redis/releases/tag/3.0.0)
+- Fix test where `http-fallback-image` renderer was failing quietly
+- Fix stat `named map providers` cache count
+- Use new signature for `onTileErrorStrategy`. Required by `windshaft@6.0.0`
+- Extract `onTileErrorStrategy` to a module
+- In tests, stop using mapnik module exposed by windshaft and require it from development dependencies
+- Stop using `MapStore` from `windshaft` while testing and create a custom one instead
+- Rename NamedMapProviderReporter by NamedMapProviderCacheReporter
+- Remove `bootstrapFonts` at process startup (now done in `windshaft@6.0.0`)
+- Stop checking the installed version of some dependencies while testing
+- Send metrics about `map views` (#1162)
+- Add custom headers in responses to allow to other components to be able to get insights about user activity
+- Update dependencies to avoid security vulnerabilities
+
+Bug Fixes:
+- Parsing date column in numeric histograms (#1160)
+- Use `Array.prototype.sort()`'s callback properly while testing. It should return a number not a boolean.
+
 ## 8.1.1
 Released 2020-02-17
 

README.md

@@ -13,8 +13,8 @@ The [`CARTO Maps API`](https://carto.com/developers/maps-api/) tiler. It extends
 
 Requirements:
 
-* [`Node 10.x (npm 6.x)`](https://nodejs.org/dist/latest-v10.x/)
-* [`PostgreSQL >= 10.0`](https://www.postgresql.org/download/)
+* [`Node 12.x `](https://nodejs.org/dist/latest-v10.x/)
+* [`PostgreSQL >= 11.0`](https://www.postgresql.org/download/)
 * [`PostGIS >= 2.4`](https://postgis.net/install/)
 * [`CARTO Postgres Extension >= 0.24.1`](https://github.com/CartoDB/cartodb-postgresql)
 * [`Redis >= 4`](https://redis.io/download)
@@ -45,7 +45,11 @@ $ npm install
 
 ### Run
 
-Create the `./config/environments/<env>.js` file (there are `.example` files to start from). Look at `./lib/server-options.js` for more on config.
+You can inject the configuration through environment variables at run time. Check the file `./config/environments/config.js` to see the ones you have available.
+
+While the migration to the new environment based configuration, you can still use the old method of copying a config file. To enabled the one with environment variables you need to pass `CARTO_WINDSHAFT_ENV_BASED_CONF=true`. You can use the docker image to run it.
+
+Old way:
 
 ```shell
 $ node app.js <env>
@@ -55,10 +59,26 @@ Where `<env>` is the name of a configuration file under `./config/environments/`
 
 ### Test
 
+You can easily run the tests against the dependencies from the `dev-env`. To do so, you need to build the test docker image:
+
 ```shell
-$ npm test
+$ docker-compose build
 ```
 
+Then you can run the tests like:
+
+```shell
+$ docker-compose run windshaft-tests
+```
+
+It will mount your code inside a volume. In case you want to play and run `npm test` or something else you can do:
+
+```shell
+$ docker-compose run --entrypoint bash windshaft-tests
+```
+
+So you will have a bash shell inside the test container, with the code from your host.
+
 ### Coverage
 
 ```shell

app.js

@@ -1,29 +1,17 @@
 'use strict';
 
-var http = require('http');
-var https = require('https');
-var path = require('path');
-var fs = require('fs');
-var _ = require('underscore');
-var semver = require('semver');
+const http = require('http');
+const https = require('https');
+const path = require('path');
+const semver = require('semver');
+
+// TODO: research it it's still needed
 const setICUEnvVariable = require('./lib/utils/icu-data-env-setter');
 
-// jshint undef:false
-var log = console.log.bind(console);
-var logError = console.error.bind(console);
-// jshint undef:true
-
-var nodejsVersion = process.versions.node;
-const { engines } = require('./package.json');
-if (!semver.satisfies(nodejsVersion, engines.node)) {
-    logError(`Node version ${nodejsVersion} is not supported, please use Node.js ${engines.node}.`);
-    process.exit(1);
-}
-
 // This function should be called before the require('yargs').
 setICUEnvVariable();
 
-var argv = require('yargs')
+const argv = require('yargs')
     .usage('Usage: node $0 <environment> [options]')
     .help('h')
     .example(
@@ -36,96 +24,70 @@ var argv = require('yargs')
     .describe('c', 'Load configuration from path')
     .argv;
 
-var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
-var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
-if (!fs.existsSync(configurationFile)) {
-    logError('Configuration file "%s" does not exist', configurationFile);
-    process.exit(1);
+const environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
+let configFileName = environmentArg;
+if (process.env.CARTO_WINDSHAFT_ENV_BASED_CONF) {
+    // we override the file with the one with env vars
+    configFileName = 'config';
 }
+const configurationFile = path.resolve(argv.config || `./config/environments/${configFileName}.js`);
 
 global.environment = require(configurationFile);
-var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.environment.environment;
-process.env.NODE_ENV = ENVIRONMENT;
+process.env.NODE_ENV = argv._[0] || process.env.NODE_ENV || global.environment.environment;
 
-var availableEnvironments = {
-    production: true,
-    staging: true,
-    development: true
-};
-
-// sanity check
-if (!availableEnvironments[ENVIRONMENT]) {
-    logError('node app.js [environment]');
-    logError('environments: %s', Object.keys(availableEnvironments).join(', '));
-    process.exit(1);
-}
-
-process.env.NODE_ENV = ENVIRONMENT;
 if (global.environment.uv_threadpool_size) {
     process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
 }
 
 // set global HTTP and HTTPS agent default configurations
 // ref https://nodejs.org/api/http.html#http_new_agent_options
-var agentOptions = _.defaults(global.environment.httpAgent || {}, {
+const agentOptions = Object.assign({
     keepAlive: false,
     keepAliveMsecs: 1000,
     maxSockets: Infinity,
     maxFreeSockets: 256
-});
+}, global.environment.httpAgent || {});
+
 http.globalAgent = new http.Agent(agentOptions);
 https.globalAgent = new https.Agent(agentOptions);
 
-global.log4js = require('log4js');
-var log4jsConfig = {
-    appenders: [],
-    replaceConsole: true
-};
-
-if (global.environment.log_filename) {
-    var logFilename = path.resolve(global.environment.log_filename);
-    var logDirectory = path.dirname(logFilename);
-    if (!fs.existsSync(logDirectory)) {
-        logError('Log filename directory does not exist: ' + logDirectory);
-        process.exit(1);
-    }
-    log('Logs will be written to ' + logFilename);
-    log4jsConfig.appenders.push(
-        { type: 'file', absolute: true, filename: logFilename }
-    );
-} else {
-    log4jsConfig.appenders.push(
-        { type: 'console', layout: { type: 'basic' } }
-    );
-}
-
-global.log4js.configure(log4jsConfig);
-global.logger = global.log4js.getLogger();
-
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartodbWindshaft = require('./lib/server');
-var serverOptions = require('./lib/server-options');
+const createServer = require('./lib/server');
+const serverOptions = require('./lib/server-options');
+const { logger } = serverOptions;
 
-var server = cartodbWindshaft(serverOptions);
+const availableEnvironments = {
+    production: true,
+    staging: true,
+    development: true
+};
 
-// Maximum number of connections for one process
-// 128 is a good number if you have up to 1024 filedescriptors
-// 4 is good if you have max 32 filedescriptors
-// 1 is good if you have max 16 filedescriptors
-var backlog = global.environment.maxConnections || 128;
+if (!availableEnvironments[process.env.NODE_ENV]) {
+    logger.fatal(new Error(`Invalid environment ${process.env.NODE_ENV} argument, valid ones: ${Object.keys(availableEnvironments).join(', ')}`));
+    process.exit(1);
+}
 
-var listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, backlog);
+const { engines } = require('./package.json');
+if (!semver.satisfies(process.versions.node, engines.node)) {
+    logger.fatal(new Error(`Node version ${process.versions.node} is not supported, please use Node.js ${engines.node}.`));
+    process.exit(1);
+}
 
-var version = require('./package').version;
+const server = createServer(serverOptions);
+
+// Specify the maximum length of the queue of pending connections for the HTTP server.
+// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
+// The default value of this parameter is 511 (not 512).
+// See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
+const backlog = global.environment.maxConnections || 128;
+
+const listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, backlog);
+const { version, name } = require('./package');
 
 listener.on('listening', function () {
-    log('Using Node.js %s', process.version);
-    log('Using configuration file "%s"', configurationFile);
-    log(
-        'Windshaft tileserver %s started on %s:%s PID=%d (%s)',
-        version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
-    );
+    const { address, port } = listener.address();
+    logger.info({ 'Node.js': process.version, pid: process.pid, environment: process.env.NODE_ENV, [name]: version, address, port, config: configurationFile }, `${name} initialized successfully`);
 });
 
 function getCPUUsage (oldUsage) {
@@ -160,22 +122,14 @@ setInterval(function cpuUsageMetrics () {
     });
 
     previousCPUUsage = CPUUsage;
-}, 5000);
+}, 5000).unref();
 
 setInterval(function () {
     var memoryUsage = process.memoryUsage();
     Object.keys(memoryUsage).forEach(function (k) {
         global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
     });
-}, 5000);
-
-process.on('SIGHUP', function () {
-    global.log4js.clearAndShutdownAppenders(function () {
-        global.log4js.configure(log4jsConfig);
-        global.logger = global.log4js.getLogger();
-        log('Log files reloaded');
-    });
-});
+}, 5000).unref();
 
 if (global.gc) {
     var gcInterval = Number.isFinite(global.environment.gc_interval)
@@ -185,7 +139,7 @@ if (global.gc) {
     if (gcInterval > 0) {
         setInterval(function gcForcedCycle () {
             global.gc();
-        }, gcInterval);
+        }, gcInterval).unref();
     }
 }
 
@@ -228,41 +182,36 @@ function getGCTypeValue (type) {
     return value;
 }
 
-addHandlers(listener, global.logger, 45000);
+const exitProcess = logger.finish((err, finalLogger, listener, signal, killTimeout) => {
+    scheduleForcedExit(killTimeout, finalLogger);
 
-function addHandlers (listener, logger, killTimeout) {
-    process.on('uncaughtException', exitProcess(listener, logger, killTimeout));
-    process.on('unhandledRejection', exitProcess(listener, logger, killTimeout));
-    process.on('ENOMEM', exitProcess(listener, logger, killTimeout));
-    process.on('SIGINT', exitProcess(listener, logger, killTimeout));
-    process.on('SIGTERM', exitProcess(listener, logger, killTimeout));
+    finalLogger.info(`Process has received signal: ${signal}`);
+
+    let code = 0;
+
+    if (err) {
+        code = 1;
+        finalLogger.fatal(err);
+    }
+
+    finalLogger.info(`Process is going to exit with code: ${code}`);
+    listener.close(() => process.exit(code));
+});
+
+function addHandlers (listener, killTimeout) {
+    process.on('uncaughtException', (err) => exitProcess(err, listener, 'uncaughtException', killTimeout));
+    process.on('unhandledRejection', (err) => exitProcess(err, listener, 'unhandledRejection', killTimeout));
+    process.on('ENOMEM', (err) => exitProcess(err, listener, 'ENOMEM', killTimeout));
+    process.on('SIGINT', () => exitProcess(null, listener, 'SIGINT', killTimeout));
+    process.on('SIGTERM', () => exitProcess(null, listener, 'SIGTERM', killTimeout));
 }
 
-function exitProcess (listener, logger, killTimeout) {
-    return function exitProcessFn (signal) {
-        scheduleForcedExit(killTimeout, logger);
+addHandlers(listener, 45000);
 
-        let code = 0;
-
-        if (!['SIGINT', 'SIGTERM'].includes(signal)) {
-            const err = signal instanceof Error ? signal : new Error(signal);
-            signal = undefined;
-            code = 1;
-
-            logger.fatal(err);
-        } else {
-            logger.info(`Process has received signal: ${signal}`);
-        }
-
-        logger.info(`Process is going to exit with code: ${code}`);
-        listener.close(() => global.log4js.shutdown(() => process.exit(code)));
-    };
-}
-
-function scheduleForcedExit (killTimeout, logger) {
+function scheduleForcedExit (killTimeout, finalLogger) {
     // Schedule exit if there is still ongoing work to deal with
     const killTimer = setTimeout(() => {
-        logger.info('Process didn\'t close on time. Force exit');
+        finalLogger.info('Process didn\'t close on time. Force exit');
         process.exit(1);
     }, killTimeout);
 

carto-package.json

@@ -2,8 +2,8 @@
     "name": "carto_windshaft",
     "current_version": {
         "requires": {
-            "node": "^10.15.1",
-            "npm": "^6.4.1",
+            "node": "^12.16.3",
+            "npm": "^6.14.4",
             "mapnik": "==3.0.15.16",
             "crankshaft": "~0.8.1"
         },

config/environments/config.js

@@ -0,0 +1,411 @@
+var config = {
+    environment: process.env.CARTO_WINDSHAFT_NODE_ENV,
+    port: 8181,
+    host: null, // null on purpouse so it listens to whatever address docker assigns
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
+    uv_threadpool_size: undefined,
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    gc_interval: 10000,
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    user_from_host: process.env.CARTO_WINDSHAFT_USER_FROM_HOST || '^(.*)\\.cartodb\\.com$',
+
+    // Base URLs for the APIs
+    //
+    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    //
+    // Note: each entry corresponds with an express' router.
+    // You must define at least one path. However, middlewares are optional.
+    routes: {
+        api: [{
+            paths: [
+                '/api/v1',
+                '/user/:user/api/v1'
+            ],
+            // Optional: attach middlewares at the begining of the router
+            // to perform custom operations.
+            middlewares: [
+                function noop () {
+                    return function noopMiddleware (req, res, next) {
+                        next();
+                    };
+                }
+            ],
+            // Base url for the Detached Maps API
+            // "/api/v1/map" is the new API,
+            map: [{
+                paths: [
+                    '/map'
+                ],
+                middlewares: [] // Optional
+            }],
+            // Base url for the Templated Maps API
+            // "/api/v1/map/named" is the new API,
+            template: [{
+                paths: [
+                    '/map/named'
+                ],
+                middlewares: [] // Optional
+            }]
+        }]
+    },
+
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `routes` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    resources_url_templates: {
+        http: process.env.CARTO_WINDSHAFT_RESOURCE_URL_TEMPLATE_HTTP || 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
+        https: process.env.CARTO_WINDSHAFT_RESOURCE_URL_TEMPLATE_HTTPS || 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    },
+    // Specify the maximum length of the queue of pending connections for the HTTP server.
+    // The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
+    // The default value of this parameter is 511 (not 512).
+    // See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
+    maxConnections: 128,
+    // Maximum number of templates per user. Unlimited by default.
+    maxUserTemplates: 1024,
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    mapConfigTTL: 7200,
+    // idle socket timeout, in milliseconds
+    socket_timeout: 600000,
+    enable_cors: true,
+    cache_enabled: true,
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    postgres_auth_user: process.env.CARTO_WINDSHAFT_DB_USER || 'cartodb_user_<%= user_id %>',
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    postgres_auth_pass: '<%= user_password %>',
+    postgres: {
+        user: 'publicuser',
+        password: 'public',
+        host: process.env.CARTO_WINDSHAFT_POSTGRES_HOST || 'localhost',
+        port: process.env.CARTO_WINDSHAFT_POSTGRES_PORT || 5432,
+        pool: {
+            // maximum number of resources to create at any given time
+            size: 16,
+            // max milliseconds a resource can go unused before it should be destroyed
+            idleTimeout: 3000,
+            // frequency to check for idle resources
+            reapInterval: 1000
+        }
+    },
+    mapnik_version: undefined,
+    mapnik_tile_format: 'png8:m=h',
+    statsd: {
+        host: process.env.CARTO_WINDSHAFT_STATSD_HOST || 'localhost',
+        port: 8125,
+        prefix: process.env.CARTO_WINDSHAFT_STATSD_PREFIX || ':host.', // could be hostname, better not containing dots
+        cacheDns: true
+        // support all allowed node-statsd options
+    },
+    renderer: {
+        // Milliseconds since last access before renderer cache item expires
+        cache_ttl: 60000,
+        statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+        mvt: {
+        // If enabled, MVTs will be generated with PostGIS directly
+        // If disabled, MVTs will be generated with Mapnik MVT
+            usePostGIS: true
+        },
+        mapnik: {
+            // The size of the pool of internal mapnik backend
+            // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+            // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+            // Important: check the configuration of uv_threadpool_size to use suitable value
+            poolSize: 8,
+
+            // The maximum number of waiting clients of the pool of internal mapnik backend
+            // This maximum number is per mapnik renderer created in Windshaft's RendererFactory
+            poolMaxWaitingClients: 64,
+
+            // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+            // This will prevent blocking the main thread.
+            useCartocssWorkers: false,
+
+            // Metatile is the number of tiles-per-side that are going
+            // to be rendered at once. If all of them will be requested
+            // we'd have saved time. If only one will be used, we'd have
+            // wasted time.
+            metatile: 2,
+
+            // tilelive-mapnik uses an internal cache to store tiles/grids
+            // generated when using metatile. This options allow to tune
+            // the behaviour for that internal cache.
+            metatileCache: {
+                // Time an object must stay in the cache until is removed
+                ttl: 0,
+                // Whether an object must be removed after the first hit
+                // Usually you want to use `true` here when ttl>0.
+                deleteOnHit: false
+            },
+
+            // Override metatile behaviour depending on the format
+            formatMetatile: {
+                png: 2,
+                'grid.json': 1
+            },
+
+            // Buffer size is the tickness in pixel of a buffer
+            // around the rendered (meta?)tile.
+            //
+            // This is important for labels and other marker that overlap tile boundaries.
+            // Setting to 128 ensures no render artifacts.
+            // 64 may have artifacts but is faster.
+            // Less important if we can turn metatiling on.
+            bufferSize: 64,
+
+            // SQL queries will be wrapped with ST_SnapToGrid
+            // Snapping all points of the  geometry to a regular grid
+            snapToGrid: false,
+
+            // SQL queries will be wrapped with ST_ClipByBox2D
+            // Returning the portion of a geometry falling within a rectangle
+            // It will only work if snapToGrid is enabled
+            clipByBox2d: true,
+
+            postgis: {
+                // Parameters to pass to datasource plugin of mapnik
+                // See http://github.com/mapnik/mapnik/wiki/PostGIS
+                user: 'publicuser',
+                password: 'public',
+                host: process.env.CARTO_WINDSHAFT_POSTGRES_HOST || '127.0.0.1',
+                port: process.env.CARTO_WINDSHAFT_POSTGRES_PORT || 5432,
+                extent: '-20037508.3,-20037508.3,20037508.3,20037508.3',
+                // max number of rows to return when querying data, 0 means no limit
+                row_limit: 65535,
+                /*
+               * Set persist_connection to false if you want
+               * database connections to be closed on renderer
+               * expiration (1 minute after last use).
+               * Setting to true (the default) would never
+               * close any connection for the server's lifetime
+               */
+                persist_connection: false,
+                simplify_geometries: true,
+                use_overviews: true, // use overviews to retrieve raster
+                max_size: 500,
+                twkb_encoding: true
+            },
+
+            limits: {
+                // Time in milliseconds a render request can take before it fails, some notes:
+                //  - 0 means no render limit
+                //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+                render: 0,
+                // As the render request will finish even if timed out, whether it should be placed in the internal
+                // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+                // the same tile will result in an immediate response, however that will use a lot of more application
+                // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+                // internal cache.
+                cacheOnTimeout: true
+            },
+
+            // If enabled Mapnik will reuse the features retrieved from the database
+            // instead of requesting them once per style inside a layer
+            'cache-features': true,
+
+            // Require metrics to the renderer
+            metrics: false,
+
+            // Options for markers attributes, ellipses and images caches
+            markers_symbolizer_caches: {
+                disabled: false
+            }
+        },
+        http: {
+            timeout: 2000, // the timeout in ms for a http tile request
+            proxy: undefined, // the url for a proxy server
+            whitelist: [ // the whitelist of urlTemplates that can be used
+                '.*', // will enable any URL
+                'http://{s}.example.com/{z}/{x}/{y}.png'
+            ],
+            // image to use as placeholder when urlTemplate is not in the whitelist
+            // if provided the http renderer will use it instead of throw an error
+            fallbackImage: {
+                type: 'fs', // 'fs' and 'url' supported
+                src: __dirname + '/../../assets/default-placeholder.png'
+            }
+        },
+        torque: {}
+    },
+    // anything analyses related
+    analysis: {
+        // batch configuration
+        batch: {
+            // Inline execution avoid the use of SQL API as batch endpoint
+            // When set to true it will run all analysis queries in series, with a direct connection to the DB
+            // This might be useful for:
+            //  - testing
+            //  - running an standalone server without any dependency on external services
+            inlineExecution: false,
+            // where the SQL API is running, it will use a custom Host header to specify the username.
+            endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
+            // the template to use for adding the host header in the batch api requests
+            hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
+        }
+    },
+    millstone: {
+        // Needs to be writable by server user
+        cache_basedir: process.env.CARTO_WINDSHAFT_TILE_CACHE || '/home/ubuntu/tile_assets/'
+    },
+    redis: {
+        host: process.env.CARTO_WINDSHAFT_REDIS_HOST || '127.0.0.1',
+        port: process.env.CARTO_WINDSHAFT_REDIS_PORT || 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 2 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 2 to know how many possible connections will be
+        // kept open by the servelsr. The default is 50.
+        max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
+    },
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
+    },
+    varnish: {
+        host: process.env.CARTO_WINDSHAFT_VARNISH_PORT || 'localhost',
+        port: process.env.CARTO_WINDSHAFT_VARNISH_PORT || 6082, // the por for the telnet interface where varnish is listening to
+        http_port: 6081, // the port for the HTTP interface where varnish is listening to
+        purge_enabled: process.env.CARTO_WINDSHAFT_VARNISH_PURGE_ENABLED === 'true' || false, // whether the purge/invalidation mechanism is enabled in varnish or not
+        secret: 'xxx',
+        ttl: 86400,
+        fallbackTtl: 300,
+        layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
+    },
+    // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
+    fastly: {
+        // whether the invalidation is enabled or not
+        enabled: false,
+        // the fastly api key
+        apiKey: 'wadus_api_key',
+        // the service that will get surrogate key invalidation
+        serviceId: 'wadus_service_id'
+    },
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various
+    // steps taken for producing the response.
+    useProfiler: false,
+    serverMetadata: {
+        cdn_url: {
+            http: process.env.CARTO_WINDSHAFT_SERVER_CDN_URL_HTTP === 'undefined' ? undefined : process.env.CARTO_WINDSHAFT_SERVER_CDN_URL_HTTP || 'api.cartocdn.com',
+            https: process.env.CARTO_WINDSHAFT_SERVER_CDN_URL_HTTPS === 'undefined' ? undefined : process.env.CARTO_WINDSHAFT_SERVER_CDN_URL_HTTPS || 'cartocdn.global.ssl.fastly.net'
+        }
+    },
+    // Settings for the health check available at /health
+    health: {
+        enabled: process.env.CARTO_WINDSHAFT_HEALTH_ENABLED === 'true' || false,
+        username: 'localhost',
+        z: 0,
+        x: 0,
+        y: 0
+    },
+    disabled_file: 'pids/disabled',
+
+    // Use this as a feature flags enabling/disabling mechanism
+    enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: false,
+        // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerStats: process.env.CARTO_WINDSHAFT_LAYERSTATS_ENABLED === 'true' || false,
+        // whether it should rate limit endpoints (global configuration)
+        rateLimitsEnabled: false,
+        // whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
+        rateLimitsByEndpoint: {
+            anonymous: false,
+            static: false,
+            static_named: false,
+            dataview: false,
+            dataview_search: false,
+            analysis: false,
+            analysis_catalog: false,
+            tile: false,
+            attributes: false,
+            named_list: false,
+            named_create: false,
+            named_get: false,
+            named: false,
+            named_update: false,
+            named_delete: false,
+            named_tiles: false
+        }
+    },
+    pubSubMetrics: {
+        enabled: process.env.CARTO_WINDSHAFT_METRICS_ENABLED === 'true' || false,
+        project_id: process.env.CARTO_WINDSHAFT_METRICS_PROJECT_ID || 'avid-wavelet-844',
+        credentials: '',
+        topic: process.env.CARTO_WINDSHAFT_METRICS_PROJECT_ID || 'raw-metric-events'
+    }
+};
+
+// override some defaults for tests
+if (process.env.NODE_ENV === 'test') {
+    config.user_from_host = '(.*)';
+    config.postgres_auth_pass = 'test_windshaft_cartodb_user_<%= user_id %>_pass';
+    config.millstone.cache_basedir = '/tmp/tile_assets';
+    config.postgres.user = 'test_windshaft_publicuser';
+    config.resources_url_templates = {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
+    };
+    config.cache_enabled = false;
+    config.postgres_auth_user = 'test_windshaft_cartodb_user_<%= user_id %>';
+    config.renderer.mapnik.postgis.twkb_encoding = false;
+    config.renderer.mapnik['cache-features'] = false;
+    config.renderer.http.whitelist = [ // the whitelist of urlTemplates that can be used
+        '.*', // will enable any URL
+        'http://{s}.example.com/{z}/{x}/{y}.png',
+        // for testing purposes
+        'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png'
+    ];
+    config.analysis.batch.inlineExecution = true;
+    config.redis.idleTimeoutMillis = 1;
+    config.redis.reapIntervalMillis = 1;
+    config.varnish.purge_enabled = false;
+    config.health.enabled = false;
+    config.enabledFeatures.layerStats = true;
+}
+
+module.exports = config;

config/environments/development.js.example

@@ -67,9 +67,10 @@ var config = {
         http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
         https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
     }
-
-    // Maximum number of connections for one process
-    // 128 is a good value with a limit of 1024 open file descriptors
+    // Specify the maximum length of the queue of pending connections for the HTTP server.
+    // The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
+    // The default value of this parameter is 511 (not 512).
+    // See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
     ,maxConnections:128
     // Maximum number of templates per user. Unlimited by default.
     ,maxUserTemplates:1024
@@ -82,12 +83,6 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: undefined
-    ,log_windshaft: true
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -262,12 +257,6 @@ var config = {
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
         },
-        logger: {
-            // If filename is given logs comming from analysis client  will be written
-            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
-            // Log file will be re-opened on receiving the HUP signal
-            filename: undefined
-        },
         // Define max execution time in ms for analyses or tags
         // If analysis or tag are not found in redis this values will be used as default.
         limits: {
@@ -383,6 +372,12 @@ var config = {
             named_tiles: false
         }
     }
+    ,pubSubMetrics: {
+        enabled: false,
+        project_id: '',
+        credentials: '',
+        topic: ''
+    }
 };
 
 module.exports = config;

config/environments/production.js.example

@@ -67,9 +67,10 @@ var config = {
         http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
         https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
     }
-
-    // Maximum number of connections for one process
-    // 128 is a good value with a limit of 1024 open file descriptors
+    // Specify the maximum length of the queue of pending connections for the HTTP server.
+    // The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
+    // The default value of this parameter is 511 (not 512).
+    // See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
     ,maxConnections:128
     // Maximum number of templates per user. Unlimited by default.
     ,maxUserTemplates:1024
@@ -82,12 +83,6 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: 'logs/node-windshaft.log'
-    ,log_windshaft: true
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -262,12 +257,6 @@ var config = {
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
         },
-        logger: {
-            // If filename is given logs comming from analysis client  will be written
-            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
-            // Log file will be re-opened on receiving the HUP signal
-            filename: 'logs/node-windshaft-analysis.log'
-        },
         // Define max execution time in ms for analyses or tags
         // If analysis or tag are not found in redis this values will be used as default.
         limits: {
@@ -383,6 +372,12 @@ var config = {
             named_tiles: false
         }
     }
+    ,pubSubMetrics: {
+        enabled: true,
+        project_id: 'avid-wavelet-844',
+        credentials: '',
+        topic: 'raw-metric-events'
+    }
 };
 
 module.exports = config;

config/environments/staging.js.example

@@ -67,9 +67,9 @@ var config = {
         http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
         https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
     }
-
-    // Maximum number of connections for one process
-    // 128 is a good value with a limit of 1024 open file descriptors
+    // Specify the maximum length of the queue of pending connections for the HTTP server.
+    // The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
+    // The default value of this parameter is 511 (not 512).
     ,maxConnections:128
     // Maximum number of templates per user. Unlimited by default.
     ,maxUserTemplates:1024
@@ -82,12 +82,6 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: 'logs/node-windshaft.log'
-    ,log_windshaft: true
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -262,12 +256,6 @@ var config = {
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
         },
-        logger: {
-            // If filename is given logs comming from analysis client  will be written
-            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
-            // Log file will be re-opened on receiving the HUP signal
-            filename: 'logs/node-windshaft-analysis.log'
-        },
         // Define max execution time in ms for analyses or tags
         // If analysis or tag are not found in redis this values will be used as default.
         limits: {
@@ -383,6 +371,12 @@ var config = {
             named_tiles: false
         }
     }
+    ,pubSubMetrics: {
+        enabled: true,
+        project_id: '',
+        credentials: '',
+        topic: 'raw-metric-events'
+    }
 };
 
 module.exports = config;

config/environments/test.js.example

@@ -67,9 +67,10 @@ var config = {
         http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
         https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
     }
-
-    // Maximum number of connections for one process
-    // 128 is a good value with a limit of 1024 open file descriptors
+    // Specify the maximum length of the queue of pending connections for the HTTP server.
+    // The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
+    // The default value of this parameter is 511 (not 512).
+    // See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
     ,maxConnections:128
     // Maximum number of templates per user. Unlimited by default.
     ,maxUserTemplates:1024
@@ -82,12 +83,6 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: '/tmp/node-windshaft.log'
-    ,log_windshaft: true
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@@ -264,12 +259,6 @@ var config = {
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
         },
-        logger: {
-            // If filename is given logs comming from analysis client  will be written
-            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
-            // Log file will be re-opened on receiving the HUP signal
-            filename: '/tmp/node-windshaft-analysis.log'
-        },
         // Define max execution time in ms for analyses or tags
         // If analysis or tag are not found in redis this values will be used as default.
         limits: {
@@ -385,6 +374,12 @@ var config = {
             named_tiles: false
         }
     }
+    ,pubSubMetrics: {
+        enabled: false,
+        project_id: '',
+        credentials: '',
+        topic: ''
+    }
 };
 
 module.exports = config;

docker/Dockerfile-nodejs-xenial-pg101:latest

@@ -1,92 +0,0 @@
-FROM ubuntu:xenial
-
-# Use UTF8 to avoid encoding problems with pgsql
-ENV LANG C.UTF-8
-ENV NPROCS 1
-ENV JOBS 1
-ENV CXX g++-4.9
-ENV PGUSER postgres
-
-# Add external repos
-RUN set -ex \
-    && apt-get update \
-    && apt-get install -y \
-        curl \
-        software-properties-common \
-        locales \
-    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
-    && add-apt-repository -y ppa:cartodb/postgresql-10 \
-    && add-apt-repository -y ppa:cartodb/gis \
-    && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash \
-    && . ~/.nvm/nvm.sh \
-    && locale-gen en_US.UTF-8 \
-    && update-locale LANG=en_US.UTF-8
-
-RUN set -ex \
-    && apt-get update \
-    && apt-get install -y \
-        g++-4.9 \
-        gcc-4.9 \
-        git \
-        libcairo2-dev \
-        libgdal-dev \
-        libgdal1i \
-        libgdal20 \
-        libgeos-dev \
-        libgif-dev \
-        libjpeg8-dev \
-        libjson-c-dev \
-        libpango1.0-dev \
-        libpixman-1-dev \
-        libproj-dev \
-        libprotobuf-c-dev \
-        libxml2-dev \
-        gdal-bin \
-        make \
-        nodejs \
-        protobuf-c-compiler \
-        pkg-config \
-        wget \
-        zip \
-        postgresql-10 \
-        postgresql-10-plproxy \
-        postgis=2.4.4.6+carto-1 \
-        postgresql-10-postgis-2.4=2.4.4.6+carto-1 \
-        postgresql-10-postgis-2.4-scripts=2.4.4.6+carto-1 \
-        postgresql-10-postgis-scripts=2.4.4.6+carto-1 \
-        postgresql-client-10 \
-        postgresql-client-common \
-        postgresql-common \
-        postgresql-contrib \
-        postgresql-plpython-10 \
-        postgresql-server-dev-10 \
-    && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
-    && tar xvzf redis-4.0.8.tar.gz \
-    && cd redis-4.0.8 \
-    && make \
-    && make install \
-    && cd .. \
-    && rm redis-4.0.8.tar.gz \
-    && rm -R redis-4.0.8 \
-    && apt-get purge -y wget protobuf-c-compiler \
-    && apt-get autoremove -y
-
-# Configure PostgreSQL
-RUN set -ex \
-    && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
-    && echo "local     all       all                     trust" >  /etc/postgresql/10/main/pg_hba.conf \
-    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/10/main/pg_hba.conf \
-    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/10/main/pg_hba.conf \
-    && /etc/init.d/postgresql start \
-    && createdb template_postgis \
-    && createuser publicuser \
-    && psql -c "CREATE EXTENSION postgis" template_postgis \
-    && /etc/init.d/postgresql stop
-
-WORKDIR /srv
-EXPOSE 5858
-
-COPY ./scripts/nodejs-install.sh /src/nodejs-install.sh
-RUN chmod 777 /src/nodejs-install.sh
-CMD /src/nodejs-install.sh
-

docker/Dockerfile-nodejs-xenial-pg1121:latest

@@ -1,85 +0,0 @@
-FROM ubuntu:xenial
-
-# Use UTF8 to avoid encoding problems with pgsql
-ENV LANG C.UTF-8
-ENV NPROCS 1
-ENV JOBS 1
-ENV CXX g++-4.9
-ENV PGUSER postgres
-
-# Add external repos
-RUN set -ex \
-    && apt-get update \
-    && apt-get install -y \
-        curl \
-        software-properties-common \
-        locales \
-    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
-    && add-apt-repository -y ppa:cartodb/postgresql-11 \
-    && add-apt-repository -y ppa:cartodb/redis-next \
-    && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash \
-    && . ~/.nvm/nvm.sh \
-    && locale-gen en_US.UTF-8 \
-    && update-locale LANG=en_US.UTF-8
-
-RUN set -ex \
-    && apt-get update \
-    && apt-get install -y \
-        g++-4.9 \
-        gcc-4.9 \
-        git \
-        libcairo2-dev \
-        libgdal-dev=2.3.2+dfsg-2build2~carto1 \
-        libgdal20=2.3.2+dfsg-2build2~carto1  \
-        libgeos-dev=3.7.1~carto1 \
-        libgif-dev \
-        libjpeg8-dev \
-        libjson-c-dev \
-        libpango1.0-dev \
-        libpixman-1-dev \
-        libproj-dev \
-        libprotobuf-c-dev \
-        libxml2-dev \
-        gdal-bin=2.3.2+dfsg-2build2~carto1 \
-        make \
-        nodejs \
-        protobuf-c-compiler \
-        pkg-config \
-        wget \
-        zip \
-        libopenscenegraph100v5 \
-        libsfcgal1 \
-        liblwgeom-2.5.0=2.5.1.4+carto-1 \
-        postgresql-11 \
-        postgresql-11-plproxy \
-        postgis=2.5.1.4+carto-1 \
-        postgresql-11-postgis-2.5=2.5.1.4+carto-1 \
-        postgresql-11-postgis-2.5-scripts=2.5.1.4+carto-1 \
-        postgresql-client-11 \
-        postgresql-client-common \
-        postgresql-common \
-        postgresql-contrib \
-        postgresql-plpython-11 \
-        postgresql-server-dev-11 \
-        redis=5:4.0.9-1carto1~xenial1 \
-    && apt-get purge -y wget protobuf-c-compiler \
-    && apt-get autoremove -y
-
-# Configure PostgreSQL
-RUN set -ex \
-    && echo "listen_addresses='*'" >> /etc/postgresql/11/main/postgresql.conf \
-    && echo "local     all       all                     trust" >  /etc/postgresql/11/main/pg_hba.conf \
-    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/11/main/pg_hba.conf \
-    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/11/main/pg_hba.conf \
-    && /etc/init.d/postgresql start \
-    && createdb template_postgis \
-    && createuser publicuser \
-    && psql -c "CREATE EXTENSION postgis" template_postgis \
-    && /etc/init.d/postgresql stop
-
-WORKDIR /srv
-EXPOSE 5858
-
-COPY ./scripts/nodejs-install.sh /src/nodejs-install.sh
-RUN chmod 777 /src/nodejs-install.sh
-CMD /src/nodejs-install.sh

docker/reference.md

@@ -1,33 +0,0 @@
-# Testing with Docker
-
-Before running the tests with docker, you'll need Docker installed and the docker image downloaded.
-
-## Install docker
-
-```shell
-$ sudo apt install docker.io && sudo usermod -aG docker $(whoami)
-```
-
-## Download image
-
-```shell
-docker pull carto/IMAGE
-```
-
-## Carto account
-
-* `https://hub.docker.com/r/carto/`
-
-## Update image
-
-* Edit the docker image file
-* Build image:
-  * `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/`
-
-* Upload to docker hub:
-  * Login into docker hub:
-    * `docker login`
-  * Create tag:
-    * `docker tag carto/IMAGE carto/IMAGE`
-  * Upload:
-    * `docker push carto/IMAGE`

docker/scripts/nodejs-install.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-
-if [ -z $NODEJS_VERSION ]; then
-    NODEJS_VERSION="10"
-    NODEJS_VERSION_OPTIONS="--lts"
-fi
-
-nvm install $NODEJS_VERSION $NODEJS_VERSION_OPTIONS
-nvm alias default $NODEJS_VERSION
-nvm use default

docker/scripts/test-setup.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-/etc/init.d/postgresql start
-
-source /src/nodejs-install.sh
-
-# Install cartodb-postgresql extension
-git clone https://github.com/CartoDB/cartodb-postgresql.git
-cd cartodb-postgresql && make && make install && cd ..
-
-cp config/environments/test.js.example config/environments/test.js
-
-npm ci
-npm test

lib/api/api-router.js

@@ -1,13 +1,11 @@
 'use strict';
 
-const path = require('path');
-
 const { Router: router } = require('express');
 
 const RedisPool = require('redis-mpool');
 const cartodbRedis = require('cartodb-redis');
 
-const windshaft = require('windshaft');
+const { factory: windshaftFactory } = require('windshaft');
 
 const PgConnection = require('../backends/pg-connection');
 const AnalysisBackend = require('../backends/analysis');
@@ -22,8 +20,8 @@ const UserLimitsBackend = require('../backends/user-limits');
 const OverviewsMetadataBackend = require('../backends/overviews-metadata');
 const FilterStatsApi = require('../backends/filter-stats');
 const TablesExtentBackend = require('../backends/tables-extent');
-
 const ClusterBackend = require('../backends/cluster');
+const PubSubMetricsBackend = require('../backends/metrics');
 
 const LayergroupAffectedTablesCache = require('../cache/layergroup-affected-tables');
 const SurrogateKeysCache = require('../cache/surrogate-keys-cache');
@@ -31,7 +29,7 @@ const VarnishHttpCacheBackend = require('../cache/backend/varnish-http');
 const FastlyCacheBackend = require('../cache/backend/fastly');
 const NamedMapProviderCache = require('../cache/named-map-provider-cache');
 const NamedMapsCacheEntry = require('../cache/model/named-maps-entry');
-const NamedMapProviderReporter = require('../stats/reporter/named-map-provider');
+const NamedMapProviderCacheReporter = require('../stats/reporter/named-map-provider-cache');
 
 const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
 const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter');
@@ -49,20 +47,23 @@ const LayergroupMetadata = require('../utils/layergroup-metadata');
 const RendererStatsReporter = require('../stats/reporter/renderer');
 
 const initializeStatusCode = require('./middlewares/initialize-status-code');
-const logger = require('./middlewares/logger');
+const initLogger = require('./middlewares/logger');
 const bodyParser = require('body-parser');
 const servedByHostHeader = require('./middlewares/served-by-host-header');
-const stats = require('./middlewares/stats');
+const profiler = require('./middlewares/profiler');
 const lzmaMiddleware = require('./middlewares/lzma');
 const cors = require('./middlewares/cors');
 const user = require('./middlewares/user');
 const sendResponse = require('./middlewares/send-response');
 const syntaxError = require('./middlewares/syntax-error');
 const errorMiddleware = require('./middlewares/error-middleware');
+const clientHeader = require('./middlewares/client-header');
 
 const MapRouter = require('./map/map-router');
 const TemplateRouter = require('./template/template-router');
 
+const getOnTileErrorStrategy = require('../utils/on-tile-error-strategy');
+
 module.exports = class ApiRouter {
     constructor ({ serverOptions, environmentOptions }) {
         this.serverOptions = serverOptions;
@@ -82,40 +83,22 @@ module.exports = class ApiRouter {
             global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
         });
 
+        const { rendererCache, tileBackend, attributesBackend, previewBackend, mapBackend, mapStore } = windshaftFactory({
+            rendererOptions: serverOptions,
+            redisPool,
+            onTileErrorStrategy: getOnTileErrorStrategy({ enabled: environmentOptions.enabledFeatures.onTileErrorStrategy }),
+            logger: this.serverOptions.logger
+        });
+
+        const rendererStatsReporter = new RendererStatsReporter(rendererCache, serverOptions.renderCache.statsInterval);
+        rendererStatsReporter.start();
+
         const metadataBackend = cartodbRedis({ pool: redisPool });
         const pgConnection = new PgConnection(metadataBackend);
 
-        const windshaftLogger = environmentOptions.log_windshaft && global.log4js
-            ? global.log4js.getLogger('[windshaft]')
-            : null;
-        const mapStore = new windshaft.storage.MapStore({
-            pool: redisPool,
-            expire_time: serverOptions.grainstore.default_layergroup_ttl,
-            logger: windshaftLogger
-        });
-
-        const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions });
-
-        const rendererCacheOpts = Object.assign({
-            ttl: 60000, // 60 seconds TTL by default
-            statsInterval: 60000 // reports stats every milliseconds defined here
-        }, serverOptions.renderCache || {});
-
-        const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
-        const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
-        rendererStatsReporter.start();
-
-        const tileBackend = new windshaft.backend.Tile(rendererCache);
-        const attributesBackend = new windshaft.backend.Attributes();
-        const concurrency = serverOptions.renderer.mapnik.poolSize +
-                            serverOptions.renderer.mapnik.poolMaxWaitingClients;
-        const previewBackend = new windshaft.backend.Preview(rendererCache, { concurrency });
-        const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
-        const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
-
         const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions);
         const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
-        const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache });
+        const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache, logger: this.serverOptions.logger });
 
         const analysisStatusBackend = new AnalysisStatusBackend();
         const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
@@ -168,14 +151,16 @@ module.exports = class ApiRouter {
             layergroupAffectedTablesCache
         );
 
-        const namedMapProviderReporter = new NamedMapProviderReporter({
+        const namedMapProviderCacheReporter = new NamedMapProviderCacheReporter({
             namedMapProviderCache,
-            intervalInMilliseconds: rendererCacheOpts.statsInterval
+            intervalInMilliseconds: serverOptions.renderCache.statsInterval
         });
+        namedMapProviderCacheReporter.start();
 
-        namedMapProviderReporter.start();
+        const metricsBackend = new PubSubMetricsBackend(serverOptions.pubSubMetrics);
 
         const collaborators = {
+            config: serverOptions,
             analysisStatusBackend,
             attributesBackend,
             dataviewBackend,
@@ -195,9 +180,11 @@ module.exports = class ApiRouter {
             layergroupMetadata,
             namedMapProviderCache,
             tablesExtentBackend,
-            clusterBackend
+            clusterBackend,
+            metricsBackend
         };
 
+        this.metadataBackend = metadataBackend;
         this.mapRouter = new MapRouter({ collaborators });
         this.templateRouter = new TemplateRouter({ collaborators });
     }
@@ -212,19 +199,21 @@ module.exports = class ApiRouter {
             const apiRouter = router({ mergeParams: true });
             const { paths, middlewares = [] } = route;
 
-            middlewares.forEach(middleware => apiRouter.use(middleware()));
-
-            apiRouter.use(logger(this.serverOptions));
-            apiRouter.use(initializeStatusCode());
-            apiRouter.use(bodyParser.json());
-            apiRouter.use(servedByHostHeader());
-            apiRouter.use(stats({
+            apiRouter.use(initLogger({ logger: this.serverOptions.logger }));
+            apiRouter.use(user(this.metadataBackend));
+            apiRouter.use(profiler({
                 enabled: this.serverOptions.useProfiler,
                 statsClient: global.statsClient
             }));
+
+            middlewares.forEach(middleware => apiRouter.use(middleware()));
+
+            apiRouter.use(initializeStatusCode());
+            apiRouter.use(bodyParser.json());
+            apiRouter.use(servedByHostHeader());
+            apiRouter.use(clientHeader());
             apiRouter.use(lzmaMiddleware());
             apiRouter.use(cors());
-            apiRouter.use(user());
 
             this.templateRouter.route(apiRouter, route.template);
             this.mapRouter.route(apiRouter, route.map);
@@ -238,25 +227,20 @@ module.exports = class ApiRouter {
     }
 };
 
-function createTemplateMaps ({ redisPool, surrogateKeysCache }) {
+function createTemplateMaps ({ redisPool, surrogateKeysCache, logger }) {
     const templateMaps = new TemplateMaps(redisPool, {
         max_user_templates: global.environment.maxUserTemplates
     });
 
-    function invalidateNamedMap (owner, templateName) {
-        var startTime = Date.now();
-        surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function (err) {
-            var logMessage = JSON.stringify({
-                username: owner,
-                type: 'named_map_invalidation',
-                elapsed: Date.now() - startTime,
-                error: err ? JSON.stringify(err.message) : undefined
-            });
+    function invalidateNamedMap (user, templateName) {
+        const startTime = Date.now();
+        surrogateKeysCache.invalidate(new NamedMapsCacheEntry(user, templateName), (err) => {
             if (err) {
-                global.logger.warn(logMessage);
-            } else {
-                global.logger.info(logMessage);
+                return logger.error({ exception: err, 'cdb-user': user, template_id: templateName }, 'Named map invalidation failed');
             }
+
+            const elapsed = Date.now() - startTime;
+            logger.info({ 'cdb-user': user, template_id: templateName, duration: elapsed / 1000, duration_ms: elapsed }, 'Named map invalidation success');
         });
     }
 
@@ -285,51 +269,3 @@ function createSurrogateKeysCacheBackends (serverOptions) {
 
     return cacheBackends;
 }
-
-const timeoutErrorTilePath = path.join(__dirname, '/../../assets/render-timeout-fallback.png');
-const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, { encoding: null });
-
-function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) {
-    var onTileErrorStrategy;
-    if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) {
-        onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile (err, tile, headers, stats, format, callback) {
-            function isRenderTimeoutError (err) {
-                return err.message === 'Render timed out';
-            }
-
-            function isDatasourceTimeoutError (err) {
-                return err.message && err.message.match(/canceling statement due to statement timeout/i);
-            }
-
-            function isTimeoutError (err) {
-                return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
-            }
-
-            function isRasterFormat (format) {
-                return format === 'png' || format === 'jpg';
-            }
-
-            if (isTimeoutError(err) && isRasterFormat(format)) {
-                return callback(null, timeoutErrorTile, {
-                    'Content-Type': 'image/png'
-                }, {});
-            } else {
-                return callback(err, tile, headers, stats);
-            }
-        };
-    }
-
-    const rendererFactory = new windshaft.renderer.Factory({
-        onTileErrorStrategy: onTileErrorStrategy,
-        mapnik: {
-            redisPool: redisPool,
-            grainstore: serverOptions.grainstore,
-            mapnik: serverOptions.renderer.mapnik
-        },
-        http: serverOptions.renderer.http,
-        mvt: serverOptions.renderer.mvt,
-        torque: serverOptions.renderer.torque
-    });
-
-    return rendererFactory;
-}

lib/api/map/analyses-catalog-controller.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const PSQL = require('cartodb-psql');
+const tag = require('../middlewares/tag');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
 const authorize = require('../middlewares/authorize');
@@ -23,6 +24,7 @@ module.exports = class AnalysesController {
 
     middlewares () {
         return [
+            tag({ tags: ['analysis', 'catalog'] }),
             credentials(),
             authorize(this.authBackend),
             dbConnSetup(this.pgConnection),

lib/api/map/analysis-layergroup-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
@@ -23,6 +24,7 @@ module.exports = class AnalysisLayergroupController {
 
     middlewares () {
         return [
+            tag({ tags: ['analysis', 'node'] }),
             layergroupToken(),
             credentials(),
             authorize(this.authBackend),

lib/api/map/anonymous-map-controller.js

@@ -3,11 +3,11 @@
 const windshaft = require('windshaft');
 const MapConfig = windshaft.model.MapConfig;
 const Datasource = windshaft.model.Datasource;
+const tag = require('../middlewares/tag');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
 const dbConnSetup = require('../middlewares/db-conn-setup');
 const authorize = require('../middlewares/authorize');
-const initProfiler = require('../middlewares/init-profiler');
 const checkJsonContentType = require('../middlewares/check-json-content-type');
 const incrementMapViewCount = require('../middlewares/increment-map-view-count');
 const augmentLayergroupData = require('../middlewares/augment-layergroup-data');
@@ -23,6 +23,7 @@ const mapError = require('../middlewares/map-error');
 const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider');
 const rateLimit = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
+const metrics = require('../middlewares/metrics');
 
 module.exports = class AnonymousMapController {
     /**
@@ -39,6 +40,7 @@ module.exports = class AnonymousMapController {
      * @constructor
      */
     constructor (
+        config,
         pgConnection,
         templateMaps,
         mapBackend,
@@ -49,8 +51,10 @@ module.exports = class AnonymousMapController {
         mapConfigAdapter,
         statsBackend,
         authBackend,
-        layergroupMetadata
+        layergroupMetadata,
+        metricsBackend
     ) {
+        this.config = config;
         this.pgConnection = pgConnection;
         this.templateMaps = templateMaps;
         this.mapBackend = mapBackend;
@@ -62,6 +66,7 @@ module.exports = class AnonymousMapController {
         this.statsBackend = statsBackend;
         this.authBackend = authBackend;
         this.layergroupMetadata = layergroupMetadata;
+        this.metricsBackend = metricsBackend;
     }
 
     route (mapRouter) {
@@ -71,19 +76,32 @@ module.exports = class AnonymousMapController {
     }
 
     middlewares () {
-        const isTemplateInstantiation = false;
         const useTemplateHash = false;
         const includeQuery = true;
         const label = 'ANONYMOUS LAYERGROUP';
         const addContext = true;
+        const metricsTags = {
+            event: 'map_view',
+            attributes: { map_type: 'anonymous' },
+            from: {
+                req: {
+                    query: { client: 'client' }
+                }
+            }
+        };
 
         return [
+            tag({ tags: ['map', 'anonymous'] }),
+            metrics({
+                enabled: this.config.pubSubMetrics.enabled,
+                metricsBackend: this.metricsBackend,
+                tags: metricsTags
+            }),
             credentials(),
             authorize(this.authBackend),
             dbConnSetup(this.pgConnection),
             rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS),
             cleanUpQueryParams(['aggregation']),
-            initProfiler(isTemplateInstantiation),
             checkJsonContentType(),
             checkCreateLayergroup(),
             prepareAdapterMapConfig(this.mapConfigAdapter),
@@ -124,7 +142,6 @@ function checkCreateLayergroup () {
             }
         }
 
-        req.profiler.done('checkCreateLayergroup');
         return next();
     };
 }
@@ -133,6 +150,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
     return function prepareAdapterMapConfigMiddleware (req, res, next) {
         const requestMapConfig = req.body;
 
+        const { logger } = res.locals;
         const { user, api_key: apiKey } = res.locals;
         const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
         const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
@@ -140,6 +158,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
         const context = {
             analysisConfiguration: {
                 user,
+                logger,
                 db: {
                     host: dbhost,
                     port: dbport,
@@ -158,12 +177,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
             requestMapConfig,
             params,
             context,
-            (err, requestMapConfig, stats = { overviewsAddedToMapconfig: false }) => {
-                req.profiler.done('anonymous.getMapConfig');
-
-                stats.mapType = 'anonymous';
-                req.profiler.add(stats);
-
+            (err, requestMapConfig) => {
                 if (err) {
                     return next(err);
                 }
@@ -207,6 +221,7 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected
         );
 
         res.locals.mapConfig = mapConfig;
+        res.locals.mapConfigProvider = mapConfigProvider;
         res.locals.analysesResults = context.analysesResults;
 
         const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport };
@@ -220,7 +235,6 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected
 
             res.statusCode = 200;
             res.body = layergroup;
-            res.locals.mapConfigProvider = mapConfigProvider;
 
             next();
         });

lib/api/map/attributes-layergroup-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
@@ -38,6 +39,7 @@ module.exports = class AttributesLayergroupController {
 
     middlewares () {
         return [
+            tag({ tags: ['attributes'] }),
             layergroupToken(),
             credentials(),
             authorize(this.authBackend),
@@ -61,8 +63,6 @@ module.exports = class AttributesLayergroupController {
 
 function getFeatureAttributes (attributesBackend) {
     return function getFeatureAttributesMiddleware (req, res, next) {
-        req.profiler.start('windshaft.maplayer_attribute');
-
         const { mapConfigProvider } = res.locals;
         const { token } = res.locals;
         const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;

lib/api/map/clustered-features-layergroup-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
@@ -38,6 +39,7 @@ module.exports = class AggregatedFeaturesLayergroupController {
 
     middlewares () {
         return [
+            tag({ tags: ['cluster'] }),
             layergroupToken(),
             credentials(),
             authorize(this.authBackend),
@@ -62,8 +64,6 @@ module.exports = class AggregatedFeaturesLayergroupController {
 
 function getClusteredFeatures (clusterBackend) {
     return function getFeatureAttributesMiddleware (req, res, next) {
-        req.profiler.start('windshaft.maplayer_cluster_features');
-
         const { mapConfigProvider } = res.locals;
         const { user, token } = res.locals;
         const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;

lib/api/map/dataview-layergroup-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
@@ -76,6 +77,7 @@ module.exports = class DataviewLayergroupController {
 
     middlewares ({ action, rateLimitGroup }) {
         return [
+            tag({ tags: ['dataview', action] }),
             layergroupToken(),
             credentials(),
             authorize(this.authBackend),

lib/api/map/map-router.js

@@ -15,6 +15,7 @@ const ClusteredFeaturesLayergroupController = require('./clustered-features-laye
 module.exports = class MapRouter {
     constructor ({ collaborators }) {
         const {
+            config,
             analysisStatusBackend,
             attributesBackend,
             dataviewBackend,
@@ -34,7 +35,8 @@ module.exports = class MapRouter {
             layergroupMetadata,
             namedMapProviderCache,
             tablesExtentBackend,
-            clusterBackend
+            clusterBackend,
+            metricsBackend
         } = collaborators;
 
         this.analysisLayergroupController = new AnalysisLayergroupController(
@@ -85,6 +87,7 @@ module.exports = class MapRouter {
         );
 
         this.anonymousMapController = new AnonymousMapController(
+            config,
             pgConnection,
             templateMaps,
             mapBackend,
@@ -95,10 +98,12 @@ module.exports = class MapRouter {
             mapConfigAdapter,
             statsBackend,
             authBackend,
-            layergroupMetadata
+            layergroupMetadata,
+            metricsBackend
         );
 
         this.previewTemplateController = new PreviewTemplateController(
+            config,
             namedMapProviderCache,
             previewBackend,
             surrogateKeysCache,
@@ -106,7 +111,8 @@ module.exports = class MapRouter {
             metadataBackend,
             pgConnection,
             authBackend,
-            userLimitsBackend
+            userLimitsBackend,
+            metricsBackend
         );
 
         this.analysesController = new AnalysesCatalogController(

lib/api/map/preview-layergroup-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const layergroupToken = require('../middlewares/layergroup-token');
 const coordinates = require('../middlewares/coordinates');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@@ -61,6 +62,7 @@ module.exports = class PreviewLayergroupController {
         }
 
         return [
+            tag({ tags: ['static', 'tile'] }),
             layergroupToken(),
             validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(),
             credentials(),
@@ -100,7 +102,6 @@ function getPreviewImageByCenter (previewBackend) {
         const options = { mapConfigProvider, format, width, height, zoom, center };
 
         previewBackend.getImage(options, (err, image, stats = {}) => {
-            req.profiler.done(`render-${format}`);
             req.profiler.add(stats);
 
             if (err) {
@@ -133,7 +134,6 @@ function getPreviewImageByBoundingBox (previewBackend) {
         const options = { mapConfigProvider, format, width, height, bbox };
 
         previewBackend.getImage(options, (err, image, stats = {}) => {
-            req.profiler.done(`render-${format}`);
             req.profiler.add(stats);
 
             if (err) {

lib/api/map/preview-template-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
 const dbConnSetup = require('../middlewares/db-conn-setup');
@@ -12,6 +13,7 @@ const lastModifiedHeader = require('../middlewares/last-modified-header');
 const checkStaticImageFormat = require('../middlewares/check-static-image-format');
 const rateLimit = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
+const metrics = require('../middlewares/metrics');
 
 const DEFAULT_ZOOM_CENTER = {
     zoom: 1,
@@ -27,6 +29,7 @@ function numMapper (n) {
 
 module.exports = class PreviewTemplateController {
     constructor (
+        config,
         namedMapProviderCache,
         previewBackend,
         surrogateKeysCache,
@@ -34,8 +37,10 @@ module.exports = class PreviewTemplateController {
         metadataBackend,
         pgConnection,
         authBackend,
-        userLimitsBackend
+        userLimitsBackend,
+        metricsBackend
     ) {
+        this.config = config;
         this.namedMapProviderCache = namedMapProviderCache;
         this.previewBackend = previewBackend;
         this.surrogateKeysCache = surrogateKeysCache;
@@ -44,6 +49,7 @@ module.exports = class PreviewTemplateController {
         this.pgConnection = pgConnection;
         this.authBackend = authBackend;
         this.userLimitsBackend = userLimitsBackend;
+        this.metricsBackend = metricsBackend;
     }
 
     route (mapRouter) {
@@ -51,7 +57,23 @@ module.exports = class PreviewTemplateController {
     }
 
     middlewares () {
+        const metricsTags = {
+            event: 'map_view',
+            attributes: { map_type: 'static' },
+            from: {
+                req: {
+                    query: { client: 'client' }
+                }
+            }
+        };
+
         return [
+            tag({ tags: ['named', 'static', 'tile'] }),
+            metrics({
+                enabled: this.config.pubSubMetrics.enabled,
+                metricsBackend: this.metricsBackend,
+                tags: metricsTags
+            }),
             credentials(),
             authorize(this.authBackend),
             dbConnSetup(this.pgConnection),
@@ -272,7 +294,7 @@ function getImage ({ previewBackend, label }) {
         if (zoom !== undefined && center) {
             const options = { mapConfigProvider, format, width, height, zoom, center };
 
-            return previewBackend.getImage(options, (err, image, stats) => {
+            return previewBackend.getImage(options, (err, image, stats = {}) => {
                 req.profiler.add(stats);
 
                 if (err) {
@@ -289,9 +311,8 @@ function getImage ({ previewBackend, label }) {
 
         const options = { mapConfigProvider, format, width, height, bbox };
 
-        previewBackend.getImage(options, (err, image, stats) => {
+        previewBackend.getImage(options, (err, image, stats = {}) => {
             req.profiler.add(stats);
-            req.profiler.done('render-' + format);
 
             if (err) {
                 err.label = label;
@@ -316,25 +337,23 @@ function setContentTypeHeader () {
     };
 }
 
-function incrementMapViewsError (ctx) {
-    return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
-}
-
 function incrementMapViews ({ metadataBackend }) {
     return function incrementMapViewsMiddleware (req, res, next) {
-        const { user, mapConfigProvider } = res.locals;
+        const { user, mapConfigProvider, logger } = res.locals;
 
         mapConfigProvider.getMapConfig((err, mapConfig) => {
             if (err) {
-                global.logger.log(incrementMapViewsError({ user, err }));
+                logger.warn({ exception: err }, 'Failed to increment mapview count');
                 return next();
             }
 
+            res.locals.mapConfig = mapConfig;
+
             const statTag = mapConfig.obj().stat_tag;
 
             metadataBackend.incMapviewCount(user, statTag, (err) => {
                 if (err) {
-                    global.logger.log(incrementMapViewsError({ user, err }));
+                    logger.warn({ exception: err }, 'Failed to increment mapview count');
                 }
 
                 next();

lib/api/map/tile-layergroup-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const layergroupToken = require('../middlewares/layergroup-token');
 const coordinates = require('../middlewares/coordinates');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@@ -59,6 +60,7 @@ module.exports = class TileLayergroupController {
 
     middlewares () {
         return [
+            tag({ tags: ['tile'] }),
             layergroupToken(),
             coordinates(),
             credentials(),
@@ -96,8 +98,6 @@ function getStatusCode (tile, format) {
 
 function getTile (tileBackend) {
     return function getTileMiddleware (req, res, next) {
-        req.profiler.start(`windshaft.${req.params.layer ? 'maplayer_tile' : 'map_tile'}`);
-
         const { mapConfigProvider } = res.locals;
         const { token } = res.locals;
         const { layer, z, x, y, format } = req.params;

lib/api/middlewares/authorize.js

@@ -3,8 +3,6 @@
 module.exports = function authorize (authBackend) {
     return function authorizeMiddleware (req, res, next) {
         authBackend.authorize(req, res, (err, authorized) => {
-            req.profiler.done('authorize');
-
             if (err) {
                 return next(err);
             }

lib/api/middlewares/cache-channel-header.js

@@ -6,11 +6,11 @@ module.exports = function setCacheChannelHeader () {
             return next();
         }
 
-        const { mapConfigProvider } = res.locals;
+        const { mapConfigProvider, logger } = res.locals;
 
         mapConfigProvider.getAffectedTables((err, affectedTables) => {
             if (err) {
-                global.logger.warn('ERROR generating Cache Channel Header:', err);
+                logger.warn({ exception: err }, 'Error generating Cache Channel Header');
                 return next();
             }
 

lib/api/middlewares/cache-control-header.js

@@ -40,11 +40,11 @@ module.exports = function setCacheControlHeader ({
             return next();
         }
 
-        const { mapConfigProvider = { getAffectedTables: callback => callback() } } = res.locals;
+        const { mapConfigProvider = { getAffectedTables: callback => callback() }, logger } = res.locals;
 
         mapConfigProvider.getAffectedTables((err, affectedTables) => {
             if (err) {
-                global.logger.warn('ERROR generating Cache Control Header:', err);
+                logger.warn({ exception: err }, 'Error generating Cache Control Header');
                 return next();
             }
 

lib/api/middlewares/check-json-content-type.js

@@ -6,8 +6,6 @@ module.exports = function checkJsonContentType () {
             return next(new Error('POST data must be of type application/json'));
         }
 
-        req.profiler.done('checkJsonContentTypeMiddleware');
-
         next();
     };
 };

lib/api/middlewares/client-header.js

@@ -0,0 +1,13 @@
+'use strict';
+
+module.exports = function clientHeader () {
+    return function clientHeaderMiddleware (req, res, next) {
+        const { client } = req.query;
+
+        if (client) {
+            res.set('Carto-Client', client);
+        }
+
+        return next();
+    };
+};

lib/api/middlewares/cors.js

@@ -6,7 +6,10 @@ module.exports = function cors () {
             'X-Requested-With',
             'X-Prototype-Version',
             'X-CSRF-Token',
-            'Authorization'
+            'Authorization',
+            'Carto-Event',
+            'Carto-Event-Source',
+            'Carto-Event-Group-Id'
         ];
 
         if (req.method === 'OPTIONS') {

lib/api/middlewares/db-conn-setup.js

@@ -7,8 +7,6 @@ module.exports = function dbConnSetup (pgConnection) {
         const { user } = res.locals;
 
         pgConnection.setDBConn(user, res.locals, (err) => {
-            req.profiler.done('dbConnSetup');
-
             if (err) {
                 if (err.message && err.message.indexOf('name not found') !== -1) {
                     err.http_status = 404;

lib/api/middlewares/error-middleware.js

@@ -1,42 +1,31 @@
 'use strict';
 
-const _ = require('underscore');
-const debug = require('debug')('windshaft:cartodb:error-middleware');
+const setCommonHeaders = require('../../utils/common-headers');
 
 module.exports = function errorMiddleware (/* options */) {
     return function error (err, req, res, next) {
-        // jshint unused:false
-        // jshint maxcomplexity:9
-        var allErrors = Array.isArray(err) ? err : [err];
+        const { logger } = res.locals;
+        const errors = populateLimitErrors(Array.isArray(err) ? err : [err]);
 
-        allErrors = populateLimitErrors(allErrors);
+        errors.forEach((err) => logger.error({ exception: err }, 'Error while handling the request'));
 
-        const label = err.label || 'UNKNOWN';
-        err = allErrors[0] || new Error(label);
-        allErrors[0] = err;
+        setCommonHeaders(req, res, () => {
+            const errorResponseBody = {
+                errors: errors.map(errorMessage),
+                errors_with_context: errors.map(errorMessageWithContext)
+            };
 
-        var statusCode = findStatusCode(err);
+            // If a callback was requested, force status to 200
+            res.status(req.query.callback ? 200 : findStatusCode(errors[0]));
 
-        setErrorHeader(allErrors, statusCode, res);
-        debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
+            if (req.query && req.query.callback) {
+                res.jsonp(errorResponseBody);
+            } else {
+                res.json(errorResponseBody);
+            }
 
-        // If a callback was requested, force status to 200
-        if (req.query && req.query.callback) {
-            statusCode = 200;
-        }
-
-        var errorResponseBody = {
-            errors: allErrors.map(errorMessage),
-            errors_with_context: allErrors.map(errorMessageWithContext)
-        };
-
-        res.status(statusCode);
-
-        if (req.query && req.query.callback) {
-            res.jsonp(errorResponseBody);
-        } else {
-            res.json(errorResponseBody);
-        }
+            return next();
+        });
     };
 };
 
@@ -113,7 +102,6 @@ module.exports.findStatusCode = findStatusCode;
 
 function statusFromErrorMessage (errMsg) {
     // Find an appropriate statusCode based on message
-    // jshint maxcomplexity:7
     var statusCode = 400;
     if (errMsg.indexOf('permission denied') !== -1) {
         statusCode = 403;
@@ -136,7 +124,7 @@ function statusFromErrorMessage (errMsg) {
 
 function errorMessage (err) {
     // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+    var message = (typeof err === 'string' ? err : err.message) || 'Unknown error';
 
     return stripConnectionInfo(message);
 }
@@ -166,7 +154,7 @@ function shouldBeExposed (prop) {
 
 function errorMessageWithContext (err) {
     // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+    var message = (typeof err === 'string' ? err : err.message) || 'Unknown error';
 
     var error = {
         type: err.type || 'unknown',
@@ -182,53 +170,3 @@ function errorMessageWithContext (err) {
 
     return error;
 }
-
-function setErrorHeader (errors, statusCode, res) {
-    const errorsCopy = errors.slice(0);
-    const mainError = errorsCopy.shift();
-
-    const errorsLog = {
-        mainError: {
-            statusCode: statusCode || 200,
-            message: mainError.message,
-            name: mainError.name,
-            label: mainError.label,
-            type: mainError.type,
-            subtype: mainError.subtype
-        }
-    };
-
-    errorsLog.moreErrors = errorsCopy.map(error => {
-        return {
-            message: error.message,
-            name: error.name,
-            label: error.label,
-            type: error.type,
-            subtype: error.subtype
-        };
-    });
-
-    res.set('X-Tiler-Errors', stringifyForLogs(errorsLog));
-}
-
-/**
- * Remove problematic nested characters
- * from object for logs RegEx
- *
- * @param {Object} object
- */
-function stringifyForLogs (object) {
-    Object.keys(object).map(key => {
-        if (typeof object[key] === 'string') {
-            object[key] = object[key].replace(/[^a-zA-Z0-9]/g, ' ');
-        } else if (typeof object[key] === 'object') {
-            stringifyForLogs(object[key]);
-        } else if (object[key] instanceof Array) {
-            for (const element of object[key]) {
-                stringifyForLogs(element);
-            }
-        }
-    });
-
-    return JSON.stringify(object);
-}

lib/api/middlewares/increment-map-view-count.js

@@ -2,14 +2,12 @@
 
 module.exports = function incrementMapViewCount (metadataBackend) {
     return function incrementMapViewCountMiddleware (req, res, next) {
-        const { mapConfig, user } = res.locals;
-
-        // Error won't blow up, just be logged.
-        metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => {
-            req.profiler.done('incMapviewCount');
+        const { mapConfig, user, logger } = res.locals;
+        const statTag = mapConfig.obj().stat_tag;
 
+        metadataBackend.incMapviewCount(user, statTag, (err) => {
             if (err) {
-                global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`);
+                logger.warn({ exception: err }, 'Failed to increment mapview count');
             }
 
             next();

lib/api/middlewares/init-profiler.js

@@ -1,11 +0,0 @@
-'use strict';
-
-module.exports = function initProfiler (isTemplateInstantiation) {
-    const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
-
-    return function initProfilerMiddleware (req, res, next) {
-        req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`);
-        req.profiler.done(`${operation}.initProfilerMiddleware`);
-        next();
-    };
-};

lib/api/middlewares/last-modified-header.js

@@ -6,7 +6,7 @@ module.exports = function setLastModifiedHeader () {
             return next();
         }
 
-        const { mapConfigProvider, cache_buster: cacheBuster } = res.locals;
+        const { mapConfigProvider, cache_buster: cacheBuster, logger } = res.locals;
 
         if (cacheBuster) {
             const cacheBusterTimestamp = parseInt(cacheBuster, 10);
@@ -21,7 +21,7 @@ module.exports = function setLastModifiedHeader () {
 
         mapConfigProvider.getAffectedTables((err, affectedTables) => {
             if (err) {
-                global.logger.warn('ERROR generating Last Modified Header:', err);
+                logger.warn({ exception: err }, 'Error generating Last Modified Header');
                 return next();
             }
 
@@ -36,6 +36,8 @@ module.exports = function setLastModifiedHeader () {
 
             res.set('Last-Modified', lastModifiedDate.toUTCString());
 
+            res.locals.cache_buster = lastUpdatedAt;
+
             next();
         });
     };

lib/api/middlewares/last-updated-time-layergroup.js

@@ -11,6 +11,10 @@ module.exports = function setLastUpdatedTimeToLayergroup () {
             }
 
             if (!affectedTables) {
+                res.locals.cache_buster = 0;
+                layergroup.layergroupid = `${layergroup.layergroupid}:${res.locals.cache_buster}`;
+                layergroup.last_updated = new Date(res.locals.cache_buster).toISOString();
+
                 return next();
             }
 
@@ -22,6 +26,8 @@ module.exports = function setLastUpdatedTimeToLayergroup () {
             layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
             layergroup.last_updated = new Date(lastUpdateTime).toISOString();
 
+            res.locals.cache_buster = lastUpdateTime;
+
             next();
         });
     };

lib/api/middlewares/layergroup-id-header.js

@@ -6,8 +6,9 @@ module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash)
         const layergroup = res.body;
 
         if (useTemplateHash) {
-            var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
+            const templateHash = templateMaps.fingerPrint(template).substring(0, 8);
             layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
+            res.locals.templateHash = templateHash;
         }
 
         res.set('X-Layergroup-Id', layergroup.layergroupid);

lib/api/middlewares/layergroup-token.js

@@ -13,6 +13,10 @@ module.exports = function layergroupToken () {
         res.locals.token = layergroupToken.token;
         res.locals.cache_buster = layergroupToken.cacheBuster;
 
+        if (layergroupToken.templateHash) {
+            res.locals.templateHash = layergroupToken.templateHash;
+        }
+
         if (layergroupToken.signer) {
             res.locals.signer = layergroupToken.signer;
 

lib/api/middlewares/logger.js

@@ -1,24 +1,12 @@
 'use strict';
 
-module.exports = function logger (options) {
-    if (!global.log4js || !options.log_format) {
-        return function dummyLoggerMiddleware (req, res, next) {
-            next();
-        };
-    }
+const uuid = require('uuid');
 
-    const opts = {
-        level: 'info',
-        // Allowing for unbuffered logging is mainly
-        // used to avoid hanging during unit testing.
-        // TODO: provide an explicit teardown function instead,
-        //       releasing any event handler or timer set by
-        //       this component.
-        buffer: !options.unbuffered_logging,
-        // optional log format
-        format: options.log_format
+module.exports = function initLogger ({ logger }) {
+    return function initLoggerMiddleware (req, res, next) {
+        res.locals.logger = logger.child({ request_id: req.get('X-Request-Id') || uuid.v4(), 'cdb-user': res.locals.user });
+        res.locals.logger.info({ client_request: req }, 'Incoming request');
+        res.on('finish', () => res.locals.logger.info({ server_response: res, status: res.statusCode }, 'Response sent'));
+        next();
     };
-    const logger = global.log4js.getLogger();
-
-    return global.log4js.connectLogger(logger, opts);
 };

lib/api/middlewares/lzma.js

@@ -24,8 +24,6 @@ module.exports = function lzma () {
                 delete req.query.lzma;
                 Object.assign(req.query, JSON.parse(result));
 
-                req.profiler.done('lzma');
-
                 next();
             } catch (err) {
                 next(new Error('Error parsing lzma as JSON: ' + err));

lib/api/middlewares/map-error.js

@@ -4,7 +4,6 @@ module.exports = function mapError (options) {
     const { addContext = false, label = 'MAPS CONTROLLER' } = options;
 
     return function mapErrorMiddleware (err, req, res, next) {
-        req.profiler.done('error');
         const { mapConfig } = res.locals;
 
         if (addContext) {

lib/api/middlewares/metrics.js

@@ -0,0 +1,181 @@
+'use strict';
+
+const EVENT_VERSION = '1';
+const MAX_LENGTH = 100;
+
+module.exports = function metrics ({ enabled, tags, metricsBackend }) {
+    if (!enabled) {
+        return function metricsDisabledMiddleware (req, res, next) {
+            next();
+        };
+    }
+
+    if (!tags || !tags.event) {
+        throw new Error('Missing required "event" parameter to report metrics');
+    }
+
+    return function metricsMiddleware (req, res, next) {
+        // FIXME: use parent logger as we don't want bind the error to the request
+        // but we still want to know if an error is thrown
+        const { logger } = res.locals;
+
+        res.on('finish', () => {
+            const { event, attributes } = getEventData(req, res, tags);
+
+            metricsBackend.send(event, attributes)
+                .catch((err) => logger.error({ exception: err, event }, 'Failed to publish event'));
+        });
+
+        return next();
+    };
+};
+
+function getEventData (req, res, tags) {
+    const event = tags.event;
+    const extra = {};
+    if (tags.from) {
+        if (tags.from.req) {
+            Object.assign(extra, getFromReq(req, tags.from.req));
+        }
+
+        if (tags.from.res) {
+            Object.assign(extra, getFromRes(res, tags.from.res));
+        }
+    }
+
+    const attributes = Object.assign({}, {
+        client_event: normalizedField(req.get('Carto-Event')),
+        client_event_group_id: normalizedField(req.get('Carto-Event-Group-Id')),
+        event_source: normalizedField(req.get('Carto-Event-Source')),
+        event_time: new Date().toISOString(),
+        user_id: res.locals.userId,
+        user_agent: req.get('User-Agent'),
+        map_id: getLayergroupid({ res }),
+        cache_buster: getCacheBuster({ res }),
+        template_hash: getTemplateHash({ res }),
+        stat_tag: getStatTag({ res }),
+        response_code: res.statusCode.toString(),
+        response_time: getResponseTime(req),
+        source_domain: req.hostname,
+        event_version: EVENT_VERSION
+    }, tags.attributes, extra);
+
+    // remove undefined properties
+    Object.keys(attributes).forEach(key => attributes[key] === undefined && delete attributes[key]);
+
+    return { event, attributes };
+}
+
+function normalizedField (field) {
+    if (!field) {
+        return undefined;
+    }
+
+    return field.toString().trim().substr(0, MAX_LENGTH);
+}
+
+function getLayergroupid ({ res }) {
+    if (res.locals.token) {
+        return res.locals.token;
+    }
+
+    if (res.locals.mapConfig) {
+        return res.locals.mapConfig.id();
+    }
+
+    if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) {
+        return res.locals.mapConfigProvider.mapConfig.id();
+    }
+}
+
+function getCacheBuster ({ res }) {
+    if (res.locals.cache_buster !== undefined) {
+        return `${res.locals.cache_buster}`;
+    }
+
+    if (res.locals.mapConfigProvider) {
+        return `${res.locals.mapConfigProvider.getCacheBuster()}`;
+    }
+}
+
+function getTemplateHash ({ res }) {
+    if (res.locals.templateHash) {
+        return res.locals.templateHash;
+    }
+
+    if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.getTemplateHash) {
+        let templateHash;
+
+        try {
+            templateHash = res.locals.mapConfigProvider.getTemplateHash().substring(0, 8);
+        } catch (e) {}
+
+        return templateHash;
+    }
+}
+
+function getStatTag ({ res }) {
+    if (res.locals.mapConfig) {
+        return res.locals.mapConfig.obj().stat_tag;
+    }
+
+    // FIXME: don't expect that mapConfig is already set
+    if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) {
+        return res.locals.mapConfigProvider.mapConfig.obj().stat_tag;
+    }
+}
+
+// FIXME: 'Profiler' might not be accurate enough
+function getResponseTime (req) {
+    let stats;
+
+    try {
+        stats = req.profiler.toJSON();
+    } catch (e) {
+        return undefined;
+    }
+
+    return stats && stats.total ? stats.total.toString() : undefined;
+}
+
+function getFromReq (req, { query = {}, body = {}, params = {}, headers = {} } = {}) {
+    const extra = {};
+
+    for (const [queryParam, eventName] of Object.entries(query)) {
+        extra[eventName] = req.query[queryParam];
+    }
+
+    for (const [bodyParam, eventName] of Object.entries(body)) {
+        extra[eventName] = req.body[bodyParam];
+    }
+
+    for (const [pathParam, eventName] of Object.entries(params)) {
+        extra[eventName] = req.params[pathParam];
+    }
+
+    for (const [header, eventName] of Object.entries(headers)) {
+        extra[eventName] = req.get(header);
+    }
+
+    return extra;
+}
+
+function getFromRes (res, { body = {}, headers = {}, locals = {} } = {}) {
+    const extra = {};
+
+    if (res.body) {
+        for (const [bodyParam, eventName] of Object.entries(body)) {
+            extra[eventName] = res.body[bodyParam];
+        }
+    }
+
+    for (const [header, eventName] of Object.entries(headers)) {
+        extra[eventName] = res.get(header);
+    }
+
+    for (const [localParam, eventName] of Object.entries(locals)) {
+        extra[eventName] = res.locals[localParam];
+    }
+
+    return extra;
+}

lib/api/middlewares/profiler.js

@@ -2,20 +2,28 @@
 
 const Profiler = require('../../stats/profiler-proxy');
 const debug = require('debug')('windshaft:cartodb:stats');
-const onHeaders = require('on-headers');
+const { name: prefix } = require('../../../package.json');
 
-module.exports = function stats (options) {
+module.exports = function profiler (options) {
     const { enabled = true, statsClient } = options;
 
-    return function statsMiddleware (req, res, next) {
+    return function profilerMiddleware (req, res, next) {
+        const { logger } = res.locals;
+
+        // TODO: stop using profiler and log stats instead of adding them to the profiler
         req.profiler = new Profiler({
             statsd_client: statsClient,
             profile: enabled
         });
 
-        onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString()));
+        req.profiler.start(prefix);
 
         res.on('finish', () => {
+            req.profiler.done('response');
+            req.profiler.end();
+            const stats = req.profiler.toJSON();
+            logger.info({ stats, duration: stats.response / 1000, duration_ms: stats.response }, 'Request profiling stats');
+
             try {
                 // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166
                 req.profiler.sendStats();

lib/api/middlewares/send-response.js

@@ -1,19 +1,24 @@
 'use strict';
 
+const setCommonHeaders = require('../../utils/common-headers');
+
 module.exports = function sendResponse () {
-    return function sendResponseMiddleware (req, res) {
-        req.profiler.done('res');
+    return function sendResponseMiddleware (req, res, next) {
+        setCommonHeaders(req, res, () => {
+            res.status(res.statusCode);
 
-        res.status(res.statusCode);
+            if (Buffer.isBuffer(res.body)) {
+                res.send(res.body);
+                return next();
+            }
 
-        if (Buffer.isBuffer(res.body)) {
-            return res.send(res.body);
-        }
+            if (req.query.callback) {
+                res.jsonp(res.body);
+                return next();
+            }
 
-        if (req.query.callback) {
-            return res.jsonp(res.body);
-        }
-
-        res.json(res.body);
+            res.json(res.body);
+            return next();
+        });
     };
 };

lib/api/middlewares/surrogate-key-header.js

@@ -5,7 +5,7 @@ const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named
 
 module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
     return function setSurrogateKeyHeaderMiddleware (req, res, next) {
-        const { user, mapConfigProvider } = res.locals;
+        const { user, mapConfigProvider, logger } = res.locals;
 
         if (mapConfigProvider instanceof NamedMapMapConfigProvider) {
             surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName()));
@@ -17,7 +17,7 @@ module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
 
         mapConfigProvider.getAffectedTables((err, affectedTables) => {
             if (err) {
-                global.logger.warn('ERROR generating Surrogate Key Header:', err);
+                logger.warn({ exception: err }, 'Error generating Surrogate Key Header');
                 return next();
             }
 

lib/api/middlewares/tag.js

@@ -0,0 +1,15 @@
+'use strict';
+
+module.exports = function tag ({ tags }) {
+    if (!Array.isArray(tags) || !tags.every((tag) => typeof tag === 'string')) {
+        throw new Error('Required "tags" option must be a valid Array: [string, string, ...]');
+    }
+
+    return function tagMiddleware (req, res, next) {
+        const { logger } = res.locals;
+        res.locals.tags = tags;
+        res.on('finish', () => logger.info({ tags: res.locals.tags }, 'Request tagged'));
+
+        next();
+    };
+};

lib/api/middlewares/user.js

@@ -2,12 +2,28 @@
 
 const CdbRequest = require('../../models/cdb-request');
 
-module.exports = function user () {
+module.exports = function user (metadataBackend) {
     const cdbRequest = new CdbRequest();
 
     return function userMiddleware (req, res, next) {
-        res.locals.user = cdbRequest.userByReq(req);
+        try {
+            res.locals.user = getUserNameFromRequest(req, cdbRequest);
+        } catch (err) {
+            return next(err);
+        }
 
-        next();
+        metadataBackend.getUserId(res.locals.user, (err, userId) => {
+            if (err || !userId) {
+                return next();
+            }
+
+            res.locals.userId = userId;
+
+            return next();
+        });
     };
 };
+
+function getUserNameFromRequest (req, cdbRequest) {
+    return cdbRequest.userByReq(req);
+}

lib/api/template/admin-template-controller.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { templateName } = require('../../backends/template-maps');
+const tag = require('../middlewares/tag');
 const credentials = require('../middlewares/credentials');
 const rateLimit = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
@@ -76,6 +77,7 @@ module.exports = class AdminTemplateController {
         }
 
         return [
+            tag({ tags: ['named', 'admin', action] }),
             credentials(),
             authorizedByAPIKey({ authBackend: this.authBackend, action, label }),
             rateLimit(this.userLimitsBackend, rateLimitGroup),
@@ -166,8 +168,6 @@ function updateTemplate ({ templateMaps }) {
 
 function retrieveTemplate ({ templateMaps }) {
     return function retrieveTemplateMiddleware (req, res, next) {
-        req.profiler.start('windshaft-cartodb.get_template');
-
         const { user } = res.locals;
         const templateId = templateName(req.params.template_id);
 
@@ -195,8 +195,6 @@ function retrieveTemplate ({ templateMaps }) {
 
 function destroyTemplate ({ templateMaps }) {
     return function destroyTemplateMiddleware (req, res, next) {
-        req.profiler.start('windshaft-cartodb.delete_template');
-
         const { user } = res.locals;
         const templateId = templateName(req.params.template_id);
 
@@ -215,8 +213,6 @@ function destroyTemplate ({ templateMaps }) {
 
 function listTemplates ({ templateMaps }) {
     return function listTemplatesMiddleware (req, res, next) {
-        req.profiler.start('windshaft-cartodb.get_template_list');
-
         const { user } = res.locals;
 
         templateMaps.listTemplates(user, (err, templateIds) => {

lib/api/template/named-template-controller.js

@@ -1,10 +1,10 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
 const dbConnSetup = require('../middlewares/db-conn-setup');
 const authorize = require('../middlewares/authorize');
-const initProfiler = require('../middlewares/init-profiler');
 const checkJsonContentType = require('../middlewares/check-json-content-type');
 const incrementMapViewCount = require('../middlewares/increment-map-view-count');
 const augmentLayergroupData = require('../middlewares/augment-layergroup-data');
@@ -21,6 +21,7 @@ const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named
 const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider');
 const rateLimit = require('../middlewares/rate-limit');
 const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
+const metrics = require('../middlewares/metrics');
 
 module.exports = class NamedMapController {
     /**
@@ -38,6 +39,7 @@ module.exports = class NamedMapController {
      * @constructor
      */
     constructor (
+        config,
         pgConnection,
         templateMaps,
         mapBackend,
@@ -48,8 +50,10 @@ module.exports = class NamedMapController {
         mapConfigAdapter,
         statsBackend,
         authBackend,
-        layergroupMetadata
+        layergroupMetadata,
+        metricsBackend
     ) {
+        this.config = config;
         this.pgConnection = pgConnection;
         this.templateMaps = templateMaps;
         this.mapBackend = mapBackend;
@@ -61,6 +65,7 @@ module.exports = class NamedMapController {
         this.statsBackend = statsBackend;
         this.authBackend = authBackend;
         this.layergroupMetadata = layergroupMetadata;
+        this.metricsBackend = metricsBackend;
     }
 
     route (templateRouter) {
@@ -69,19 +74,32 @@ module.exports = class NamedMapController {
     }
 
     middlewares () {
-        const isTemplateInstantiation = true;
         const useTemplateHash = true;
         const includeQuery = false;
         const label = 'NAMED MAP LAYERGROUP';
         const addContext = false;
+        const metricsTags = {
+            event: 'map_view',
+            attributes: { map_type: 'named' },
+            from: {
+                req: {
+                    query: { client: 'client' }
+                }
+            }
+        };
 
         return [
+            tag({ tags: ['map', 'named'] }),
+            metrics({
+                enabled: this.config.pubSubMetrics.enabled,
+                metricsBackend: this.metricsBackend,
+                tags: metricsTags
+            }),
             credentials(),
             authorize(this.authBackend),
             dbConnSetup(this.pgConnection),
             rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED),
             cleanUpQueryParams(['aggregation']),
-            initProfiler(isTemplateInstantiation),
             checkJsonContentType(),
             checkInstantiteLayergroup(),
             getTemplate(
@@ -131,8 +149,6 @@ function checkInstantiteLayergroup () {
             }
         }
 
-        req.profiler.done('checkInstantiteLayergroup');
-
         return next();
     };
 }
@@ -167,10 +183,9 @@ function getTemplate (
             params
         );
 
-        mapConfigProvider.getMapConfig((err, mapConfig, rendererParams, context, stats = {}) => {
-            req.profiler.done('named.getMapConfig');
+        mapConfigProvider.logger = res.locals.logger;
 
-            stats.mapType = 'named';
+        mapConfigProvider.getMapConfig((err, mapConfig, rendererParams, context, stats = {}) => {
             req.profiler.add(stats);
 
             if (err) {

lib/api/template/template-router.js

@@ -9,6 +9,7 @@ const TileTemplateController = require('./tile-template-controller');
 module.exports = class TemplateRouter {
     constructor ({ collaborators }) {
         const {
+            config,
             pgConnection,
             templateMaps,
             mapBackend,
@@ -21,10 +22,12 @@ module.exports = class TemplateRouter {
             authBackend,
             layergroupMetadata,
             namedMapProviderCache,
-            tileBackend
+            tileBackend,
+            metricsBackend
         } = collaborators;
 
         this.namedMapController = new NamedMapController(
+            config,
             pgConnection,
             templateMaps,
             mapBackend,
@@ -35,7 +38,8 @@ module.exports = class TemplateRouter {
             mapConfigAdapter,
             statsBackend,
             authBackend,
-            layergroupMetadata
+            layergroupMetadata,
+            metricsBackend
         );
 
         this.tileTemplateController = new TileTemplateController(

lib/api/template/tile-template-controller.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const tag = require('../middlewares/tag');
 const coordinates = require('../middlewares/coordinates');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
@@ -37,6 +38,7 @@ module.exports = class TileTemplateController {
 
     middlewares () {
         return [
+            tag({ tags: ['tile', 'named'] }),
             coordinates(),
             credentials(),
             authorize(this.authBackend),
@@ -67,9 +69,8 @@ function getTile ({ tileBackend, label }) {
         const { layer, z, x, y, format } = req.params;
         const params = { layer, z, x, y, format };
 
-        tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => {
+        tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
             req.profiler.add(stats);
-            req.profiler.done('render-' + format);
 
             if (err) {
                 err.label = label;

lib/backends/analysis.js

@@ -2,7 +2,6 @@
 
 var _ = require('underscore');
 var camshaft = require('camshaft');
-var fs = require('fs');
 
 var REDIS_LIMITS = {
     DB: 5,
@@ -14,7 +13,6 @@ function AnalysisBackend (metadataBackend, options) {
     this.options = options || {};
     this.options.limits = this.options.limits || {};
     this.setBatchConfig(this.options.batch);
-    this.setLoggerConfig(this.options.logger);
 }
 
 module.exports = AnalysisBackend;
@@ -27,31 +25,11 @@ AnalysisBackend.prototype.setBatchConfig = function (options) {
     this.batchConfig = batchConfig;
 };
 
-AnalysisBackend.prototype.setLoggerConfig = function (options) {
-    this.loggerConfig = options || {};
-
-    if (this.loggerConfig.filename) {
-        this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
-
-        process.on('SIGHUP', function () {
-            if (this.stream) {
-                this.stream.destroy();
-            }
-
-            this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
-        }.bind(this));
-    }
-};
-
 AnalysisBackend.prototype.create = function (analysisConfiguration, analysisDefinition, callback) {
     analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
     analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
     analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
 
-    analysisConfiguration.logger = {
-        stream: this.stream ? this.stream : process.stdout
-    };
-
     this.getAnalysesLimits(analysisConfiguration.user, function (err, limits) {
         if (err) {}
         analysisConfiguration.limits = limits || {};

lib/backends/auth.js

@@ -133,8 +133,6 @@ AuthBackend.prototype.authorize = function (req, res, callback) {
 
         if (isAuthorizedByApikey) {
             return this.pgConnection.setDBAuth(user, res.locals, 'regular', function (err) {
-                req.profiler.done('setDBAuth');
-
                 if (err) {
                     return callback(err);
                 }
@@ -150,8 +148,6 @@ AuthBackend.prototype.authorize = function (req, res, callback) {
 
             if (isAuthorizedBySigner) {
                 return this.pgConnection.setDBAuth(user, res.locals, 'master', function (err) {
-                    req.profiler.done('setDBAuth');
-
                     if (err) {
                         return callback(err);
                     }
@@ -163,8 +159,6 @@ AuthBackend.prototype.authorize = function (req, res, callback) {
             // if no signer name was given, use default api key
             if (!res.locals.signer) {
                 return this.pgConnection.setDBAuth(user, res.locals, 'default', function (err) {
-                    req.profiler.done('setDBAuth');
-
                     if (err) {
                         return callback(err);
                     }

lib/backends/metrics.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const { PubSub } = require('@google-cloud/pubsub');
+
+module.exports = class MetricsBackend {
+    constructor (options = {}) {
+        const { project_id: projectId, credentials: keyFilename, topic } = options;
+
+        this._metricsClient = new PubSub({ projectId, keyFilename });
+        this._topicName = topic;
+    }
+
+    send (event, attributes) {
+        const data = Buffer.from(event);
+        return this._metricsClient.topic(this._topicName).publish(data, attributes);
+    }
+};

lib/backends/template-maps.js

@@ -66,24 +66,20 @@ TemplateMaps.prototype._userTemplateLimit = function () {
  * @param callback - function to pass results too.
  */
 TemplateMaps.prototype._redisCmd = function (redisFunc, redisArgs, callback) {
-    this.redisPool.acquire(this.db_signatures, (err, redisClient) => {
-        if (err) {
-            return callback(err);
-        }
-
-        redisClient[redisFunc.toUpperCase()](...redisArgs, (err, data) => {
-            this.redisPool.release(this.db_signatures, redisClient);
-            if (err) {
-                return callback(err);
-            }
-            return callback(null, data);
-        });
-    });
+    this.redisPool.acquire(this.db_signatures)
+        .then((redisClient) => {
+            redisClient[redisFunc.toUpperCase()](...redisArgs, (err, data) => {
+                this.redisPool.release(this.db_signatures, redisClient)
+                    .then(() => err ? callback(err) : callback(null, data))
+                    .catch((err) => callback(err));
+            });
+        })
+        .catch((err) => callback(err));
 };
 
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_-]*$/i;
 var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
-// jshint maxcomplexity:15
+
 TemplateMaps.prototype._checkInvalidTemplate = function (template) {
     if (template.version !== '0.0.1') {
         return new Error('Unsupported template version ' + template.version);
@@ -511,12 +507,15 @@ TemplateMaps.prototype.instance = function (template, params) {
 
 // Return a fingerPrint of the object
 TemplateMaps.prototype.fingerPrint = function (template) {
-    return crypto.createHash('md5')
-        .update(JSON.stringify(template))
-        .digest('hex')
-    ;
+    return fingerPrint(template);
 };
 
+function fingerPrint (template) {
+    return crypto.createHash('md5').update(JSON.stringify(template)).digest('hex');
+}
+
+module.exports.fingerPrint = fingerPrint;
+
 module.exports.templateName = function templateName (templateId) {
     var templateIdTokens = templateId.split('@');
     var name = templateIdTokens[0];

lib/models/cdb-request.js

@@ -1,26 +1,24 @@
 'use strict';
 
-function CdbRequest () {
-    this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host ||
-            '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
-    );
-}
+module.exports = class CdbRequest {
+    constructor () {
+        // would extract "strk" from "strk.cartodb.com"
+        this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host || '^([^\\.]+)\\.');
+    }
 
-module.exports = CdbRequest;
+    userByReq (req) {
+        const host = req.headers.host || '';
 
-CdbRequest.prototype.userByReq = function (req) {
-    var host = req.headers.host || '';
-    if (req.params.user) {
-        return req.params.user;
+        if (req.params.user) {
+            return req.params.user;
+        }
+
+        const mat = host.match(this.RE_USER_FROM_HOST);
+
+        if (!mat || mat.length !== 2) {
+            throw new Error(`No username found in hostname '${host}'`);
+        }
+
+        return mat[1];
     }
-    var mat = host.match(this.RE_USER_FROM_HOST);
-    if (!mat) {
-        global.logger.error("Pattern '%s' does not match hostname '%s'", this.RE_USER_FROM_HOST, host);
-        return;
-    }
-    if (mat.length !== 2) {
-        global.logger.error("Pattern '%s' gave unexpected matches against '%s': %s", this.RE_USER_FROM_HOST, host, mat);
-        return;
-    }
-    return mat[1];
 };

lib/models/dataview/base.js

@@ -23,7 +23,7 @@ function getPGTypeName (pgType) {
 
 module.exports = class BaseDataview {
     getResult (psql, override, callback) {
-        this.sql(psql, override, (err, query, flags = null) => {
+        this.sql(psql, override, (err, query) => {
             if (err) {
                 return callback(err);
             }
@@ -36,20 +36,7 @@ module.exports = class BaseDataview {
                 result = this.format(result, override);
                 result.type = this.getType();
 
-                // Overviews logging
-                const stats = {};
-
-                if (flags && flags.usesOverviews !== undefined) {
-                    stats.usesOverviews = flags.usesOverviews;
-                } else {
-                    stats.usesOverviews = false;
-                }
-
-                if (this.getType) {
-                    stats.dataviewType = this.getType();
-                }
-
-                return callback(null, result, stats);
+                return callback(null, result);
             }, true); // use read-only transaction
         });
     }

lib/models/dataview/histograms/numeric-histogram.js

@@ -52,10 +52,18 @@ Numeric histogram:
 */
 module.exports = class NumericHistogram extends BaseHistogram {
     _buildQuery (psql, override, callback) {
+        let column = this.column;
+        let query = this.query;
+        // for date type we have to cast the column using an alias
+        // and using that alias to prevent multiple calls to the cast
+        if (this._columnType === 'date') {
+            query = `(SELECT *, ${utils.columnCastTpl({ column })} as __cdb_cast_date FROM (${this.query}) __cdb_original_query)`;
+            column = '__cdb_cast_date';
+        }
         const histogramSql = this._buildQueryTpl({
-            column: this._columnType === 'date' ? utils.columnCastTpl({ column: this.column }) : this.column,
+            column,
             isFloatColumn: this._columnType === 'float',
-            query: this.query,
+            query,
             start: this._getBinStart(override),
             end: this._getBinEnd(override),
             bins: this._getBinsCount(override),

lib/models/dataview/overviews/aggregation.js

@@ -213,7 +213,7 @@ Aggregation.prototype.sql = function (psql, override, callback) {
 
     debug(aggregationSql);
 
-    return callback(null, aggregationSql, { usesOverviews: true });
+    return callback(null, aggregationSql);
 };
 
 var aggregationFnQueryTpl = {

lib/models/dataview/overviews/formula.js

@@ -76,5 +76,5 @@ Formula.prototype.sql = function (psql, override, callback) {
 
     debug(formulaSql);
 
-    return callback(null, formulaSql, { usesOverviews: true });
+    return callback(null, formulaSql);
 };

lib/models/dataview/overviews/histogram.js

@@ -179,7 +179,7 @@ Histogram.prototype.sql = function (psql, override, callback) {
 
     var histogramSql = this._buildQuery(override);
 
-    return callback(null, histogramSql, { usesOverviews: true });
+    return callback(null, histogramSql);
 };
 
 Histogram.prototype._buildQuery = function (override) {

lib/models/layergroup-token.js

@@ -4,7 +4,7 @@
  * @param {String} token might match the following pattern: {user}@{tpl_id}@{token}:{cache_buster}
  */
 function parse (token) {
-    var signer, cacheBuster;
+    var signer, cacheBuster, templateHash;
 
     var tokenSplit = token.split(':');
 
@@ -17,7 +17,7 @@ function parse (token) {
     if (tokenSplit.length > 1) {
         signer = tokenSplit.shift();
         if (tokenSplit.length > 1) {
-            /* var template_hash = */tokenSplit.shift(); // unused
+            templateHash = tokenSplit.shift();
         }
         token = tokenSplit.shift();
     }
@@ -25,7 +25,9 @@ function parse (token) {
     return {
         token: token,
         signer: signer,
-        cacheBuster: cacheBuster
+        cacheBuster: cacheBuster,
+        templateHash: templateHash
     };
 }
+
 module.exports.parse = parse;

lib/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -15,7 +15,6 @@ function AnalysisMapConfigAdapter (analysisBackend) {
 module.exports = AnalysisMapConfigAdapter;
 
 AnalysisMapConfigAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
-    // jshint maxcomplexity:7
     var self = this;
 
     if (!shouldAdaptLayers(requestMapConfig)) {

lib/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -25,58 +25,50 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function (user, requestMapCon
 
     layers.forEach(layer => augmentLayersQueue.defer(this._augmentLayer.bind(this), user, layer, analysesResults));
 
-    augmentLayersQueue.awaitAll(function layersAugmentQueueFinish (err, results) {
+    augmentLayersQueue.awaitAll(function layersAugmentQueueFinish (err, layers) {
         if (err) {
             return callback(err);
         }
 
-        const layers = results.map(result => result.layer);
-        const overviewsAddedToMapconfig = results.some(result => result.overviewsAddedToMapconfig);
-
         if (!layers || layers.length === 0) {
             return callback(new Error('Missing layers array from layergroup config'));
         }
 
         requestMapConfig.layers = layers;
 
-        const stats = { overviewsAddedToMapconfig };
-
-        return callback(null, requestMapConfig, stats);
+        return callback(null, requestMapConfig);
     });
 };
 
 MapConfigOverviewsAdapter.prototype._augmentLayer = function (user, layer, analysesResults, callback) {
-    let overviewsAddedToMapconfig = false;
     if (layer.type !== 'mapnik' && layer.type !== 'cartodb') {
-        return callback(null, { layer, overviewsAddedToMapconfig });
+        return callback(null, layer);
     }
 
     this.overviewsMetadataBackend.getOverviewsMetadata(user, layer.options.sql, (err, metadata) => {
         if (err) {
-            return callback(err, { layer, overviewsAddedToMapconfig });
+            return callback(err);
         }
 
         if (_.isEmpty(metadata)) {
-            return callback(null, { layer, overviewsAddedToMapconfig });
+            return callback(null, layer);
         }
 
         var filters = getFilters(analysesResults, layer);
 
-        overviewsAddedToMapconfig = true;
-
         if (!filters) {
             layer.options = Object.assign({}, layer.options, getQueryRewriteData(layer, analysesResults, {
                 overviews: metadata
             }));
 
-            return callback(null, { layer, overviewsAddedToMapconfig });
+            return callback(null, layer);
         }
 
         var unfilteredQuery = getUnfilteredQuery(analysesResults, layer);
 
         this.filterStatsBackend.getFilterStats(user, unfilteredQuery, filters, function (err, stats) {
             if (err) {
-                return callback(null, { layer, overviewsAddedToMapconfig });
+                return callback(null, layer);
             }
 
             layer.options = Object.assign({}, layer.options, getQueryRewriteData(layer, analysesResults, {
@@ -84,7 +76,7 @@ MapConfigOverviewsAdapter.prototype._augmentLayer = function (user, layer, analy
                 filter_stats: stats
             }));
 
-            return callback(null, { layer, overviewsAddedToMapconfig });
+            return callback(null, layer);
         });
     });
 };

lib/models/mapconfig/provider/named-map-provider.js

@@ -4,7 +4,7 @@ const BaseMapConfigProvider = require('./base-mapconfig-adapter');
 const crypto = require('crypto');
 const dot = require('dot');
 const MapConfig = require('windshaft').model.MapConfig;
-const templateName = require('../../../backends/template-maps').templateName;
+const { templateName, fingerPrint: templateFingerPrint } = require('../../../backends/template-maps');
 
 // Configure bases for cache keys suitable for string interpolation
 const baseKey = '{{=it.dbname}}:{{=it.user}}:{{=it.templateName}}';
@@ -40,6 +40,10 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
         this.authToken = authToken;
         this.params = params;
 
+        // FIXME: why is this different than that:
+        // this.cacheBuster = params.cache_buster || 0;
+        // test: "should fail to use template from named map provider after template deletion"
+        // check named-map-provider-cache invalidation
         this.cacheBuster = Date.now();
 
         // use template after call to mapConfig
@@ -94,17 +98,16 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
 
                 const { user, rendererParams } = this;
 
-                this.mapConfigAdapter.getMapConfig(
-                    user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => {
-                        if (err) {
-                            return callback(err);
-                        }
+                this.mapConfigAdapter.getMapConfig(user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => {
+                    if (err) {
+                        return callback(err);
+                    }
 
-                        this.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
-                        this.analysesResults = context.analysesResults || [];
+                    this.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
+                    this.analysesResults = context.analysesResults || [];
 
-                        return callback(null, this.mapConfig, this.rendererParams, this.context, stats);
-                    });
+                    return callback(null, this.mapConfig, this.rendererParams, this.context, stats);
+                });
             });
         });
     }
@@ -126,6 +129,7 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
 
                 context.analysisConfiguration = {
                     user: this.user,
+                    logger: this.logger,
                     db: {
                         host: rendererParams.dbhost,
                         port: rendererParams.dbport,
@@ -254,6 +258,14 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
     getTemplateName () {
         return this.templateName;
     }
+
+    getTemplateHash () {
+        if (!this.template) {
+            throw new Error('Missing template, call "getTemplate()" method first');
+        }
+
+        return templateFingerPrint(this.template);
+    }
 };
 
 function createConfigHash (config) {

lib/server-info-controller.js

@@ -5,10 +5,9 @@ var HealthCheck = require('./monitoring/health-check');
 var WELCOME_MSG = 'This is the CartoDB Maps API, ' +
     'see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html';
 
-function ServerInfoController (versions) {
+function ServerInfoController () {
     this.healthConfig = global.environment.health || {};
     this.healthCheck = new HealthCheck(global.environment.disabled_file);
-    this.versions = versions || {};
 }
 
 module.exports = ServerInfoController;
@@ -16,7 +15,6 @@ module.exports = ServerInfoController;
 ServerInfoController.prototype.route = function (monitorRouter) {
     monitorRouter.get('/health', this.health.bind(this));
     monitorRouter.get('/', this.welcome.bind(this));
-    monitorRouter.get('/version', this.version.bind(this));
 };
 
 ServerInfoController.prototype.welcome = function (req, res) {

lib/server-options.js

@@ -3,6 +3,7 @@
 const fqdn = require('@carto/fqdn-sync');
 var _ = require('underscore');
 var OverviewsQueryRewriter = require('./utils/overviews-query-rewriter');
+const Logger = require('./utils/logger');
 
 var rendererConfig = _.defaults(global.environment.renderer || {}, {
     cache_ttl: 60000, // milliseconds
@@ -52,9 +53,6 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
         endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
         hostHeaderTemplate: '{{=it.username}}.localhost.lan'
     },
-    logger: {
-        filename: undefined
-    },
     limits: {}
 });
 
@@ -102,8 +100,8 @@ module.exports = {
     },
     statsd: global.environment.statsd,
     renderCache: {
-        ttl: rendererConfig.cache_ttl,
-        statsInterval: rendererConfig.statsInterval
+        ttl: rendererConfig.cache_ttl || 60000,
+        statsInterval: rendererConfig.statsInterval || 60000
     },
     renderer: {
         mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt),
@@ -118,9 +116,6 @@ module.exports = {
             endpoint: analysisConfig.batch.endpoint,
             hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate
         },
-        logger: {
-            filename: analysisConfig.logger.filename
-        },
         limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
@@ -133,6 +128,7 @@ module.exports = {
     varnish_purge_enabled: global.environment.varnish.purge_enabled,
     fastly: global.environment.fastly || {},
     cache_enabled: global.environment.cache_enabled,
-    log_format: global.environment.log_format,
-    useProfiler: global.environment.useProfiler
+    useProfiler: global.environment.useProfiler,
+    pubSubMetrics: Object.assign({ enabled: false }, global.environment.pubSubMetrics),
+    logger: new Logger()
 };

lib/server.js

@@ -1,16 +1,9 @@
 'use strict';
 
-const _ = require('underscore');
-const semver = require('semver');
 const express = require('express');
-const windshaft = require('windshaft');
-const { mapnik } = windshaft;
-
 const jsonReplacer = require('./utils/json-replacer');
-
 const ApiRouter = require('./api/api-router');
 const ServerInfoController = require('./server-info-controller');
-
 const StatsClient = require('./stats/client');
 
 module.exports = function createServer (serverOptions) {
@@ -21,10 +14,6 @@ module.exports = function createServer (serverOptions) {
     // Make stats client globally accessible
     global.statsClient = StatsClient.getInstance(serverOptions.statsd);
 
-    serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);
-
-    bootstrapFonts(serverOptions);
-
     const app = express();
 
     app.enable('jsonp callback');
@@ -32,72 +21,13 @@ module.exports = function createServer (serverOptions) {
     app.disable('etag');
     app.set('json replacer', jsonReplacer());
 
+    // FIXME: do not pass 'global.environment' as 'serverOptions' should keep defaults from 'global.environment'
     const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment });
 
-    // TODO: remove it before releasing next major version
-    if (!Array.isArray(serverOptions.routes.api)) {
-        serverOptions.routes.api = [serverOptions.routes.api];
-    }
-
     apiRouter.route(app, serverOptions.routes.api);
 
-    const versions = getAndValidateVersions(serverOptions);
-
-    const serverInfoController = new ServerInfoController(versions);
+    const serverInfoController = new ServerInfoController();
     serverInfoController.route(app);
 
     return app;
 };
-
-function bootstrapFonts (opts) {
-    // Set carto renderer configuration for MMLStore
-    opts.grainstore.carto_env = opts.grainstore.carto_env || {};
-    var cenv = opts.grainstore.carto_env;
-    cenv.validation_data = cenv.validation_data || {};
-    if (!cenv.validation_data.fonts) {
-        mapnik.register_system_fonts();
-        mapnik.register_default_fonts();
-        cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
-    }
-}
-
-function mapnikVersion (opts) {
-    return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
-}
-
-function getAndValidateVersions (options) {
-    var warn = console.warn.bind(console); // jshint ignore:line
-
-    var packageDefinition = require('../package.json');
-
-    var declaredDependencies = packageDefinition.dependencies || {};
-    var installedDependenciesVersions = {
-        camshaft: require('camshaft').version,
-        grainstore: windshaft.grainstore.version(),
-        mapnik: windshaft.mapnik.versions.mapnik,
-        node_mapnik: windshaft.mapnik.version,
-        'turbo-carto': require('turbo-carto').version,
-        windshaft: windshaft.version,
-        windshaft_cartodb: packageDefinition.version
-    };
-
-    if (process.env.NODE_ENV !== 'test') {
-        var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
-        dependenciesToValidate.forEach(function (depName) {
-            var declaredDependencyVersion = declaredDependencies[depName];
-            var installedDependencyVersion = installedDependenciesVersions[depName];
-            if (!semver.satisfies(installedDependencyVersion, declaredDependencyVersion)) {
-                warn(`Dependency="${depName}" installed version="${installedDependencyVersion}" does ` +
-                     `not match declared version="${declaredDependencyVersion}". Check your installation.`);
-            }
-        });
-
-        // Be nice and warn if configured mapnik version is != installed mapnik version
-        if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
-            warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' +
-                ' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
-        }
-    }
-
-    return installedDependenciesVersions;
-}

lib/stats/profiler-proxy.js

@@ -52,4 +52,8 @@ ProfilerProxy.prototype.toJSONString = function () {
     return this.profile ? this.profiler.toJSONString() : '{}';
 };
 
+ProfilerProxy.prototype.toJSON = function () {
+    return this.profile ? JSON.parse(this.profiler.toJSONString()) : {};
+};
+
 module.exports = ProfilerProxy;

lib/stats/reporter/named-map-provider-cache.js

@@ -2,10 +2,10 @@
 
 const statKeyTemplate = ctx => `windshaft.named-map-provider-cache.${ctx.metric}`;
 
-module.exports = class NamedMapProviderReporter {
+module.exports = class NamedMapProviderCacheReporter {
     constructor ({ namedMapProviderCache, intervalInMilliseconds } = {}) {
         this.namedMapProviderCache = namedMapProviderCache;
-        this.intervalInMilliseconds = intervalInMilliseconds;
+        this.intervalInMilliseconds = intervalInMilliseconds || 60000;
         this.intervalId = null;
     }
 
@@ -14,8 +14,8 @@ module.exports = class NamedMapProviderReporter {
         const { statsClient: stats } = global;
 
         this.intervalId = setInterval(() => {
-            stats.gauge(statKeyTemplate({ metric: 'named-map.count' }), cache.length);
             const providers = cache.dump();
+            stats.gauge(statKeyTemplate({ metric: 'named-map.count' }), providers.length);
 
             const namedMapInstantiations = providers.reduce((acc, { v: providers }) => {
                 acc += Object.keys(providers).length;

lib/stats/reporter/renderer.js

@@ -6,8 +6,6 @@
 //      - the mapnik-pool status: count, unused and waiting
 //      - the internally cached objects: png and grid
 
-var _ = require('underscore');
-
 function RendererStatsReporter (rendererCache, statsInterval) {
     this.rendererCache = rendererCache;
     this.statsInterval = statsInterval || 6e4;
@@ -17,50 +15,16 @@ function RendererStatsReporter (rendererCache, statsInterval) {
 module.exports = RendererStatsReporter;
 
 RendererStatsReporter.prototype.start = function () {
-    var self = this;
-    this.renderersStatsIntervalId = setInterval(function () {
-        var rendererCacheEntries = self.rendererCache.renderers;
+    this.renderersStatsIntervalId = setInterval(() => {
+        const rendererStats = this.rendererCache.getStats();
 
-        if (!rendererCacheEntries) {
-            return null;
-        }
-
-        global.statsClient.gauge('windshaft.rendercache.count', _.keys(rendererCacheEntries).length);
-
-        var renderersStats = _.reduce(rendererCacheEntries, function (_rendererStats, cacheEntry) {
-            var stats = cacheEntry.renderer && cacheEntry.renderer.getStats && cacheEntry.renderer.getStats();
-            if (!stats) {
-                return _rendererStats;
-            }
-
-            _rendererStats.pool.count += stats.pool.count;
-            _rendererStats.pool.unused += stats.pool.unused;
-            _rendererStats.pool.waiting += stats.pool.waiting;
-
-            _rendererStats.cache.grid += stats.cache.grid;
-            _rendererStats.cache.png += stats.cache.png;
-
-            return _rendererStats;
-        },
-        {
-            pool: {
-                count: 0,
-                unused: 0,
-                waiting: 0
-            },
-            cache: {
-                png: 0,
-                grid: 0
+        for (const [stat, value] of rendererStats) {
+            if (stat.startsWith('rendercache')) {
+                global.statsClient.gauge(`windshaft.${stat}`, value);
+            } else {
+                global.statsClient.gauge(`windshaft.mapnik-${stat}`, value);
             }
         }
-        );
-
-        global.statsClient.gauge('windshaft.mapnik-cache.png', renderersStats.cache.png);
-        global.statsClient.gauge('windshaft.mapnik-cache.grid', renderersStats.cache.grid);
-
-        global.statsClient.gauge('windshaft.mapnik-pool.count', renderersStats.pool.count);
-        global.statsClient.gauge('windshaft.mapnik-pool.unused', renderersStats.pool.unused);
-        global.statsClient.gauge('windshaft.mapnik-pool.waiting', renderersStats.pool.waiting);
     }, this.statsInterval);
 
     this.rendererCache.on('err', rendererCacheErrorListener);

lib/utils/common-headers.js

@@ -0,0 +1,125 @@
+'use strict';
+
+module.exports = function setCommonHeaders (req, res, callback) {
+    const { logger } = res.locals;
+
+    res.set('X-Request-Id', logger.bindings().request_id);
+
+    // TODO: x-layergroupid header??
+
+    const user = getUser({ res });
+
+    if (user) {
+        res.set('Carto-User', user);
+    }
+
+    const userId = getUserId({ res });
+
+    if (userId) {
+        res.set('Carto-User-Id', `${userId}`);
+    }
+
+    const mapId = getMapId({ res });
+
+    if (mapId) {
+        res.set('Carto-Map-Id', mapId);
+    }
+
+    const cacheBuster = getCacheBuster({ res });
+
+    if (cacheBuster) {
+        res.set('Carto-Cache-Buster', cacheBuster);
+    }
+
+    const templateHash = getTemplateHash({ res });
+
+    if (templateHash) {
+        res.set('Carto-Template-Hash', templateHash);
+    }
+
+    getStatTag({ res }, (err, statTag) => {
+        if (err) {
+            logger.warn({ exception: err }, 'Error generating Stat Tag header');
+        }
+
+        if (statTag) {
+            res.set('Carto-Stat-Tag', statTag);
+        }
+
+        callback();
+    });
+};
+
+function getUser ({ res }) {
+    if (res.locals.user) {
+        return res.locals.user;
+    }
+}
+
+function getUserId ({ res }) {
+    if (res.locals.userId) {
+        return res.locals.userId;
+    }
+}
+
+function getMapId ({ res }) {
+    if (res.locals.token) {
+        return res.locals.token;
+    }
+
+    if (res.locals.mapConfig) {
+        return res.locals.mapConfig.id();
+    }
+
+    if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) {
+        return res.locals.mapConfigProvider.mapConfig.id();
+    }
+}
+
+function getCacheBuster ({ res }) {
+    if (res.locals.cache_buster !== undefined) {
+        return `${res.locals.cache_buster}`;
+    }
+
+    if (res.locals.mapConfigProvider) {
+        return `${res.locals.mapConfigProvider.getCacheBuster()}`;
+    }
+}
+
+function getTemplateHash ({ res }) {
+    const { logger } = res.locals;
+
+    if (res.locals.templateHash) {
+        return res.locals.templateHash;
+    }
+
+    if (res.locals.mapConfigProvider && typeof res.locals.mapConfigProvider.getTemplateHash === 'function') {
+        let templateHash;
+
+        try {
+            templateHash = res.locals.mapConfigProvider.getTemplateHash().substring(0, 8);
+        } catch (err) {
+            logger.warn({ exception: err }, 'Error generating Stat Tag header');
+        }
+
+        return templateHash;
+    }
+}
+
+function getStatTag ({ res }, callback) {
+    if (res.locals.mapConfig) {
+        return callback(null, res.locals.mapConfig.obj().stat_tag);
+    }
+
+    if (!res.locals.mapConfigProvider) {
+        return callback();
+    }
+
+    res.locals.mapConfigProvider.getMapConfig((err, mapConfig) => {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, mapConfig.obj().stat_tag);
+    });
+}

lib/utils/logger.js

@@ -0,0 +1,73 @@
+'use strict';
+
+const pino = require('pino');
+const { req: requestSerializer, res: responseSerializer, err, wrapErrorSerializer } = pino.stdSerializers;
+const DEV_ENVS = ['test', 'development'];
+
+module.exports = class Logger {
+    constructor () {
+        const { LOG_LEVEL, NODE_ENV } = process.env;
+        const logLevelFromNodeEnv = NODE_ENV === 'test' ? 'fatal' : 'info';
+        const errorSerializer = DEV_ENVS.includes(NODE_ENV) ? err : wrapErrorSerializer(err => {
+            if (Object.prototype.hasOwnProperty.call(err, 'stack')) {
+                err.stack = err.stack.split('\n').slice(0, 3).join('\n');
+            }
+            return err;
+        });
+        const options = {
+            base: null, // Do not bind hostname, pid and friends by default
+            level: LOG_LEVEL || logLevelFromNodeEnv,
+            formatters: {
+                level (label) {
+                    if (label === 'warn') {
+                        return { levelname: 'warning' };
+                    }
+
+                    return { levelname: label };
+                }
+            },
+            messageKey: 'event_message',
+            timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
+            serializers: {
+                client_request: requestSerializer,
+                server_response: responseSerializer,
+                exception: errorSerializer
+            }
+        };
+        const dest = pino.destination({ sync: false }); // stdout
+
+        this._logger = pino(options, dest);
+    }
+
+    trace (...args) {
+        this._logger.trace(...args);
+    }
+
+    debug (...args) {
+        this._logger.debug(...args);
+    }
+
+    info (...args) {
+        this._logger.info(...args);
+    }
+
+    warn (...args) {
+        this._logger.warn(...args);
+    }
+
+    error (...args) {
+        this._logger.error(...args);
+    }
+
+    fatal (...args) {
+        this._logger.fatal(...args);
+    }
+
+    child (...args) {
+        return this._logger.child(...args);
+    }
+
+    finish (callback) {
+        return pino.final(this._logger, callback);
+    }
+};

lib/utils/on-tile-error-strategy.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const path = require('path');
+const timeoutErrorTilePath = path.join(__dirname, '/../../assets/render-timeout-fallback.png');
+const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, { encoding: null });
+
+module.exports = function getOnTileErrorStrategy ({ enabled }) {
+    let onTileErrorStrategy;
+
+    if (enabled !== false) {
+        onTileErrorStrategy = async function onTileErrorStrategy$TimeoutTile (err, format) {
+            function isRenderTimeoutError (err) {
+                return err.message === 'Render timed out';
+            }
+
+            function isDatasourceTimeoutError (err) {
+                return err.message && err.message.match(/canceling statement due to statement timeout/i);
+            }
+
+            function isTimeoutError (err) {
+                return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+            }
+
+            function isRasterFormat (format) {
+                return format === 'png' || format === 'jpg';
+            }
+
+            if (isTimeoutError(err) && isRasterFormat(format)) {
+                return { buffer: timeoutErrorTile, headers: { 'Content-Type': 'image/png' }, stats: {} };
+            } else {
+                throw err;
+            }
+        };
+    }
+
+    return onTileErrorStrategy;
+};

metro/config.json

@@ -0,0 +1,94 @@
+{
+    "metrics": {
+        "port": 9145,
+        "definitions": [
+            {
+                "type": "counter",
+                "options": {
+                    "name": "maps_api_requests_total",
+                    "help": "MAPS API requests total"
+                },
+                "valuePath": "server_response.statusCode",
+                "shouldMeasure": "({ value }) => Number.isFinite(value)",
+                "measure": "({ metric }) => metric.inc()"
+            },
+            {
+                "type": "counter",
+                "options": {
+                    "name": "maps_api_requests_ok_total",
+                    "help": "MAPS API requests ok total"
+                },
+                "valuePath": "server_response.statusCode",
+                "shouldMeasure": "({ value }) => value >= 200 && value < 400",
+                "measure": "({ metric }) => metric.inc()"
+            },
+            {
+                "type": "counter",
+                "options": {
+                    "name": "maps_api_requests_errors_total",
+                    "help": "MAPS API requests errors total"
+                },
+                "valuePath": "server_response.statusCode",
+                "shouldMeasure": "({ value }) => value >= 400",
+                "measure": "({ metric }) => metric.inc()"
+            },
+            {
+                "type": "histogram",
+                "options": {
+                    "name": "maps_api_response_time_total",
+                    "help": "MAPS API response time total"
+                },
+                "valuePath": "stats.response",
+                "shouldMeasure": "({ value }) => Number.isFinite(value)",
+                "measure": "({ metric, value }) => metric.observe(value)"
+            },
+            {
+                "type": "counter",
+                "options": {
+                    "name": "maps_api_requests",
+                    "help": "MAPS API requests per user",
+                    "labelNames": ["user", "http_code"]
+                },
+                "labelPaths": ["cdb-user", "server_response.statusCode"],
+                "shouldMeasure": "({ labels }) => labels.every((label) => label !== undefined)",
+                "measure": "({ metric, labels }) => metric.labels(...labels).inc()"
+            },
+            {
+                "type": "counter",
+                "options": {
+                    "name": "maps_api_requests_ok",
+                    "help": "MAPS API requests per user with success HTTP code",
+                    "labelNames": ["user", "http_code"]
+                },
+                "labelPaths": ["cdb-user", "server_response.statusCode"],
+                "valuePath": "server_response.statusCode",
+                "shouldMeasure": "({ labels, value }) => labels.every((label) => label !== undefined) && value >= 200 && value < 400",
+                "measure": "({ metric, labels }) => metric.labels(...labels).inc()"
+            },
+            {
+                "type": "counter",
+                "options": {
+                    "name": "maps_api_requests_errors",
+                    "help": "MAPS API requests per user with error HTTP code",
+                    "labelNames": ["user", "http_code"]
+                },
+                "labelPaths": ["cdb-user", "server_response.statusCode"],
+                "valuePath": "server_response.statusCode",
+                "shouldMeasure": "({ labels, value }) => labels.every((label) => label !== undefined) && value >= 400",
+                "measure": "({ metric, labels }) => metric.labels(...labels).inc()"
+            },
+            {
+                "type": "histogram",
+                "options": {
+                    "name": "maps_api_response_time",
+                    "help": "MAPS API response time total",
+                    "labelNames": ["user"]
+                },
+                "labelPaths": ["cdb-user"],
+                "valuePath": "stats.response",
+                "shouldMeasure": "({ labels, value }) => labels.every((label) => label !== undefined) && Number.isFinite(value)",
+                "measure": "({ metric, labels, value }) => metric.labels(...labels).observe(value)"
+            }
+        ]
+    }
+}

metro/index.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const metro = require('./metro');
+const path = require('path');
+const fs = require('fs');
+
+const { CONFIG_PATH = path.resolve(__dirname, './config.json') } = process.env;
+const existsConfigFile = fs.existsSync(CONFIG_PATH);
+
+if (!existsConfigFile) {
+    exit(4)(new Error(`Wrong path for CONFIG_PATH env variable: ${CONFIG_PATH} no such file`));
+}
+
+let config;
+
+if (existsConfigFile) {
+    config = fs.readFileSync(CONFIG_PATH);
+    try {
+        config = JSON.parse(config);
+    } catch (e) {
+        exit(5)(new Error('Wrong config format: invalid JSON'));
+    }
+}
+
+metro({ metrics: config && config.metrics })
+    .then(exit(0))
+    .catch(exit(1));
+
+process.on('uncaughtException', exit(2));
+process.on('unhandledRejection', exit(3));
+
+function exit (code = 1) {
+    return function (err) {
+        if (err) {
+            console.error(err);
+        }
+
+        process.exit(code);
+    };
+}

metro/metrics-collector.js

@@ -0,0 +1,102 @@
+'use strict';
+
+const http = require('http');
+const { Counter, Histogram, register } = require('prom-client');
+const flatten = require('flat');
+const { Transform, PassThrough } = require('stream');
+const DEV_ENVS = ['test', 'development'];
+
+const factory = {
+    counter: Counter,
+    histogram: Histogram
+};
+
+module.exports = class MetricsCollector {
+    constructor ({ port = 0, definitions } = {}) {
+        this._port = port;
+        this._definitions = definitions;
+        this._server = null;
+        this._stream = createTransformStream(this._definitions);
+    }
+
+    get stream () {
+        return this._stream;
+    }
+
+    start () {
+        return new Promise((resolve, reject) => {
+            this._server = http.createServer((req, res) => {
+                res.writeHead(200, { 'Content-Type': register.contentType });
+                res.end(register.metrics());
+            });
+
+            this._server.once('error', err => reject(err));
+            this._server.once('listening', () => resolve());
+            this._server.listen(this._port);
+        });
+    }
+
+    stop () {
+        return new Promise((resolve) => {
+            register.clear();
+            if (!this._server) {
+                return resolve();
+            }
+
+            this._server.once('close', () => {
+                this._server = null;
+                resolve();
+            });
+
+            this._server.close();
+        });
+    };
+};
+
+function createTransformStream (definitions) {
+    if (typeof definitions !== 'object') {
+        return new PassThrough();
+    }
+
+    const metrics = [];
+
+    for (const { type, options, valuePath, labelPaths, shouldMeasure, measure } of definitions) {
+        metrics.push({
+            instance: new factory[type](options),
+            valuePath,
+            labelPaths,
+            shouldMeasure: eval(shouldMeasure), // eslint-disable-line no-eval
+            measure: eval(measure) // eslint-disable-line no-eval
+        });
+    }
+
+    return new Transform({
+        transform (chunk, enc, callback) {
+            let entry;
+
+            try {
+                entry = JSON.parse(chunk);
+            } catch (e) {
+                if (DEV_ENVS.includes(process.env.NODE_ENV)) {
+                    this.push(chunk + '\n');
+                }
+                return callback();
+            }
+
+            const flatEntry = flatten(entry);
+
+            for (const metric of metrics) {
+                const value = flatEntry[metric.valuePath];
+                const labels = Array.isArray(metric.labelPaths) && metric.labelPaths.map(path => flatEntry[path]);
+
+                if (metric.shouldMeasure({ labels, value })) {
+                    metric.measure({ metric: metric.instance, labels, value });
+                }
+            }
+
+            this.push(`${JSON.stringify(entry)}\n`);
+
+            return callback();
+        }
+    });
+}

metro/metro.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const util = require('util');
+const stream = require('stream');
+const pipeline = util.promisify(stream.pipeline);
+const split = require('split2');
+const MetricsCollector = require('./metrics-collector');
+
+module.exports = async function metro ({ input = process.stdin, output = process.stdout, metrics = {} } = {}) {
+    const metricsCollector = new MetricsCollector(metrics);
+    const { stream: metricsStream } = metricsCollector;
+
+    try {
+        await metricsCollector.start();
+        await pipeline(input, split(), metricsStream, output);
+    } finally {
+        await metricsCollector.stop();
+    }
+};

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "8.1.1",
+  "version": "9.0.0",
   "description": "CARTO Maps API tiler",
   "keywords": [
     "carto",
@@ -35,44 +35,51 @@
   "main": "app.js",
   "dependencies": {
     "@carto/fqdn-sync": "0.2.2",
+    "@google-cloud/pubsub": "1.5.0",
+    "assign-deep": "^1.0.1",
     "basic-auth": "2.0.0",
     "body-parser": "1.18.3",
-    "camshaft": "^0.65.2",
+    "camshaft": "^0.67.2",
     "cartodb-psql": "0.14.0",
     "cartodb-query-tables": "^0.7.0",
-    "cartodb-redis": "2.1.0",
+    "cartodb-redis": "^3.0.0",
     "debug": "3.1.0",
     "dot": "1.1.2",
     "express": "4.16.3",
     "fastly-purge": "1.0.1",
-    "gc-stats": "1.2.1",
+    "gc-stats": "^1.4.0",
     "glob": "7.1.2",
-    "log4js": "github:cartodb/log4js-node#cdb",
     "lru-cache": "4.1.3",
     "lzma": "2.3.2",
     "node-statsd": "0.1.1",
-    "on-headers": "1.0.1",
+    "pino": "^6.3.1",
+    "prom-client": "^12.0.0",
     "queue-async": "1.1.0",
-    "redis-mpool": "0.7.0",
+    "redis-mpool": "^0.8.0",
     "request": "2.87.0",
     "semver": "5.5.0",
+    "split2": "^3.1.1",
     "step-profiler": "0.3.0",
     "turbo-carto": "0.21.2",
     "underscore": "1.6.0",
-    "windshaft": "5.6.4",
-    "yargs": "11.1.0"
+    "uuid": "^8.1.0",
+    "windshaft": "^7.0.1",
+    "yargs": "^15.3.1"
   },
   "devDependencies": {
+    "@carto/mapnik": "^3.6.2-carto.16",
     "eslint": "^6.5.1",
     "eslint-config-standard": "^14.1.0",
     "eslint-plugin-import": "^2.18.2",
     "eslint-plugin-node": "^10.0.0",
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
-    "mocha": "5.2.0",
+    "mocha": "^7.2.0",
     "moment": "2.22.1",
     "nock": "9.2.6",
+    "nodemon": "^2.0.6",
     "nyc": "^14.1.1",
+    "pino-pretty": "^4.0.0",
     "redis": "2.8.0",
     "step": "1.0.0",
     "strftime": "0.10.0"
@@ -88,11 +95,10 @@
     "posttest": "npm run test:teardown",
     "test:teardown": "NODE_ENV=test node test teardown",
     "cover": "nyc --reporter=lcov npm test",
-    "test:docker": "docker run -e \"NODEJS_VERSION=$NODE_VERSION\" -v `pwd`:/srv $DOCKER_IMAGE bash docker/scripts/test-setup.sh && docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v",
-    "docker:bash": "docker run -it -v `pwd`:/srv $DOCKER_IMAGE bash"
+    "dev": "NODE_ENV=development nodemon app.js"
   },
   "engines": {
-    "node": "^10.15.1",
-    "npm": "^6.4.1"
+    "node": "^12.16.3",
+    "npm": "^6.14.4"
   }
 }

test/acceptance/analysis/named-maps-test.js

@@ -4,7 +4,7 @@ var assert = require('../../support/assert');
 
 var helper = require('../../support/test-helper');
 
-var CartodbWindshaft = require('../../../lib/server');
+const createServer = require('../../../lib/server');
 var serverOptions = require('../../../lib/server-options');
 var TestClient = require('../../support/test-client');
 
@@ -14,7 +14,7 @@ describe('named-maps analysis', function () {
     var server;
 
     before(function () {
-        server = new CartodbWindshaft(serverOptions);
+        server = createServer(serverOptions);
     });
 
     var IMAGE_TOLERANCE_PER_MIL = 20;

test/acceptance/auth/authorization-basic-use-cases-test.js

@@ -2,7 +2,7 @@
 
 const assert = require('../../support/assert');
 const testHelper = require('../../support/test-helper');
-const CartodbWindshaft = require('../../../lib/server');
+const createServer = require('../../../lib/server');
 const serverOptions = require('../../../lib/server-options');
 var LayergroupToken = require('../../../lib/models/layergroup-token');
 
@@ -47,7 +47,7 @@ describe('Basic authorization use cases', function () {
     var server;
 
     before(function () {
-        server = new CartodbWindshaft(serverOptions);
+        server = createServer(serverOptions);
     });
 
     beforeEach(function () {

test/acceptance/auth/authorization-test.js

@@ -4,7 +4,7 @@ require('../../support/test-helper');
 
 const assert = require('../../support/assert');
 const TestClient = require('../../support/test-client');
-const mapnik = require('windshaft').mapnik;
+const mapnik = require('@carto/mapnik');
 
 const PERMISSION_DENIED_RESPONSE = {
     status: 403,

test/acceptance/buffer-size-format-test.js

@@ -6,7 +6,7 @@ var fs = require('fs');
 var assert = require('../support/assert');
 var TestClient = require('../support/test-client');
 var serverOptions = require('../../lib/server-options');
-var mapnik = require('windshaft').mapnik;
+const mapnik = require('@carto/mapnik');
 var IMAGE_TOLERANCE_PER_MIL = 5;
 
 var CARTOCSS_LABELS = [

test/acceptance/cache/cache-headers-test.js

@@ -5,7 +5,7 @@ var testHelper = require('../../support/test-helper');
 var assert = require('../../support/assert');
 var qs = require('querystring');
 
-var CartodbWindshaft = require('../../../lib/server');
+const createServer = require('../../../lib/server');
 var serverOptions = require('../../../lib/server-options');
 
 var LayergroupToken = require('../../../lib/models/layergroup-token');
@@ -14,7 +14,7 @@ describe('get requests with cache headers', function () {
     var server;
 
     before(function () {
-        server = new CartodbWindshaft(serverOptions);
+        server = createServer(serverOptions);
         server.setMaxListeners(0);
     });
 
@@ -302,15 +302,6 @@ describe('get requests with cache headers', function () {
             );
         });
 
-        it('/version', function (done) {
-            assert.response(
-                server,
-                getRequest('/version'),
-                statusOkResponse,
-                noCacheHeaders(done)
-            );
-        });
-
         it('/health', function (done) {
             assert.response(
                 server,

test/acceptance/cache/surrogate-keys-invalidation-test.js

@@ -7,7 +7,7 @@ var step = require('step');
 var FastlyPurge = require('fastly-purge');
 var _ = require('underscore');
 var NamedMapsCacheEntry = require('../../../lib/cache/model/named-maps-entry');
-var CartodbWindshaft = require('../../../lib/server');
+const createServer = require('../../../lib/server');
 var nock = require('nock');
 
 describe('templates surrogate keys', function () {
@@ -33,7 +33,7 @@ describe('templates surrogate keys', function () {
     var server;
 
     before(function () {
-        server = new CartodbWindshaft(serverOptions);
+        server = createServer(serverOptions);
         nock.disableNetConnect();
         nock.enableNetConnect(/(127.0.0.1|cartocdn.com)/);
     });

test/acceptance/cluster-test.js

@@ -16,6 +16,7 @@ const POINTS_SQL_1 = `
             ELSE 'odd'
         END AS type
     from generate_series(-3, 3) x
+    order by cartodb_id asc
 `;
 
 const defaultLayers = [{
@@ -62,14 +63,14 @@ describe('cluster', function () {
                 }
 
                 assert.deepStrictEqual(body, {
-                    errors: ['Map f697fb370c6479559ae2f66d684e8227 has no aggregation defined for layer 0'],
+                    errors: ['Map f105928729b4d3f67ae3578a163778c9 has no aggregation defined for layer 0'],
                     errors_with_context: [
                         {
                             layer: {
                                 index: '0',
                                 type: 'cartodb'
                             },
-                            message: 'Map f697fb370c6479559ae2f66d684e8227 has no aggregation defined for layer 0',
+                            message: 'Map f105928729b4d3f67ae3578a163778c9 has no aggregation defined for layer 0',
                             subtype: 'aggregation',
                             type: 'layer'
                         }
@@ -104,14 +105,14 @@ describe('cluster', function () {
                 }
 
                 assert.deepStrictEqual(body, {
-                    errors: ['Map 7521bcd1029c401289dd651ce91d5d9d has no aggregation defined for layer 0'],
+                    errors: ['Map 9d19316644f831332b44de7b3158aa77 has no aggregation defined for layer 0'],
                     errors_with_context: [
                         {
                             layer: {
                                 index: '0',
                                 type: 'cartodb'
                             },
-                            message: 'Map 7521bcd1029c401289dd651ce91d5d9d has no aggregation defined for layer 0',
+                            message: 'Map 9d19316644f831332b44de7b3158aa77 has no aggregation defined for layer 0',
                             subtype: 'aggregation',
                             type: 'layer'
                         }

test/acceptance/dataviews/histogram-test.js

@@ -325,6 +325,15 @@ describe('histogram-dataview for date column type', function () {
                     column: 'd',
                     aggregation: 'minute'
                 }
+            },
+            date_histogram_no_agg: {
+                options: {
+                    column: 'd'
+                },
+                source: {
+                    id: 'minute-histogram-source-tz'
+                },
+                type: 'histogram'
             }
         },
         [
@@ -1232,6 +1241,21 @@ describe('histogram-dataview for date column type', function () {
             done();
         });
     });
+
+    it('should work with dates without aggregation', function (done) {
+        var params = {
+            start: 1171583400,
+            end: 1171584600
+        };
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('date_histogram_no_agg', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.strictEqual(dataview.type, 'histogram');
+            assert.strictEqual(dataview.bins.length, 6);
+            assert.strictEqual(dataview.bins_count, 6);
+            done();
+        });
+    });
 });
 
 describe('histogram-dataview: special float valuer', function () {

test/acceptance/dataviews/overviews-test.js

@@ -55,7 +55,6 @@ describe('dataviews using tables without overviews', function () {
                 return done(err);
             }
             assert.deepStrictEqual(formulaResult, { operation: 'count', result: 7313, nulls: 0, type: 'formula' });
-            assert(getUsesOverviewsFromHeaders(headers) === false); // Overviews logging
 
             testClient.drain(done);
         });
@@ -269,8 +268,6 @@ describe('dataviews using tables with overviews', function () {
                 nulls: 0,
                 type: 'formula'
             });
-            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
-            assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging
 
             testClient.drain(done);
         });
@@ -290,8 +287,6 @@ describe('dataviews using tables with overviews', function () {
                 infinities: 0,
                 nans: 0
             });
-            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
-            assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging
 
             testClient.drain(done);
         });
@@ -311,8 +306,6 @@ describe('dataviews using tables with overviews', function () {
                 infinities: 0,
                 nans: 0
             });
-            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
-            assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging
 
             testClient.drain(done);
         });
@@ -387,8 +380,6 @@ describe('dataviews using tables with overviews', function () {
             assert.ok(histogram);
             assert.strictEqual(histogram.type, 'histogram');
             assert.ok(Array.isArray(histogram.bins));
-            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
-            assert(getDataviewTypeFromHeaders(headers) === 'histogram'); // Overviews logging
 
             testClient.drain(done);
         });
@@ -480,7 +471,7 @@ describe('dataviews using tables with overviews', function () {
                         nans: 0,
                         type: 'formula'
                     });
-                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
+
                     testClient.drain(done);
                 });
             });
@@ -499,7 +490,6 @@ describe('dataviews using tables with overviews', function () {
                         nans: 0,
                         type: 'formula'
                     });
-                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
 
                     testClient.drain(done);
                 });
@@ -519,7 +509,6 @@ describe('dataviews using tables with overviews', function () {
                         nulls: 0,
                         type: 'formula'
                     });
-                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
 
                     testClient.drain(done);
                 });
@@ -611,9 +600,6 @@ describe('dataviews using tables with overviews', function () {
                         type: 'aggregation'
                     });
 
-                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
-                    assert(getDataviewTypeFromHeaders(headers) === 'aggregation'); // Overviews logging
-
                     testClient.drain(done);
                 });
             });
@@ -830,11 +816,3 @@ describe('dataviews using tables with overviews', function () {
         });
     });
 });
-
-function getUsesOverviewsFromHeaders (headers) {
-    return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).usesOverviews;
-}
-
-function getDataviewTypeFromHeaders (headers) {
-    return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).dataviewType;
-}

test/acceptance/dynamic-styling-named-maps-test.js

@@ -4,14 +4,14 @@ var assert = require('../support/assert');
 var step = require('step');
 var LayergroupToken = require('../../lib/models/layergroup-token');
 var testHelper = require('../support/test-helper');
-var CartodbWindshaft = require('../../lib/server');
+var createServer = require('../../lib/server');
 var serverOptions = require('../../lib/server-options');
 
 describe('dynamic styling for named maps', function () {
     var server;
 
     before(function () {
-        server = new CartodbWindshaft(serverOptions);
+        server = createServer(serverOptions);
     });
 
     var keysToDelete;

test/acceptance/error-middleware-test.js

@@ -1,44 +0,0 @@
-'use strict';
-
-const assert = require('../support/assert');
-const TestClient = require('../support/test-client');
-
-describe('error middleware', function () {
-    it('should returns a errors header', function (done) {
-        const mapConfig = {
-            version: '1.6.0',
-            layers: [{
-                type: 'mapnik',
-                options: {}
-            }]
-        };
-
-        const errorHeader = {
-            mainError: {
-                statusCode: 400,
-                message: 'Missing cartocss for layer 0 options',
-                name: 'Error',
-                label: 'ANONYMOUS LAYERGROUP',
-                type: 'layer'
-            },
-            moreErrors: []
-        };
-
-        this.testClient = new TestClient(mapConfig, 1234);
-
-        const params = {
-            response: {
-                status: 400,
-                headers: {
-                    'Content-Type': 'application/json; charset=utf-8',
-                    'X-Tiler-Errors': JSON.stringify(errorHeader)
-                }
-            }
-        };
-
-        this.testClient.getLayergroup(params, (err) => {
-            assert.ifError(err);
-            done();
-        });
-    });
-});

test/acceptance/errors-with-context-test.js

@@ -1,14 +1,14 @@
 'use strict';
 
 var assert = require('../support/assert');
-var CartodbWindshaft = require('../../lib/server');
+var createServer = require('../../lib/server');
 var serverOptions = require('../../lib/server-options');
 
 describe('error with context', function () {
     var server;
 
     before(function () {
-        server = new CartodbWindshaft(serverOptions);
+        server = createServer(serverOptions);
     });
 
     var layerOK = {