Release 3.6.3

Upgrades windshaft to 3.1.1

.gitignore

@@ -2,6 +2,8 @@ node_modules*
 config.status*
 config/environments/*.js
 .idea
+.vscode
+.nvmrc
 tools/munin/windshaft.conf
 logs/
 pids/

.travis.yml

@@ -1,37 +1,27 @@
+dist: trusty
 addons:
+  postgresql: "9.5"
   apt:
+    sources:
+      - ubuntu-toolchain-r-test
     packages:
+      - postgresql-9.5-postgis-2.3
+      - postgresql-plpython-9.5
       - pkg-config
       - libcairo2-dev
       - libjpeg8-dev
       - libgif-dev
+      - libpango1.0-dev
+      - g++-4.9
 
 before_install:
-  - npm install -g npm@2
-  - lsb_release -a
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
-  - sudo apt-get -qq purge postgis* postgresql*
-  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
-  - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.5
-  - sudo apt-add-repository --yes ppa:cartodb/gis
-  - sudo apt-get update
-  - sudo apt-get install -q postgresql-9.5-postgis-2.2
-  - sudo apt-get install -q postgresql-plpython-9.5
-  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.5/main/pg_hba.conf
-  - sudo service postgresql restart
   - createdb template_postgis
   - createuser publicuser
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
 env:
-  - NPROCS=1 JOBS=1 PGUSER=postgres
+  - NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9
 
 language: node_js
 node_js:
-  - "0.10"
-
-notifications:
-  irc:
-    channels:
-      - "irc.freenode.org#cartodb"
-    use_notice: true
+  - "6"

CONTRIBUTING.md

@@ -8,4 +8,4 @@ We love pull requests from everyone, see [Contributing to Open Source on GitHub]
 
 ## Submitting Contributions
 
-* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

HOWTO_RELEASE

@@ -1,11 +1,10 @@
 1. Test (make clean all check), fix if broken before proceeding
-2. Ensure proper version in package.json 
+2. Ensure proper version in package.json
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
-5. Commit package.json, npm-shrinwrap.json, NEWS
+4. Recreate yarn.lock with: `yarn upgrade`
+5. Commit package.json, yarn.lock, NEWS
 6. git tag -a Major.Minor.Patch # use NEWS section as content
-7. Announce on cartodb@googlegroups.com
-8. Stub NEWS/package for next version
+7. Stub NEWS/package for next version
 
 Versions:
 

INSTALL.md

@@ -4,11 +4,11 @@
 Make sure that you have the requirements needed. These are
 
 - Core
-  - Node.js >=0.8
-  - npm >=1.2.1 <2.0.0
+  - Node.js >=6.9.x
+  - yarn >=0.21.3
   - PostgreSQL >8.3.x, PostGIS >1.5.x
   - Redis >2.4.0 (http://www.redis.io)
-  - Mapnik 2.0.1, 2.0.2, 2.1.0, 2.2.0, 2.3.0. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
+  - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
   - Windshaft: check [Windshaft dependencies and installation notes](https://github.com/CartoDB/Windshaft#dependencies)
   - libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support
 
@@ -43,11 +43,11 @@ psql -d template_postgis -c 'CREATE EXTENSION postgis;'
 To fetch and build all node-based dependencies, run:
 
 ```
-npm install
+yarn
 ```
 
-Note that the ```npm install``` step will populate the node_modules/
+Note that the ```yarn``` step will populate the node_modules/
 directory with modules, some of which being compiled on demand. If you
 happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
-```npm install``` again.
+```yarn``` again.

Makefile

@@ -7,7 +7,7 @@ all:
 	@$(SHELL) ./scripts/install.sh
 
 clean:
-	rm -rf node_modules/*
+	rm -rf node_modules/
 
 distclean: clean
 	rm config.status*

NEWS.md

@@ -1,5 +1,504 @@
 # Changelog
 
+## 3.6.3
+Released 2017-04-25
+
+Announcements:
+ - Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1).
+
+
+## 3.6.2
+Released 2017-04-24
+
+Announcements:
+ - Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3).
+
+
+## 3.6.1
+Released 2017-04-24
+
+Announcements:
+ - Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1).
+
+
+## 3.6.0
+Released 2017-04-20
+
+Announcements:
+ - Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0).
+
+
+## 3.5.1
+Released 2017-04-11
+
+Announcements:
+ - Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1).
+
+
+## 3.5.0
+Released 2017-04-10
+
+Bug fixes:
+ - Fix invalidation of cache for maps with analyses #638.
+
+Announcements:
+ - Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0).
+
+
+## 3.4.0
+Released 2017-04-03
+
+Announcements:
+ - Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0).
+
+
+## 3.3.0
+Released 2017-04-03
+
+New features:
+ - Static map endpoints allow specifying the layers to render #653.
+
+
+## 3.2.0
+Released 2017-03-30
+
+Announcements:
+ - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0).
+ - Active GC interval.
+
+
+## 3.1.1
+Released 2017-03-23
+
+Bug fixes:
+ - Use crc32 instead of md5 for computing subdomain candidate #642
+
+
+## 3.1.0
+Released 2017-03-22
+
+Features:
+ - Generate URLs for resources based on CDN and template rules
+
+
+## 3.0.2
+Released 2017-03-22
+
+Bug fixes:
+ - Upgrade dependencies
+ - Improve docs: remove mentions to NPM and use yarn instead
+ - Remove script to generate npm-shrinkwrap file
+
+
+## 3.0.1
+Released 2017-03-21
+
+Announcements:
+ - Upgrades windshaft to [3.0.1](https://github.com/CartoDB/windshaft/releases/tag/3.0.1).
+
+
+## 3.0.0
+Released 2017-03-21
+
+Announcements:
+ - Supports Node v6.9.x
+ - Drops support for Node v0.10.x
+ - Upgrades windshaft to 3.0.0
+ - Upgrades cartodb-query-tables to 0.2.0
+ - Upgrades cartodb-redis to 0.13.2
+ - Upgrades redis-mpool to 0.4.1
+
+**Note**: Due to this [issue](https://github.com/npm/npm/issues/15713), Windshaft-cartodb must be installed with `yarn` instead of `npm` providing just a `yarn.lock` to get consistent installs across machines.
+
+## 2.89.0
+Released 2017-03-17
+
+**Deprecation warning**: v2.89.0 is the last release that supports Node v0.10.x. Next mayor release will support Node v6.9.x and further versions.
+
+Announcements:
+ - Upgrades windshaft to [2.8.0](https://github.com/CartoDB/windshaft/releases/tag/2.8.0).
+
+Bug fixes:
+ - Histogram column type discovery query uses non-filtered query #637
+
+
+## 2.88.4
+Released 2017-03-10
+
+Announcements:
+ - Upgrades camshaft to [0.50.3](https://github.com/CartoDB/camshaft/releases/tag/0.50.3).
+
+
+## 2.88.3
+Released 2017-03-02
+
+Bug fixes:
+- Category dataviews now uses the proper aggregation function for the 'Other' category. See https://github.com/CartoDB/Windshaft-cartodb/issues/628
+
+## 2.88.2
+Released 2017-02-23
+
+Announcements:
+ - Upgrades camshaft to [0.50.2](https://github.com/CartoDB/camshaft/releases/tag/0.50.2).
+
+
+## 2.88.1
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.1](https://github.com/CartoDB/camshaft/releases/tag/0.50.1)
+
+
+## 2.88.0
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.0](https://github.com/CartoDB/camshaft/releases/tag/0.50.0).
+ - Upgrades cartodb-psql to [0.7.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.7.1).
+ - Upgrades windshaft to [2.7.0](https://github.com/CartoDB/windshaft/releases/tag/2.7.0).
+
+
+## 2.87.5
+Released 2017-02-02
+
+Bug fixes:
+ - Cast dataview override values to Number or throw error.
+
+
+## 2.87.4
+Released 2017-01-20
+
+Bug fixes:
+ - Be able to not compute NULL categories and null values  in aggregation dataviews #617.
+
+
+## 2.87.3
+Released 2016-12-19
+
+Bug fixes:
+ - Fix overviews-related dataviews problems. See https://github.com/CartoDB/Windshaft-cartodb/pull/604
+
+
+## 2.87.2
+Released 2016-12-19
+
+- Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
+
+
+## 2.87.1
+Released 2016-12-13
+
+Announcements:
+ - Upgrades windshaft to [2.6.4](https://github.com/CartoDB/Windshaft/releases/tag/2.6.4).
+ - Upgrades request dependency.
+ - Regenerate npm-shrinkwrap.json: missing dependency updates.
+
+
+## 2.87.0
+Released 2016-12-12
+
+Enhancements:
+ - Upgrade turbo-carto dependency to version 0.19.0.
+
+## 2.86.1
+Released 2016-12-02
+
+Bug fixes:
+ - Maps with analyses and `sql_wrap` were broken #599.
+
+
+## 2.86.0
+Released 2016-12-02
+
+Announcements:
+ - Upgrades windshaft to [2.6.3](https://github.com/CartoDB/Windshaft/releases/tag/2.6.3).
+
+
+## 2.85.1
+Released 2016-11-30
+
+Announcements:
+ - Upgrades camshaft to [0.48.4](https://github.com/CartoDB/camshaft/releases/tag/0.48.4).
+
+
+## 2.85.0
+Released 2016-11-24
+
+New features:
+ - Allow to set resource URL templates with substitution tokens #594.
+
+
+## 2.84.2
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.3](https://github.com/CartoDB/camshaft/releases/tag/0.48.3).
+
+
+## 2.84.1
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.2](https://github.com/CartoDB/camshaft/releases/tag/0.48.2).
+
+
+## 2.84.0
+Released 2016-11-11
+
+New features:
+ - Analyses limit configuration allows to set other limits than timeout.
+
+
+## 2.83.1
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.1](https://github.com/CartoDB/camshaft/releases/tag/0.48.1).
+
+
+## 2.83.0
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.0](https://github.com/CartoDB/camshaft/releases/tag/0.48.0).
+
+
+## 2.82.0
+Released 2016-11-08
+
+Announcements:
+ - Upgrades camshaft to [0.47.0](https://github.com/CartoDB/camshaft/releases/tag/0.47.0).
+
+
+## 2.81.1
+Released 2016-11-05
+
+Announcements:
+ - Upgrades windshaft to [2.6.2](https://github.com/CartoDB/windshaft/releases/tag/2.6.2).
+ - Upgrades camshaft to [0.46.3](https://github.com/CartoDB/camshaft/releases/tag/0.46.3).
+
+
+## 2.81.0
+Released 2016-11-02
+
+Enhancements:
+ - Returns errors with context when query layer does not retrieve geometry column
+
+Announcements:
+ - Upgrades windshaft to [2.6.1](https://github.com/CartoDB/windshaft/releases/tag/2.6.1).
+ - Upgrades camshaft to [0.46.2](https://github.com/CartoDB/camshaft/releases/tag/0.46.2).
+
+
+## 2.80.2
+Released 2016-10-26
+
+Bug fixes:
+ - Fix order in categories query to get ramps
+
+
+## 2.80.1
+Released 2016-10-25
+
+Announcements:
+ - Upgrades camshaft to [0.46.1](https://github.com/CartoDB/camshaft/releases/tag/0.46.1).
+
+
+## 2.80.0
+Released 2016-10-20
+
+Announcements:
+ - Upgrades camshaft to [0.46.0](https://github.com/CartoDB/camshaft/releases/tag/0.46.0).
+
+New features:
+ - Default analyses limits can be defined in configuration.
+
+
+## 2.79.0
+Released 2016-10-11
+
+New features:
+ - Retrieve analysis limits and pass them into camshaft.
+
+Announcements:
+ - Upgrades turbo-carto to [0.18.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.18.0).
+ - Upgrades camshaft to [0.45.0](https://github.com/CartoDB/camshaft/releases/tag/0.45.0).
+
+
+## 2.78.1
+Released 2016-09-30
+
+Announcements:
+ - Upgrades camshaft to [0.44.2](https://github.com/CartoDB/camshaft/releases/tag/0.44.2).
+
+
+## 2.78.0
+Released 2016-09-29
+
+New features:
+ - Add metadata about processed turbo-carto rules.
+
+Announcements:
+ - Upgrades turbo-carto to [0.17.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.17.1).
+
+
+## 2.77.1
+
+Released 2016-09-28
+
+Announcements:
+ - Upgrades camshaft to [0.44.1](https://github.com/CartoDB/camshaft/releases/tag/0.44.1).
+
+
+## 2.77.0
+
+Released 2016-09-26
+
+Announcements:
+ - Upgrades camshaft to [0.44.0](https://github.com/CartoDB/camshaft/releases/tag/0.44.0).
+ - Adds a new configuration for camshaft: logger stream.
+
+
+## 2.76.0
+
+Released 2016-09-15
+
+New features:
+ - Allow to use `--config /path/to/config.js` to specify configuration file.
+   - Environment will be loaded from config file if `environment` key is present, otherwise it keeps current behaviour.
+
+Bug fixes:
+ - Allow to use absolute paths for log files #570.
+
+
+## 2.75.0
+
+Released 2016-09-14
+
+Announcements:
+ - Upgrades camshaft to [0.43.0](https://github.com/CartoDB/camshaft/releases/tag/0.43.0).
+
+
+## 2.74.1
+
+Released 2016-09-07
+
+Announcements:
+ - Upgrades camshaft to [0.42.1](https://github.com/CartoDB/camshaft/releases/tag/0.42.1).
+
+
+## 2.74.0
+
+Released 2016-09-06
+
+Enhancements:
+ - Layers in previews can be shown or hidden using `preview_layers` property in template map
+
+
+## 2.73.1
+
+Released 2016-09-06
+
+Bug fixes:
+ - Fixes missing column in fixture table `cdb_analysis_catalog`.
+
+
+## 2.73.0
+
+Released 2016-09-06
+
+Announcements:
+ - Upgrades camshaft to [0.42.0](https://github.com/CartoDB/camshaft/releases/tag/0.42.0).
+
+
+## 2.72.0
+
+Released 2016-08-23
+
+Announcements:
+ - Upgrades camshaft to [0.41.0](https://github.com/CartoDB/camshaft/releases/tag/0.41.0).
+
+
+## 2.71.0
+
+Released 2016-08-17
+
+Announcements:
+ - Upgrades windshaft to [2.5.0](https://github.com/CartoDB/windshaft/releases/tag/2.5.0).
+
+
+## 2.70.0
+
+Released 2016-08-16
+
+Announcements:
+ - Upgrades camshaft to [0.40.0](https://github.com/CartoDB/camshaft/releases/tag/0.40.0).
+
+
+## 2.69.1
+
+Released 2016-08-12
+
+Announcements:
+ - Upgrades windshaft to [2.4.2](https://github.com/CartoDB/windshaft/releases/tag/2.4.2).
+
+
+## 2.69.0
+
+Released 2016-08-11
+
+Announcements:
+ - Upgrades camshaft to [0.39.0](https://github.com/CartoDB/camshaft/releases/tag/0.39.0).
+
+
+## 2.68.0
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades turbo-carto to [0.16.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.16.0).
+
+
+## 2.67.1
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades camshaft to [0.38.1](https://github.com/CartoDB/camshaft/releases/tag/0.38.1).
+
+
+## 2.67.0
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades camshaft to [0.38.0](https://github.com/CartoDB/camshaft/releases/tag/0.38.0).
+
+
+## 2.66.2
+
+Released 2016-07-20
+
+Announcements:
+ - Upgrades turbo-carto to [0.15.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.15.1).
+
+
+## 2.66.1
+
+Released 2016-07-20
+
+Announcements:
+ - Upgrades turbo-carto to [0.15.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.15.0).
+
+
+## 2.66.0
+
+Released 2016-07-18
+
+Announcements:
+ - Available new endpoint to check user analyses.
+ - Upgrades camshaft to [0.37.1](https://github.com/CartoDB/camshaft/releases/tag/0.37.1).
+
+
 ## 2.65.0
 
 Released 2016-07-15

README.md

@@ -32,14 +32,14 @@ Upgrading
 Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following:
 
 ```
-rm -rf node_modules; npm install
+rm -rf node_modules; yarn
 ```
 
 Run
 ---
 
 ```
-node app.js <env> 
+node app.js <env>
 ```
 
 Where <env> is the name of a configuration file under config/environments/.
@@ -71,12 +71,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md).
 ### Developing with a custom windshaft version
 
 If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
-to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
+to use `yarn link`. You can read more about it at [yarn-link: Symlink a package folder](https://yarnpkg.com/en/docs/cli/link).
 
 **Quick start**:
 
 ```shell
-~/windshaft-directory $ npm install
-~/windshaft-directory $ npm link
-~/windshaft-cartodb-directory $ npm link windshaft
+~/windshaft-directory $ yarn
+~/windshaft-directory $ yarn link
+~/windshaft-cartodb-directory $ yarn link windshaft
 ```

app.js

@@ -5,20 +5,35 @@ var fs = require('fs');
 
 var _ = require('underscore');
 
-var ENVIRONMENT;
-if ( process.argv[2] ) {
-    ENVIRONMENT = process.argv[2];
-} else if ( process.env.NODE_ENV ) {
-    ENVIRONMENT = process.env.NODE_ENV;
-} else {
-    ENVIRONMENT = 'development';
-}
-
 // jshint undef:false
 var log = console.log.bind(console);
 var logError = console.error.bind(console);
 // jshint undef:true
 
+var argv = require('yargs')
+    .usage('Usage: $0 <environment> [options]')
+    .help('h')
+    .example(
+    '$0 production -c /etc/sql-api/config.js',
+    'start server in production environment with /etc/sql-api/config.js as config file'
+)
+    .alias('h', 'help')
+    .alias('c', 'config')
+    .nargs('c', 1)
+    .describe('c', 'Load configuration from path')
+    .argv;
+
+var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
+var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
+if (!fs.existsSync(configurationFile)) {
+    logError('Configuration file "%s" does not exist', configurationFile);
+    process.exit(1);
+}
+
+global.environment = require(configurationFile);
+var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.environment.environment;
+process.env.NODE_ENV = ENVIRONMENT;
+
 var availableEnvironments = {
     production: true,
     staging: true,
@@ -33,16 +48,6 @@ if (!availableEnvironments[ENVIRONMENT]){
 }
 
 process.env.NODE_ENV = ENVIRONMENT;
-
-// set environment specific variables
-global.environment = require('./config/environments/' + ENVIRONMENT);
-
-global.log4js = require('log4js');
-var log4js_config = {
-  appenders: [],
-  replaceConsole: true
-};
-
 if (global.environment.uv_threadpool_size) {
     process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
 }
@@ -58,25 +63,31 @@ var agentOptions = _.defaults(global.environment.httpAgent || {}, {
 http.globalAgent = new http.Agent(agentOptions);
 https.globalAgent = new https.Agent(agentOptions);
 
+
+global.log4js = require('log4js');
+var log4jsConfig = {
+    appenders: [],
+    replaceConsole: true
+};
+
 if ( global.environment.log_filename ) {
-  var logdir = path.dirname(global.environment.log_filename);
-  // See cwd inlog4js.configure call below
-  logdir = path.resolve(__dirname, logdir);
-  if ( ! fs.existsSync(logdir) ) {
-    logError("Log filename directory does not exist: " + logdir);
-    process.exit(1);
-  }
-  log("Logs will be written to " + global.environment.log_filename);
-  log4js_config.appenders.push(
-    { type: "file", filename: global.environment.log_filename }
-  );
+    var logFilename = path.resolve(global.environment.log_filename);
+    var logDirectory = path.dirname(logFilename);
+    if (!fs.existsSync(logDirectory)) {
+        logError("Log filename directory does not exist: " + logDirectory);
+        process.exit(1);
+    }
+    log("Logs will be written to " + logFilename);
+    log4jsConfig.appenders.push(
+        { type: "file", absolute: true, filename: logFilename }
+    );
 } else {
-  log4js_config.appenders.push(
-    { type: "console", layout: { type:'basic' } }
-  );
+    log4jsConfig.appenders.push(
+        { type: "console", layout: { type:'basic' } }
+    );
 }
 
-global.log4js.configure(log4js_config, { cwd: __dirname });
+global.log4js.configure(log4jsConfig);
 global.logger = global.log4js.getLogger();
 
 global.environment.api_hostname = require('os').hostname().split('.')[0];
@@ -99,6 +110,8 @@ var listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, b
 var version = require("./package").version;
 
 listener.on('listening', function() {
+    log("Using Node.js %s", process.version);
+    log('Using configuration file "%s"', configurationFile);
     log(
         "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
         version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
@@ -114,7 +127,7 @@ setInterval(function() {
 
 process.on('SIGHUP', function() {
     global.log4js.clearAndShutdownAppenders(function() {
-        global.log4js.configure(log4js_config);
+        global.log4js.configure(log4jsConfig);
         global.logger = global.log4js.getLogger();
         log('Log files reloaded');
     });
@@ -123,3 +136,17 @@ process.on('SIGHUP', function() {
 process.on('uncaughtException', function(err) {
     global.logger.error('Uncaught exception: ' + err.stack);
 });
+
+if (global.gc) {
+    var gcInterval = Number.isFinite(global.environment.gc_interval) ?
+        global.environment.gc_interval :
+        10000;
+
+    if (gcInterval > 0) {
+        setInterval(function gcForcedCycle() {
+            var start = Date.now();
+            global.gc();
+            global.statsClient.timing('windshaft.gc', Date.now() - start);
+        }, gcInterval);
+    }
+}

config/environments/development.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.localhost'
@@ -23,6 +26,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -62,6 +80,7 @@ var config = {
         extent: "-180,-90,180,90",
         srid: 4326,
         */
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -95,6 +114,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -209,6 +232,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/production.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -23,6 +26,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -56,6 +74,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         /*
          * Set persist_connection to false if you want
@@ -89,6 +108,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -203,6 +226,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/staging.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -23,6 +26,21 @@ var config = {
     // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -56,6 +74,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -89,6 +108,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -203,6 +226,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/test.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '(.*)'
@@ -23,6 +26,20 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -56,6 +73,7 @@ var config = {
         host: '127.0.0.1',
         port: 5432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -89,6 +107,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -204,6 +226,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

docs/anonymous_maps.md

@@ -141,9 +141,6 @@ https://{username}.carto.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
 Some notes about filtering:
 
   - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors.
-  - Once a Mapnik layer is selected, all Mapnik layers will get blended. As this may change in the future **it is
-  recommended** to always select all Mapnik layers if you want to select at least one so you will get a consistent
-  behavior in the future.
   - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this
   may change in the future **it is recommended** to always select the layers in ascending order so you will get a
   consistent behavior in the future.

docs/named_maps.md

@@ -1,10 +1,10 @@
 # Named Maps
 
-Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private). 
+Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private).
 
-The Named Map workflow consists of uploading a MapConfig file to CARTO servers, to select data from your CARTO user database by using SQL, and specifying the CartoCSS for your map. 
+The Named Map workflow consists of uploading a MapConfig file to CARTO servers, to select data from your CARTO user database by using SQL, and specifying the CartoCSS for your map.
 
-The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps. 
+The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps.
 
 **Tip:** You can also use a Named Map that you created (which is defined by its `name`), to create a map using CARTO.js. This is achieved by adding the [`namedmap` type](http://docs.carto.com/carto-engine/carto-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
 
@@ -84,6 +84,10 @@ The `name` argument defines how to name this "template_name".json. Note that the
       "south": -45,
       "east": 45,
       "north": 45
+    },
+    "preview_layers": {
+      "0": true,
+      "layer1": false
     }
   }
 }
@@ -95,22 +99,22 @@ Params | Description
 --- | ---
 name | There can only be _one_ template with the same name for any user. Valid names start with a letter or a number, and only contain letters, numbers, dashes (-), or underscores (_). _This is specific to the name of your Named Map that is specified in the `name` property of the template file_.
 
-auth | 
+auth |
 --- | ---
 |_ method | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
 |_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.carto.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
 placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
 layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.carto.com/carto-engine/maps-api/mapconfig/) for more information.
-view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box. Also it is possible to choose which layers are visible or not with `preview_layers` indicating its visibility by layer index or id (visible by default).
 --- | ---
 |_ zoom | The zoom level to use
 
-|_ center | 
+|_ center |
 --- | ---
 |_ |_ lng | The longitude to use for the center
 |_ |_ lat | The latitude to use for the center
 
-|_ bounds | 
+|_ bounds |
 --- | ---
 |_ |_ west | LowerCorner longitude for the bounding box, in decimal degrees (aka most western)
 |_ |_ south | LowerCorner latitude for the bounding box, in decimal degrees (aka most southern)
@@ -120,7 +124,7 @@ view (optional) | extra keys to specify the view area for the map. It can be use
 
 ### Placeholder Format
 
-Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergoup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
+Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergroup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
 
 Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced inside the Named Maps API.
 
@@ -160,7 +164,7 @@ curl -X POST \
 
 #### Response
 
-The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL. 
+The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL.
 
 ```javascript
 {
@@ -516,18 +520,18 @@ If you are creating a Torque layer in a Named Map without using the Torque.js li
     })
     .addTo(map)
     .done(function(layer) {
-  
+
     });
 }
 ```
 
 #### Examples of Named Maps created with CARTO.js
 
-- [Named Map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+- [Named Map selectors with interaction](http://bl.ocks.org/andy-esch/515a8af1f99d5e690484)
 
-- [Named Map with interactivity](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+- [Named Map with interactivity](http://bl.ocks.org/andy-esch/d1a45b8ff5e7bd90cd68)
 
-- [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)
+- [Toggling sublayers in a Named Map](http://bl.ocks.org/andy-esch/c1a0f4913610eec53cd3)
 
 ## Fetching XYZ Tiles for Named Maps
 
@@ -535,7 +539,7 @@ Optionally, authenticated users can fetch projected tiles (XYZ tiles or Mapnik R
 
 ### Fetch XYZ Tiles Directly with a URL
 
-Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map. 
+Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map.
 
 To call a template_id in a URL:
 

docs/static_maps_api.md

@@ -146,10 +146,10 @@ It is important to note that generated images are cached from the live data refe
 
 ### Limits
 
-* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
-* Image resolution by default is set to 72 DPI
-* JPEG quality by default is 85%
-* Timeout limits for generating static maps are the same across the CARTO Editor and CARTO Engine. It is important to ensure timely processing of queries.
+* While images can encompass an entirety of a map, the limit for pixel range is 8192 x 8192.
+* Image resolution is set to 72 DPI
+* JPEG quality is 85%
+* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
 
 ## Examples
 

lib/cartodb/api/auth_api.js

@@ -95,9 +95,7 @@ AuthApi.prototype.authorize = function(req, callback) {
             self.authorizedByAPIKey(user, req, this);
         },
         function checkApiKey(err, authorized){
-            if (req.profiler) {
-                req.profiler.done('authorizedByAPIKey');
-            }
+            req.profiler.done('authorizedByAPIKey');
             assert.ifError(err);
 
             // if not authorized by api_key, continue
@@ -131,9 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) {
             }
 
             self.pgConnection.setDBAuth(user, req.params, function(err) {
-                if (req.profiler) {
-                    req.profiler.done('setDBAuth');
-                }
+                req.profiler.done('setDBAuth');
                 callback(err, true); // authorized (or error)
             });
         }

lib/cartodb/backends/analysis.js

@@ -1,19 +1,93 @@
-var camshaft = require('camshaft');
+'use strict';
 
-function AnalysisBackend(options) {
-    var batchConfig = options.batch || {};
+var _ = require('underscore');
+var camshaft = require('camshaft');
+var fs = require('fs');
+
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.options.limits = this.options.limits || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
+}
+
+module.exports = AnalysisBackend;
+
+AnalysisBackend.prototype.setBatchConfig = function (options) {
+    var batchConfig = options || {};
     batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job';
     batchConfig.inlineExecution = batchConfig.inlineExecution || false;
     batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan';
     this.batchConfig = batchConfig;
-}
+};
 
-module.exports = AnalysisBackend;
+AnalysisBackend.prototype.setLoggerConfig = function (options) {
+    this.loggerConfig = options || {};
+
+    if (this.loggerConfig.filename) {
+        this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+
+        process.on('SIGHUP', function () {
+            if (this.stream) {
+                this.stream.destroy();
+            }
+
+            this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+        }.bind(this));
+    }
+};
 
 AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) {
     analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
     analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
     analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    analysisConfiguration.logger = {
+        stream: this.stream ? this.stream : process.stdout
+    };
+
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+
+    var analysesLimits = {
+        analyses: {
+            // buffer: {
+            //     timeout: 1000,
+            //     maxNumberOfRows: 1e6
+            // }
+        }
+    };
+
+    Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) {
+        analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]);
+    });
+
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        // analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 }
+        analysesTimeouts = analysesTimeouts || {};
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = _.defaults(
+                {
+                    timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+                },
+                analysesLimits.analyses[analysisType]
+            );
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/dataview.js

@@ -79,7 +79,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
 
             var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
                 function castNumbers(overrides, val, k) {
-                    overrides[k] = Number.isFinite(+val) ? +val : val;
+                    if (!Number.isFinite(+val)) {
+                        throw new Error('Invalid number format for parameter \'' + k + '\'');
+                    }
+                    overrides[k] = +val;
                     return overrides;
                 },
                 {ownFilter: ownFilter}

lib/cartodb/backends/template_maps.js

@@ -55,11 +55,9 @@ util.inherits(TemplateMaps, EventEmitter);
 module.exports = TemplateMaps;
 
 
-var o = TemplateMaps.prototype;
-
 //--------------- PRIVATE METHODS --------------------------------
 
-o._userTemplateLimit = function() {
+TemplateMaps.prototype._userTemplateLimit = function() {
   return this.opts.max_user_templates || 0;
 };
 
@@ -70,7 +68,7 @@ o._userTemplateLimit = function() {
  * @param redisArgs - the arguments for the redis function in an array
  * @param callback - function to pass results too.
  */
-o._redisCmd = function(redisFunc, redisArgs, callback) {
+TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
   var redisClient;
   var that = this;
   var db = that.db_signatures;
@@ -97,7 +95,7 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
 var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
 // jshint maxcomplexity:15
-o._checkInvalidTemplate = function(template) {
+TemplateMaps.prototype._checkInvalidTemplate = function(template) {
   if ( template.version !== '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
   }
@@ -200,7 +198,7 @@ function templateDefaults(template) {
 // @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
-o.addTemplate = function(owner, template, callback) {
+TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
     var self = this;
 
     template = templateDefaults(template);
@@ -258,7 +256,7 @@ o.addTemplate = function(owner, template, callback) {
 //
 // @param callback function(err)
 //
-o.delTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function deleteTemplate() {
@@ -297,7 +295,8 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //
 // @param callback function(err)
 //
-o.updTemplate = function(owner, tpl_id, template, callback) {
+TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
+    
     var self = this;
 
     template = templateDefaults(template);
@@ -356,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
 //
-o.listTemplates = function(owner, callback) {
+TemplateMaps.prototype.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
 
@@ -370,7 +369,7 @@ o.listTemplates = function(owner, callback) {
 // @param callback function(err, template)
 //        Return full template definition
 //
-o.getTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function getTemplate() {
@@ -386,7 +385,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
     );
 };
 
-o.isAuthorized = function(template, authTokens) {
+TemplateMaps.prototype.isAuthorized = function(template, authTokens) {
     if (!template) {
         return false;
     }
@@ -438,7 +437,7 @@ function _replaceVars (str, params) {
     });
     return str;
 }
-o.instance = function(template, params) {
+TemplateMaps.prototype.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
   Object.keys(phold).forEach(function(k) {
@@ -500,7 +499,7 @@ o.instance = function(template, params) {
 };
 
 // Return a fingerPrint of the object
-o.fingerPrint = function(template) {
+TemplateMaps.prototype.fingerPrint = function(template) {
   return crypto.createHash('md5')
     .update(JSON.stringify(template))
     .digest('hex')

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -6,6 +6,9 @@ dot.templateSettings.strip = false;
 function createTemplate(method) {
     return dot.template([
         'SELECT',
+        'min({{=it._column}}) min_val,',
+        'max({{=it._column}}) max_val,',
+        'avg({{=it._column}}) avg_val,',
         method,
         'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
     ].join('\n'));
@@ -29,17 +32,12 @@ methodTemplates.category = dot.template([
     '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
     '  FROM ({{=it._sql}}) _cdb_aggregation_all',
     '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
+    '  ORDER BY 2 DESC, 1 ASC',
     '),',
     'agg_categories AS (',
-    '  SELECT \'__other\' category',
+    '  SELECT category',
     '  FROM categories',
-    '  WHERE rank >= {{=it._buckets}}',
-    '  GROUP BY 1',
-    '  UNION ALL',
-    '  SELECT CAST(category AS text)',
-    '  FROM categories',
-    '  WHERE rank < {{=it._buckets}}',
+    '  WHERE rank <= {{=it._buckets}}',
     ')',
     'SELECT array_agg(category) AS category FROM agg_categories'
 ].join('\n'));
@@ -79,11 +77,14 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             return callback(err);
         }
 
-        resultSet = resultSet || {};
-        var result = resultSet.rows || [];
-
+        var result = getResult(resultSet);
         var strategy = method2strategy[methodName];
-        var ramp = result[0][methodName] || [];
+        var ramp = result[methodName] || [];
+        var stats = {
+            min_val: result.min_val,
+            max_val: result.max_val,
+            avg_val: result.avg_val
+        };
         // Skip null values from ramp
         // Generated turbo-carto won't be correct, but better to keep it working than failing
         // TODO fix cartodb-postgres extension quantification functions
@@ -94,8 +95,16 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             });
         }
 
-        return callback(null, { ramp: ramp, strategy: strategy });
+        return callback(null, { ramp: ramp, strategy: strategy, stats: stats });
     }, true); // use read-only transaction
 };
 
+function getResult(resultSet) {
+    resultSet = resultSet || {};
+    var result = resultSet.rows || [];
+    result = result[0] || {};
+
+    return result;
+}
+
 module.exports = PostgresDatasource;

lib/cartodb/controllers/base.js

@@ -36,7 +36,7 @@ function BaseController(authApi, pgConnection) {
 
 module.exports = BaseController;
 
-// jshint maxcomplexity:9
+// jshint maxcomplexity:10
 /**
  * Whitelist input and get database name & default geometry type from
  * subdomain/user metadata held in CartoDB Redis
@@ -61,9 +61,7 @@ BaseController.prototype.req2params = function(req, callback){
         lzmaWorker.decompress(
             lzma,
             function(result) {
-                if (req.profiler) {
-                    req.profiler.done('lzma');
-                }
+                req.profiler.done('lzma');
                 try {
                     delete req.query.lzma;
                     _.extend(req.query, JSON.parse(result));
@@ -77,7 +75,11 @@ BaseController.prototype.req2params = function(req, callback){
         return;
     }
 
-    req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
+    var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
+    if (Array.isArray(req.context.allowedQueryParams)) {
+        allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);
+    }
+    req.query = _.pick(req.query, allowedQueryParams);
     req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
 
     var user = req.context.user;
@@ -115,18 +117,14 @@ BaseController.prototype.req2params = function(req, callback){
     // bring all query values onto req.params object
     _.extend(req.params, req.query);
 
-    if (req.profiler) {
-        req.profiler.done('req2params.setup');
-    }
+    req.profiler.done('req2params.setup');
 
     step(
         function getPrivacy(){
             self.authApi.authorize(req, this);
         },
         function validateAuthorization(err, authorized) {
-            if (req.profiler) {
-                req.profiler.done('authorize');
-            }
+            req.profiler.done('authorize');
             assert.ifError(err);
             if(!authorized) {
                 err = new Error("Sorry, you are unauthorized (permission denied)");
@@ -167,9 +165,7 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
         res.set('X-Served-By-DB-Host', req.params.dbhost);
     }
 
-    if (req.profiler) {
-        res.set('X-Tiler-Profiler', req.profiler.toJSONString());
-    }
+    res.set('X-Tiler-Profiler', req.profiler.toJSONString());
 
     if (headers) {
         res.set(headers);
@@ -187,14 +183,12 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
         res.send(body);
     }
 
-    if (req.profiler) {
-        try {
-            // May throw due to dns, see
-            // See http://github.com/CartoDB/Windshaft/issues/166
-            req.profiler.sendStats();
-        } catch (err) {
-            debug("error sending profiling stats: " + err);
-        }
+    try {
+        // May throw due to dns, see
+        // See http://github.com/CartoDB/Windshaft/issues/166
+        req.profiler.sendStats();
+    } catch (err) {
+        debug("error sending profiling stats: " + err);
     }
 };
 // jshint maxcomplexity:6

lib/cartodb/controllers/layergroup.js

@@ -6,6 +6,7 @@ var BaseController = require('./base');
 
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
 
 var DataviewBackend = require('../backends/dataview');
 var AnalysisStatusBackend = require('../backends/analysis-status');
@@ -67,11 +68,13 @@ LayergroupController.prototype.register = function(app) {
         this.attributes.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
+        '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        cors(), userMiddleware, allowQueryParams(['layer']),
         this.center.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
+        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        cors(), userMiddleware, allowQueryParams(['layer']),
         this.bbox.bind(this));
 
     // Undocumented/non-supported API endpoint methods.
@@ -338,6 +341,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
 LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
     var self = this;
 
+    req.profiler.done('res');
+
     res.set('Cache-Control', 'public,max-age=31536000');
 
     // Set Last-Modified header
@@ -386,13 +391,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 function getSQL(err, mapConfig) {
                     assert.ifError(err);
 
-                    var queries = mapConfig.getLayers()
-                        .map(function(lyr) {
-                            return lyr.options.sql;
-                        })
-                        .filter(function(sql) {
-                            return !!sql;
-                        });
+                    var queries = [];
+                    mapConfig.getLayers().forEach(function(layer) {
+                        queries.push(layer.options.sql);
+                        if (layer.options.affected_tables) {
+                            layer.options.affected_tables.map(function(table) {
+                                queries.push('SELECT * FROM ' + table + ' LIMIT 0');
+                            });
+                        }
+                    });
 
                     return queries.length ? queries.join(';') : null;
                 },

lib/cartodb/controllers/map.js

@@ -4,6 +4,8 @@ var step = require('step');
 var windshaft = require('windshaft');
 var QueryTables = require('cartodb-query-tables');
 
+var ResourceLocator = require('../models/resource-locator');
+
 var util = require('util');
 var BaseController = require('./base');
 
@@ -44,6 +46,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.mapConfigAdapter = mapConfigAdapter;
+    this.resourceLocator = new ResourceLocator(global.environment);
 }
 
 util.inherits(MapController, BaseController);
@@ -84,9 +87,7 @@ MapController.prototype.createPost = function(req, res) {
 };
 
 MapController.prototype.instantiate = function(req, res) {
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.instance_template_post');
-    }
+    req.profiler.start('windshaft-cartodb.instance_template_post');
 
     this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
         if (!req.is('application/json')) {
@@ -97,9 +98,7 @@ MapController.prototype.instantiate = function(req, res) {
 };
 
 MapController.prototype.jsonp = function(req, res) {
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.instance_template_get');
-    }
+    req.profiler.start('windshaft-cartodb.instance_template_get');
 
     this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
         var err = null;
@@ -166,11 +165,30 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function finish(err, layergroup) {
             if (err) {
+                if (Number.isFinite(err.layerIndex)) {
+                    var error = new Error(err.message);
+                    error.http_status = err.http_status;
+
+                    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
+                        error.http_status = 400;
+                    }
+
+                    error.type = 'layer';
+                    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
+                    error.layer = {
+                        id: mapConfig.getLayerId(err.layerIndex),
+                        index: err.layerIndex,
+                        type: mapConfig.layerType(err.layerIndex)
+                    };
+
+                    err = error;
+                }
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
                 var analysesResults = context.analysesResults || [];
-                addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
-                addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                self.addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
+                self.addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                addContextMetadata(layergroup, mapConfig.obj(), context);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -178,6 +196,17 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
     );
 };
 
+function addContextMetadata(layergroup, mapConfig, context) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
+                layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
+            }
+            return layer;
+        });
+    }
+}
+
 MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
     var self = this;
 
@@ -229,8 +258,10 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
-                addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj());
-                addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+                var _mapConfig = mapConfig.obj();
+                self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
+                self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+                addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context);
 
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
@@ -268,18 +299,22 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
     // take place before proceeding. Error will be logged
     // asynchronously
     this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) {
-        if (req.profiler) {
-            req.profiler.done('incMapviewCount');
-        }
+        req.profiler.done('incMapviewCount');
         if ( err ) {
             global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
         }
         done();
     });
 
-    var sql = mapconfig.getLayers().map(function(layer) {
-        return layer.options.sql;
-    }).join(';');
+    var sql = [];
+    mapconfig.getLayers().forEach(function(layer) {
+        sql.push(layer.options.sql);
+        if (layer.options.affected_tables) {
+            layer.options.affected_tables.map(function(table) {
+                sql.push('SELECT * FROM ' + table + ' LIMIT 0');
+            });
+        }
+    });
 
     var dbName = req.params.dbname;
     var layergroupId = layergroup.layergroupid;
@@ -290,12 +325,10 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
         },
         function getAffectedTablesAndLastUpdatedTime(err, connection) {
             assert.ifError(err);
-            QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+            QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this);
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
-            if (req.profiler) {
-                req.profiler.done('queryTablesAndLastUpdated');
-            }
+            req.profiler.done('queryTablesAndLastUpdated');
             assert.ifError(err);
             // feed affected tables cache so it can be reused from, for instance, layergroup controller
             self.layergroupAffectedTables.set(dbName, layergroupId, result);
@@ -338,7 +371,7 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
     }, lastUpdateTime);
 }
 
-function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) {
+MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
     includeQuery = includeQuery || false;
     analysesResults = analysesResults || [];
     layergroup.metadata.analyses = [];
@@ -351,7 +384,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                     var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
                     var nodeRepr = {
                         status: node.getStatus(),
-                        url: getUrls(username, nodeResource)
+                        url: this.resourceLocator.getUrls(username, nodeResource)
                     };
                     if (includeQuery) {
                         nodeRepr.query = node.getQuery();
@@ -363,30 +396,30 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                 }
 
                 return nodesIdMap;
-            }, {})
+            }.bind(this), {})
         });
-    });
-}
+    }.bind(this));
+};
 
 // TODO this should take into account several URL patterns
-function addDataviewsAndWidgetsUrls(username, layergroup, mapConfig) {
-    addDataviewsUrls(username, layergroup, mapConfig);
-    addWidgetsUrl(username, layergroup, mapConfig);
-}
+MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) {
+    this.addDataviewsUrls(username, layergroup, mapConfig);
+    this.addWidgetsUrl(username, layergroup, mapConfig);
+};
 
-function addDataviewsUrls(username, layergroup, mapConfig) {
+MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) {
     layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
     var dataviews = mapConfig.dataviews || {};
 
     Object.keys(dataviews).forEach(function(dataviewName) {
         var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
         layergroup.metadata.dataviews[dataviewName] = {
-            url: getUrls(username, resource)
+            url: this.resourceLocator.getUrls(username, resource)
         };
-    });
-}
+    }.bind(this));
+};
 
-function addWidgetsUrl(username, layergroup, mapConfig) {
+MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) {
     if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
         layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
             var mapConfigLayer = mapConfig.layers[layerIndex];
@@ -396,26 +429,11 @@ function addWidgetsUrl(username, layergroup, mapConfig) {
                     var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
                     layer.widgets[widgetName] = {
                         type: mapConfigLayer.options.widgets[widgetName].type,
-                        url: getUrls(username, resource)
+                        url: this.resourceLocator.getUrls(username, resource)
                     };
-                });
+                }.bind(this));
             }
             return layer;
-        });
+        }.bind(this));
     }
-}
-
-function getUrls(username, resource) {
-    var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
-    if (cdnUrl) {
-        return {
-            http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource,
-            https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource
-        };
-    } else {
-        var port = global.environment.port;
-        return {
-            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
-        };
-    }
-}
+};

lib/cartodb/controllers/named_maps.js

@@ -8,6 +8,7 @@ var BaseController = require('./base');
 
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
 
 function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
@@ -31,7 +32,7 @@ NamedMapsController.prototype.register = function(app) {
         this.tile.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
+        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']),
         this.staticMap.bind(this));
 };
 
@@ -100,9 +101,7 @@ NamedMapsController.prototype.tile = function(req, res) {
             self.tileBackend.getTile(namedMapProvider, req.params, this);
         },
         function handleImage(err, tile, headers, stats) {
-            if (req.profiler) {
-                req.profiler.add(stats);
-            }
+            req.profiler.add(stats);
             if (err) {
                 self.sendError(req, res, err, 'NAMED_MAP_TILE');
             } else {
@@ -137,9 +136,15 @@ NamedMapsController.prototype.staticMap = function(req, res) {
                 this
             );
         },
-        function prepareImageOptions(err, _namedMapProvider) {
+        function prepareLayerVisibility(err, _namedMapProvider) {
             assert.ifError(err);
+
             namedMapProvider = _namedMapProvider;
+
+            self.prepareLayerFilterFromPreviewLayers(cdbUser, req, namedMapProvider, this);
+        },
+        function prepareImageOptions(err) {
+            assert.ifError(err);
             self.getStaticImageOptions(cdbUser, req.params, namedMapProvider, this);
         },
         function getImage(err, imageOpts) {
@@ -170,10 +175,8 @@ NamedMapsController.prototype.staticMap = function(req, res) {
             });
         },
         function handleImage(err, image, headers, stats) {
-            if (req.profiler) {
-                req.profiler.done('render-' + format);
-                req.profiler.add(stats || {});
-            }
+            req.profiler.done('render-' + format);
+            req.profiler.add(stats || {});
 
             if (err) {
                 self.sendError(req, res, err, 'STATIC_VIZ_MAP');
@@ -184,6 +187,45 @@ NamedMapsController.prototype.staticMap = function(req, res) {
     );
 };
 
+NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (user, req, namedMapProvider, callback) {
+    var self = this;
+    namedMapProvider.getTemplate(function (err, template) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!template || !template.view || !template.view.preview_layers) {
+           return callback();
+        }
+
+        var previewLayers = template.view.preview_layers;
+        var layerVisibilityFilter = [];
+
+        template.layergroup.layers.forEach(function (layer, index) {
+            if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
+                layerVisibilityFilter.push(''+index);
+            }
+        });
+
+        if (!layerVisibilityFilter.length) {
+            return callback();
+        }
+
+        // overwrites 'all' default filter
+        req.params.layer = layerVisibilityFilter.join(',');
+
+        // recreates the provider
+        self.namedMapProviderCache.get(
+            user,
+            req.params.template_id,
+            req.query.config,
+            req.query.auth_token,
+            req.params,
+            callback
+        );
+    });
+};
+
 var DEFAULT_ZOOM_CENTER = {
     zoom: 1,
     center: {

lib/cartodb/controllers/named_maps_admin.js

@@ -91,9 +91,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
 NamedMapsAdminController.prototype.retrieve = function(req, res) {
     var self = this;
 
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.get_template');
-    }
+    req.profiler.start('windshaft-cartodb.get_template');
 
     var cdbuser = req.context.user;
     var tpl_id;
@@ -127,9 +125,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
 NamedMapsAdminController.prototype.destroy = function(req, res) {
     var self = this;
 
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.delete_template');
-    }
+    req.profiler.start('windshaft-cartodb.delete_template');
 
     var cdbuser = req.context.user;
     var tpl_id;
@@ -154,9 +150,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
 
 NamedMapsAdminController.prototype.list = function(req, res) {
     var self = this;
-    if ( req.profiler ) {
-        req.profiler.start('windshaft-cartodb.get_template_list');
-    }
+    req.profiler.start('windshaft-cartodb.get_template_list');
 
     var cdbuser = req.context.user;
 

lib/cartodb/middleware/allow-query-params.js

@@ -0,0 +1,9 @@
+module.exports = function allowQueryParams(params) {
+    if (!Array.isArray(params)) {
+        throw new Error('allowQueryParams must receive an Array of params');
+    }
+    return function allowQueryParamsMiddleware(req, res, next) {
+        req.context.allowedQueryParams = params;
+        next();
+    };
+};

lib/cartodb/models/dataview/aggregation.js

@@ -19,25 +19,38 @@ var rankedCategoriesQueryTpl = dot.template([
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
     '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM ({{=it._query}}) _cdb_categories',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
     'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
+    '       count, categories_count',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
 ].join('\n'));
@@ -45,7 +58,7 @@ var rankedAggregationQueryTpl = dot.template([
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
     '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
     'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
     'ORDER BY value DESC'
 ].join('\n'));
@@ -114,24 +127,10 @@ Aggregation.prototype.sql = function(psql, override, callback) {
     var _query = this.query;
 
     var aggregationSql;
+
     if (!!override.ownFilter) {
         aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql()
-                }),
-                categoriesSummaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
+            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
             aggregationQueryTpl({
                 _query: _query,
                 _column: this.column,
@@ -141,25 +140,11 @@ Aggregation.prototype.sql = function(psql, override, callback) {
         ].join('\n');
     } else {
         aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql()
-                }),
-                categoriesSummaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
+            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
             rankedAggregationQueryTpl({
                 _query: _query,
                 _column: this.column,
+                _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
                 _limit: CATEGORIES_LIMIT
             })
         ].join('\n');
@@ -170,6 +155,32 @@ Aggregation.prototype.sql = function(psql, override, callback) {
     return callback(null, aggregationSql);
 };
 
+Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
+    return [
+                "WITH",
+                [
+                    summaryQueryTpl({
+                        _query: query,
+                        _column: column
+                    }),
+                    rankedCategoriesQueryTpl({
+                        _query: query,
+                        _column: column,
+                        _aggregation: this.getAggregationSql(),
+                        _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
+                    }),
+                    categoriesSummaryMinMaxQueryTpl({
+                        _query: query,
+                        _column: column
+                    }),
+                    categoriesSummaryCountQueryTpl({
+                        _query: query,
+                        _column: column
+                    })
+                ].join(',\n')
+            ].join('\n');
+};
+
 var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
 Aggregation.prototype.getAggregationSql = function() {
     return aggregationFnQueryTpl({

lib/cartodb/models/dataview/factory.js

@@ -11,7 +11,7 @@ var DataviewFactory = {
         if (!this.dataviews[type]) {
             throw new Error('Invalid dataview type: "' + type + '"');
         }
-        return new this.dataviews[type](query, dataviewDefinition.options);
+        return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql);
     }
 };
 

lib/cartodb/models/dataview/histogram.js

@@ -109,12 +109,13 @@ var TYPE = 'histogram';
      }
  }
  */
-function Histogram(query, options) {
+function Histogram(query, options, queries) {
     if (!_.isString(options.column)) {
         throw new Error('Histogram expects `column` in widget options');
     }
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.bins = options.bins;
 
@@ -143,7 +144,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
     var _column = this.column;
 
     var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.query
+        column: _column, query: this.queries.no_filters
     });
 
     if (this._columnType === null) {

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -18,25 +18,37 @@ var rankedCategoriesQueryTpl = dot.template([
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
     '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM ({{=it._query}}) _cdb_categories',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
     'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
     'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
 ].join('\n'));
@@ -44,7 +56,7 @@ var rankedAggregationQueryTpl = dot.template([
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
     '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
     'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
     'ORDER BY value DESC'
 ].join('\n'));
@@ -65,7 +77,7 @@ Aggregation.prototype.constructor = Aggregation;
 
 module.exports = Aggregation;
 
-Aggregation.prototype.sql = function(psql, filters, override, callback) {
+Aggregation.prototype.sql = function(psql, override, callback) {
     if (!callback) {
         callback = override;
         override = {};
@@ -85,9 +97,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
@@ -110,9 +127,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })

lib/cartodb/models/dataview/overviews/base.js

@@ -55,20 +55,20 @@ BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
 };
 
 // Default behaviour
-BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
+BaseOverviewsDataview.prototype.defaultSql = function(psql, override, callback) {
     var query = this.query;
     var dataview = this.baseDataview;
     if ( SETTINGS.defaultOverviews ) {
         query = this.rewrittenQuery(query);
         dataview = new this.BaseDataview(query, this.queryOptions);
     }
-    return dataview.sql(psql, filters, override, callback);
+    return dataview.sql(psql, override, callback);
 };
 
 // default implementation that can be override in derived classes:
 
-BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
-    return this.defaultSql(psql, filters, override, callback);
+BaseOverviewsDataview.prototype.sql = function(psql, override, callback) {
+    return this.defaultSql(psql, override, callback);
 };
 
 BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {

lib/cartodb/models/dataview/overviews/factory.js

@@ -14,7 +14,8 @@ OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinit
         return parentFactory.getDataview(query, dataviewDefinition);
     }
     return new dataviews[type](
-        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
+        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options,
+        dataviewDefinition.sql
     );
 };
 

lib/cartodb/models/dataview/overviews/formula.js

@@ -36,7 +36,7 @@ Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-Formula.prototype.sql = function(psql, filters, override, callback) {
+Formula.prototype.sql = function(psql, override, callback) {
     var formulaQueryTpl = formulaQueryTpls[this.operation];
 
     if ( formulaQueryTpl ) {
@@ -52,5 +52,5 @@ Formula.prototype.sql = function(psql, filters, override, callback) {
     }
 
     // default behaviour
-    return this.defaultSql(psql, filters, override, callback);
+    return this.defaultSql(psql, override, callback);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -8,7 +8,6 @@ dot.templateSettings.strip = false;
 var columnTypeQueryTpl = dot.template(
     'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
 );
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
 
 var BIN_MIN_NUMBER = 6;
 var BIN_MAX_NUMBER = 48;
@@ -97,10 +96,11 @@ var histogramQueryTpl = dot.template([
     'ORDER BY bin'
 ].join('\n'));
 
-function Histogram(query, options, queryRewriter, queryRewriteData, params) {
+function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
     BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.bins = options.bins;
 
@@ -130,7 +130,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
     var _column = this.column;
 
     var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.rewrittenQuery(this.query)
+        column: _column, query: this.rewrittenQuery(this.queries.no_filters)
     });
 
     if (this._columnType === null) {
@@ -149,7 +149,9 @@ Histogram.prototype.sql = function(psql, override, callback) {
     }
 
     if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
+        // overviews currently aggregate dates to NULL
+        // to avoid problem we don't use overviews for histograms of date columns
+        return this.defaultSql(psql, override, callback);
     }
 
     var _query = this.rewrittenQuery(this.query);

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
                     }
                     layer.options.sql = analysisSql;
                     layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
+                    layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
                 } else {
                     missingNodesErrors.push(
                         new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@@ -330,4 +331,13 @@ function AnalysisError(message) {
     this.message = message;
 }
 
+function getAllAffectedTablesFromSourceNodes(node) {
+    var affectedTables = node.getAllInputNodes(function (node) {
+        return node.getType() === 'source';
+    }).reduce(function(list, node) {
+        return list.concat(node.getAffectedTables());
+    },[]);
+    return affectedTables;
+}
+
 require('util').inherits(AnalysisError, Error);

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -33,7 +33,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf
                step(
                    function collectFiltersData() {
                        var filters, unfiltered_query;
-                       if ( layer.options.source && analysesResults ) {
+                       if ( layer.options.source && analysesResults && !layer.options.sql_wrap) {
                            var sourceId = layer.options.source.id;
                            var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
                            if ( node ) {

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -4,6 +4,13 @@ var dot = require('dot');
 dot.templateSettings.strip = false;
 var queue = require('queue-async');
 var PSQL = require('cartodb-psql');
+/**
+ * cartodb-psql creates `global.Promise` as an empty constructor.
+ * However, `turbo-carto` relies on a polyfil that fails to create the polyfil
+ * as it finds `global.Promise` but it doesn't find `Promise.resolve`.
+ */
+global.Promise = global.Promise || function() {};
+global.Promise.resolve = global.Promise.resolve || function() {};
 var turboCarto = require('turbo-carto');
 
 var SubstitutionTokens = require('../../../utils/substitution-tokens');
@@ -48,6 +55,11 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
         }
 
         requestMapConfig.layers = results.map(function(result) { return result.layer; });
+        context.turboCarto = {
+            layers: results.map(function(result) {
+                return result.meta;
+            })
+        };
 
         return callback(null, requestMapConfig);
     });
@@ -84,7 +96,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
     }
 
     var pg = new PSQL(dbParamsFromReqParams(params));
-    function processCallback(err, cartocss) {
+    function processCallback(err, cartocss, meta) {
         // Only return turbo-carto errors
         if (err && err.name === 'TurboCartoError') {
             var error = new Error(err.message);
@@ -105,7 +117,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
         if (cartocss) {
             layer.options.cartocss = cartocss;
         }
-        return callback(null, { layer: layer });
+        return callback(null, { layer: layer, meta: meta });
     }
 
     var layerSql = layer.options.sql;

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -120,6 +120,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
             self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
             self.analysesResults = context.analysesResults || [];
             self.rendererParams = rendererParams;
+            self.context = context;
             self.context.limits = renderLimits || {};
             return callback(self.err, self.mapConfig, self.rendererParams, self.context);
         }

lib/cartodb/models/resource-locator.js

@@ -0,0 +1,119 @@
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function ResourceLocator(environment) {
+    this.environment = environment;
+
+    this.resourcesUrlTemplates = null;
+    if (this.environment.resources_url_templates) {
+        var templates = environment.resources_url_templates;
+
+        if (templates.http) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
+        }
+        if (templates.https) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
+        }
+    }
+}
+
+module.exports = ResourceLocator;
+
+ResourceLocator.prototype.getUrls = function(username, resource) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, resource);
+    }
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource);
+    if (cdnDomain) {
+        return {
+            http: 'http://' + cdnDomain.http + '/' + username + '/api/v1/map/' + resource,
+            https: 'https://' + cdnDomain.https + '/' + username + '/api/v1/map/' + resource
+        };
+    } else {
+        var port = this.environment.port;
+        return {
+            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
+        };
+    }
+};
+
+
+ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource) {
+    var urls = {};
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = this.resourcesUrlTemplates.http({
+            cdn_url: cdnDomain.http,
+            user: username,
+            port: this.environment.port,
+            resource: resource
+        });
+    }
+
+    if (this.resourcesUrlTemplates.https) {
+        urls.https = this.resourcesUrlTemplates.https({
+            cdn_url: cdnDomain.https,
+            user: username,
+            port: this.environment.port,
+            resource: resource
+        });
+    }
+
+    return urls;
+};
+
+function getCdnDomain(serverMetadata, resource) {
+    if (serverMetadata && serverMetadata.cdn_url) {
+        var cdnUrl = serverMetadata.cdn_url;
+        var http = cdnUrl.http;
+        var https = cdnUrl.https;
+        if (cdnUrl.templates) {
+            var templates = cdnUrl.templates;
+            var httpUrlTemplate = templates.http.url;
+            var httpsUrlTemplate = templates.https.url;
+            http = httpUrlTemplate
+                .replace(/^(http[s]*:\/\/)/, '')
+                .replace('{s}', subdomain(templates.http.subdomains, resource));
+            https = httpsUrlTemplate
+                .replace(/^(http[s]*:\/\/)/, '')
+                .replace('{s}', subdomain(templates.https.subdomains, resource));
+        }
+        return {
+            http: http,
+            https: https,
+        };
+    }
+    return null;
+}
+
+// ref https://jsperf.com/js-crc32
+function crcTable() {
+    var c;
+    var table = [];
+    for (var n = 0; n < 256; n++) {
+      c = n;
+      for (var k = 0; k < 8; k++) {
+        c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+      }
+      table[n] = c;
+    }
+    return table;
+}
+var CRC_TABLE = crcTable();
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+      crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return (crc ^ (-1)) >>> 0;
+}
+
+function subdomain(subdomains, resource) {
+    var index = crc32(resource) % subdomains.length;
+    return subdomains[index];
+}
+module.exports.subdomain = subdomain;

lib/cartodb/server.js

@@ -147,7 +147,8 @@ module.exports = function(serverOptions) {
     var tileBackend = new windshaft.backend.Tile(rendererCache);
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

lib/cartodb/server_options.js

@@ -36,7 +36,11 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
         inlineExecution: false,
         endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
         hostHeaderTemplate: '{{=it.username}}.localhost.lan'
-    }
+    },
+    logger: {
+        filename: undefined
+    },
+    limits: {}
 });
 
 module.exports = {
@@ -66,6 +70,7 @@ module.exports = {
         },
         datasource: global.environment.postgres,
         cachedir: global.environment.millstone.cache_basedir,
+        use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
         mapnik_version: global.environment.mapnik_version,
         mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
         default_layergroup_ttl: global.environment.mapConfigTTL || 7200
@@ -95,7 +100,11 @@ module.exports = {
             inlineExecution: analysisConfig.batch.inlineExecution,
             endpoint: analysisConfig.batch.endpoint,
             hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate
-        }
+        },
+        logger: {
+            filename: analysisConfig.logger.filename
+        },
+        limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.65.0",
+  "version": "3.6.3",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,33 +20,34 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.37.0",
-    "cartodb-psql": "~0.6.1",
-    "cartodb-query-tables": "~0.1.0",
-    "cartodb-redis": "0.13.1",
+    "camshaft": "0.54.1",
+    "cartodb-psql": "~0.7.1",
+    "cartodb-query-tables": "0.2.0",
+    "cartodb-redis": "0.13.2",
     "debug": "~2.2.0",
     "dot": "~1.0.2",
     "express": "~4.13.3",
     "fastly-purge": "~1.0.1",
-    "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
+    "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "2.6.5",
     "lzma": "~2.3.2",
     "node-statsd": "~0.0.7",
     "queue-async": "~1.0.7",
-    "redis-mpool": "~0.4.0",
-    "request": "~2.62.0",
+    "redis-mpool": "0.4.1",
+    "request": "~2.79.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.14.0",
+    "turbo-carto": "0.19.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.4.0"
+    "windshaft": "3.1.1",
+    "yargs": "~5.0.0"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",
     "jshint": "~2.6.0",
     "mocha": "~1.21.4",
     "nock": "~2.11.0",
-    "redis": "~0.8.6",
+    "redis": "~0.12.1",
     "semver": "~1.1.4",
     "strftime": "~0.8.2"
   },
@@ -55,7 +56,7 @@
     "test": "make test-all"
   },
   "engines": {
-    "node": ">=0.8 <0.11",
-    "npm": ">=2.14.16"
+    "node": ">=6.9",
+    "yarn": "^0.21.3"
   }
 }

scripts/install.sh

@@ -4,4 +4,4 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
     export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig
 fi
 
-npm install
+yarn

test/acceptance/cache/cache_headers.js

@@ -0,0 +1,393 @@
+var testHelper = require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var qs = require('querystring');
+
+var CartodbWindshaft = require('../../../lib/cartodb/server');
+var serverOptions = require('../../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+server.setMaxListeners(0);
+
+var LayergroupToken = require('../../support/layergroup-token');
+
+describe('get requests with cache headers', function() {
+
+    var keysToDelete;
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        testHelper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    var statusOkResponse = {
+        status: 200
+    };
+
+    var mapConfigs = [
+        {
+            "description": "cache headers should be present",
+            "cache_headers": {
+                "x_cache_channel": {
+                    "db_name": "test_windshaft_cartodb_user_1_db",
+                    "tables": ["public.test_table"]
+                },
+                "surrogate_keys": "t:77pJnX"
+            },
+            "data":
+                {
+                    version: '1.5.0',
+                    layers: [
+                        {
+                            options: {
+                                source: {
+                                    id: "2570e105-7b37-40d2-bdf4-1af889598745"
+                                },
+                                sql: 'select * from test_table limit 2',
+                                cartocss: '#layer { marker-fill:red; }',
+                                cartocss_version: '2.3.0',
+                                attributes: {
+                                    id:'cartodb_id',
+                                    columns: [
+                                        'name',
+                                        'address'
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    analyses: [
+                        {
+                            "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from test_table limit 2"
+                            }
+                        }
+                    ]
+                },
+        },
+        {
+            "description": "cache headers should be present and be composed with source table name",
+            "cache_headers": {
+                "x_cache_channel": {
+                    "db_name": "test_windshaft_cartodb_user_1_db",
+                    "tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d",
+                               "public.test_table"]
+                },
+                "surrogate_keys": "t:77pJnX t:iL4eth"
+            },
+            "data":
+            {
+                version: '1.5.0',
+                layers: [
+                    {
+                        options: {
+                            source: {
+                                id: "2570e105-7b37-40d2-bdf4-1af889598745"
+                            },
+                            sql: 'select * from test_table limit 2',
+                            cartocss: '#layer { marker-fill:red; }',
+                            cartocss_version: '2.3.0',
+                            attributes: {
+                                id:'cartodb_id',
+                                columns: [
+                                    'name',
+                                    'address'
+                                ]
+                            }
+                        }
+                    }
+                ],
+                analyses: [
+                    {
+                        "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                        "type": "buffer",
+                        "params": {
+                            "source": {
+                                "type": "source",
+                                "params": {
+                                    "query": "select * from test_table limit 2"
+                                }
+                            },
+                            "radius": 50000
+                        }
+                    }
+                ]
+            }
+        }];
+
+    var layergroupRequest = function(mapConfig) {
+        return {
+            url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)),
+            method: 'GET',
+            headers: {
+                host: 'localhost'
+            }
+        };
+    };
+
+    function getRequest(url, addApiKey, callbackName) {
+        var params = {};
+        if (!!addApiKey) {
+            params.api_key = '1234';
+        }
+        if (!!callbackName) {
+            params.callback = callbackName;
+        }
+
+        return {
+            url: url + '?' + qs.stringify(params),
+            method: 'GET',
+            headers: {
+                host: 'localhost',
+                'Content-Type': 'application/json'
+            }
+        };
+    }
+
+    function validateCacheHeaders(done, expectedCacheHeaders) {
+        return function(res, err) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(res.headers['x-cache-channel']);
+            assert.ok(res.headers['surrogate-key']);
+            if (expectedCacheHeaders) {
+                validateXChannelHeaders(res.headers, expectedCacheHeaders);
+                assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys);
+            }
+
+            done();
+        };
+    }
+
+    function validateXChannelHeaders(headers, expectedCacheHeaders) {
+        var dbName = headers['x-cache-channel'].split(':')[0];
+        var tables = headers['x-cache-channel'].split(':')[1].split(',').sort();
+        assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name);
+        assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort());
+    }
+
+    function noCacheHeaders(done) {
+        return function(res, err) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(
+                !res.headers['x-cache-channel'],
+                'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
+            );
+            assert.ok(
+                !res.headers['surrogate-key'],
+                'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`'
+            );
+            done();
+        };
+    }
+
+    function withLayergroupId(mapConfig, callback) {
+        assert.response(
+            server,
+            layergroupRequest(mapConfig),
+            statusOkResponse,
+            function(res, err) {
+                if (err) {
+                    return callback(err);
+                }
+                var layergroupId = JSON.parse(res.body).layergroupid;
+                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                callback(null, layergroupId, res);
+            }
+        );
+    }
+
+    mapConfigs.forEach(function(mapConfigData) {
+        describe(mapConfigData.description, function() {
+            var mapConfig = mapConfigData.data;
+            var expectedCacheHeaders = mapConfigData.cache_headers;
+            it('/api/v1/map Map instantiation', function(done) {
+                var testFn = validateCacheHeaders(done, expectedCacheHeaders);
+                withLayergroupId(mapConfig, function(err, layergroupId, res) {
+                    testFn(res);
+                });
+            });
+
+            it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+        });
+    });
+
+    describe('cache headers should NOT be present', function() {
+
+        it('/', function(done) {
+            assert.response(
+                server,
+                getRequest('/'),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        it('/version', function(done) {
+            assert.response(
+                server,
+                getRequest('/version'),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        it('/health', function(done) {
+            assert.response(
+                server,
+                getRequest('/health'),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        it('/api/v1/map/named list named maps', function(done) {
+            assert.response(
+                server,
+                getRequest('/api/v1/map/named', true),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        describe('with named maps', function() {
+
+            var templateName = 'x_cache';
+
+            beforeEach(function(done) {
+                var template =  {
+                    version: '0.0.1',
+                    name: templateName,
+                    auth: {
+                        method: 'open'
+                    },
+                    layergroup:  mapConfigs[0].data
+                };
+
+                var namedMapRequest = {
+                    url: '/api/v1/map/named?api_key=1234',
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(template)
+                };
+
+                assert.response(
+                    server,
+                    namedMapRequest,
+                    statusOkResponse,
+                    function(res, err) {
+                        done(err);
+                    }
+                );
+            });
+
+            afterEach(function(done) {
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + templateName + '?api_key=1234',
+                        method: 'DELETE',
+                        headers: {
+                            host: 'localhost'
+                        }
+                    },
+                    {
+                        status: 204
+                    },
+                    function(res, err) {
+                        done(err);
+                    }
+                );
+            });
+
+
+            it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
+                assert.response(
+                    server,
+                    getRequest('/api/v1/map/named/' + templateName, true),
+                    statusOkResponse,
+                    noCacheHeaders(done)
+                );
+            });
+
+            it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
+                assert.response(
+                    server,
+                    getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
+                    statusOkResponse,
+                    noCacheHeaders(done)
+                );
+            });
+        });
+    });
+});

test/acceptance/dataviews/aggregation.js

@@ -1,9 +1,8 @@
 require('../../support/test_helper');
-
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
-describe('aggregations', function() {
+describe('aggregations happy cases', function() {
 
     afterEach(function(done) {
         if (this.testClient) {
@@ -13,30 +12,36 @@ describe('aggregations', function() {
         }
     });
 
-    function aggregationOperationMapConfig(operation) {
-        return {
+    function aggregationOperationMapConfig(operation, query, column, aggregationColumn) {
+        column = column || 'adm0name';
+        aggregationColumn = aggregationColumn || 'pop_max';
+        query = query || 'select * from populated_places_simple_reduced';
+
+        var mapConfig = {
             version: '1.5.0',
             layers: [
                 {
                     type: 'mapnik',
                     options: {
-                        sql: 'select * from populated_places_simple_reduced',
+                        sql: query,
                         cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
                         cartocss_version: '2.0.1',
-                        widgets: {
-                            adm0name: {
-                                type: 'aggregation',
-                                options: {
-                                    column: 'adm0name',
-                                    aggregation: operation,
-                                    aggregationColumn: 'pop_max'
-                                }
-                            }
-                        }
+                        widgets: {}
                     }
                 }
             ]
         };
+
+        mapConfig.layers[0].options.widgets[column] = {
+            type: 'aggregation',
+            options: {
+                column: column,
+                aggregation: operation,
+                aggregationColumn: aggregationColumn
+            }
+        };
+
+        return mapConfig;
     }
 
     var operations = ['count', 'sum', 'avg', 'max', 'min'];
@@ -56,4 +61,88 @@ describe('aggregations', function() {
             });
         });
     });
+
+    var query = [
+        'select 1 as val, \'a\' as cat, ST_Transform(ST_SetSRID(ST_MakePoint(0,0),4326),3857) as the_geom_webmercator',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(0,1),4326),3857)',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(1,0),4326),3857)',
+        'select null, null, ST_Transform(ST_SetSRID(ST_MakePoint(1,1),4326),3857)'
+    ].join(' UNION ALL ');
+
+    operations.forEach(function (operation) {
+        var not = operation === 'count' ? ' not ' : ' ';
+        var description = 'should' +
+            not +
+            'handle NULL values in category and aggregation columns using "' +
+            operation +
+            '" as aggregation operation';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 3);
+                assert.equal(aggregation.count, 4);
+                assert.equal(aggregation.nulls, 1);
+
+                var hasNullCategory = false;
+                aggregation.categories.forEach(function (category) {
+                    if (category.category === null) {
+                        hasNullCategory = true;
+                    }
+                });
+
+                if (operation === 'count') {
+                    assert.ok(hasNullCategory, 'aggregation has not a category NULL');
+                } else {
+                    assert.ok(!hasNullCategory, 'aggregation has category NULL');
+                }
+
+                done();
+            });
+        });
+    });
+
+    var operations_and_values = {'count': 9, 'sum': 45, 'avg': 5, 'max': 9, 'min': 1};
+
+    var query_other = [
+        'select generate_series(1,3) as val, \'other_a\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(4,6) as val, \'other_b\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(7,9) as val, \'other_c\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_1\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_2\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_3\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_4\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_5\' as cat, NULL as the_geom_webmercator'
+    ].join(' UNION ALL ');
+
+    Object.keys(operations_and_values).forEach(function (operation) {
+        var description = 'should aggregate OTHER category using "' + operation + '"';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 8);
+                assert.equal(aggregation.count, 24);
+                assert.equal(aggregation.nulls, 0);
+
+                var aggregated_categories = aggregation.categories.filter( function(category) {
+                    return category.agg === true;
+                });
+                assert.equal(aggregated_categories.length, 1);
+                assert.equal(aggregated_categories[0].value, operations_and_values[operation]);
+
+                done();
+            });
+        });
+    });
 });

test/acceptance/dataviews/histogram.js

@@ -77,4 +77,24 @@ describe('histogram-dataview', function() {
             done();
         });
     });
+
+    it('should cast all overridable params to numbers', function(done) {
+        var params = {
+            bins: '256 AS other, (select 256 * 2) AS bins_number--',
+            start: 1e3,
+            end: 0,
+            response: TestClient.RESPONSE.ERROR
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('pop_max_histogram', params, function(err, res) {
+            assert.ok(!err, err);
+
+            assert.ok(res.errors);
+            assert.equal(res.errors.length, 1);
+            assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
+
+            done();
+        });
+    });
 });

test/acceptance/dataviews/overviews.js

@@ -144,6 +144,22 @@ describe('dataviews using tables with overviews', function() {
                     aggregationColumn: 'name',
                 }
             },
+            test_histogram: {
+                type: 'histogram',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    bins: 2
+                }
+            },
+            test_histogram_date: {
+                type: 'histogram',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'updated_at',
+                    bins: 2
+                }
+            },
             test_avg: {
                 type: 'formula',
                 source: {id: 'data-source'},
@@ -265,8 +281,83 @@ describe('dataviews using tables with overviews', function() {
         });
     });
 
+    it("should expose a histogram", function (done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_histogram', function (err, histogram) {
+            if (err) {
+                return done(err);
+            }
+            assert.ok(histogram);
+            assert.equal(histogram.type, 'histogram');
+            assert.ok(Array.isArray(histogram.bins));
+            testClient.drain(done);
+        });
+    });
+
     describe('filters', function() {
 
+        describe('histogram', function () {
+
+            it("should expose a filtered histogram", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram: { min: 2 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 4);
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered histogram with no results", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram: { max: -1 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 0);
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered date histogram with no results", function (done) {
+                // This most likely works because the overviews will pass
+                // the responsibility to the normal dataviews.
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram_date: { max: -1 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram_date', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 0);
+                    testClient.drain(done);
+                });
+            });
+        });
+
         describe('category', function () {
 
             var params = {

test/acceptance/errors-with-context.js

@@ -0,0 +1,68 @@
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+describe('error with context', function () {
+    var layerOK = {
+        options: {
+            sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator ' +
+                 'from test_table',
+            cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.1',
+            interactivity: 'cartodb_id'
+        }
+    };
+    var layerKO = {
+        options: {
+            sql: 'select cartodb_id from test_table offset 3', // it doesn't return the_geom_webmercator so it must fail
+            cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.2',
+            interactivity: 'cartodb_id'
+        }
+    };
+
+    var DB_ERROR_MESSAGE = 'Postgis Plugin: ERROR:  column "the_geom_webmercator" does not exist';
+    var scenarios = [{
+        description: 'layergroup with 2 layers, second one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerOK, layerKO]
+        },
+        expectedFailingLayer: { id: 'layer1', index: 1, type: 'mapnik' }
+    }, {
+        description: 'layergroup with 2 layers, first one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerKO, layerOK]
+        },
+        expectedFailingLayer: { id: 'layer0', index: 0, type: 'mapnik' }
+    }];
+
+    scenarios.forEach(function (scenario) {
+        it(scenario.description, function (done) {
+            assert.response(server, {
+                url: '/api/v1/map',
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(scenario.layergroup)
+            }, {
+                status: 400
+            }, function (res) {
+                var parsedBody = JSON.parse(res.body);
+
+                assert.ok(Array.isArray(parsedBody.errors_with_context));
+
+                var err = parsedBody.errors_with_context[0];
+                assert.equal(err.type, 'layer');
+                assert.equal(err.subtype, 'query');
+                assert.ok(err.message.indexOf(DB_ERROR_MESSAGE) >= 0);
+                assert.deepEqual(err.layer, scenario.expectedFailingLayer);
+                done();
+            });
+        });
+    });
+});

test/acceptance/multilayer.js

@@ -5,6 +5,9 @@ var step        = require('step');
 var strftime    = require('strftime');
 var redis_stats_db = 5;
 
+var mapnik = require('windshaft').mapnik;
+var semver = require('semver');
+
 var helper = require(__dirname + '/../support/test_helper');
 var LayergroupToken = require('../support/layergroup-token');
 
@@ -574,7 +577,7 @@ describe(suiteName, function() {
             if ( err ) {
               return done(err);
             }
-            // trip epoch
+            // strip epoch
             expected_token = expected_token.split(':')[0];
             keysToDelete['map_cfg|' + expected_token] = 0;
             keysToDelete['user:localhost:mapviews:global'] = 5;
@@ -973,70 +976,72 @@ describe(suiteName, function() {
     });
 
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/93
-    it("accepts unused directives", function(done) {
-      var layergroup =  {
-        version: '1.0.0',
-        layers: [
-           { options: {
-               sql: "select 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
-               cartocss: '#layer { point-transform:"scale(20)"; }',
-               cartocss_version: '2.0.1'
-             } }
-        ]
-      };
-      var expected_token; // = "e34dd7e235138a062f8ba7ad051aa3a7";
-      step(
-        function do_post()
-        {
-          var next = this;
-          assert.response(server, {
-              url: layergroup_url,
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' },
-              data: JSON.stringify(layergroup)
-          }, {}, function(res) {
-              assert.equal(res.statusCode, 200, res.body);
-              var parsedBody = JSON.parse(res.body);
-              if ( expected_token ) {
-                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
-                assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
-              }
-              else {
-                var token_components = parsedBody.layergroupid.split(':');
-                expected_token = token_components[0];
-                expected_last_updated_epoch = token_components[1];
-              }
-              next(null, res);
-          });
-        },
-        function do_get_tile(err)
-        {
-            assert.ifError(err);
-          var next = this;
-          assert.response(server, {
-              url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
-              method: 'GET',
-              headers: {host: 'localhost' },
-              encoding: 'binary'
-          }, {}, function(res) {
-              assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "image/png");
-              assert.imageBufferIsSimilarToFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
-                  IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
-                      next(err);
+    if (semver.satisfies(mapnik.versions.mapnik, '2.3.x')) {
+        it("accepts unused directives", function(done) {
+          var layergroup =  {
+            version: '1.0.0',
+            layers: [
+               { options: {
+                   sql: "select 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+                   cartocss: '#layer { point-transform:"scale(20)"; }',
+                   cartocss_version: '2.0.1'
+                 } }
+            ]
+          };
+          var expected_token; // = "e34dd7e235138a062f8ba7ad051aa3a7";
+          step(
+            function do_post()
+            {
+              var next = this;
+              assert.response(server, {
+                  url: layergroup_url,
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' },
+                  data: JSON.stringify(layergroup)
+              }, {}, function(res) {
+                  assert.equal(res.statusCode, 200, res.body);
+                  var parsedBody = JSON.parse(res.body);
+                  if ( expected_token ) {
+                    assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+                    assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
                   }
-              );
-          });
-        },
-        function finish(err) {
-            keysToDelete['user:localhost:mapviews:global'] = 5;
-            keysToDelete['map_cfg|' + expected_token] = 0;
-
-            done(err);
-        }
-      );
-    });
+                  else {
+                    var token_components = parsedBody.layergroupid.split(':');
+                    expected_token = token_components[0];
+                    expected_last_updated_epoch = token_components[1];
+                  }
+                  next(null, res);
+              });
+            },
+            function do_get_tile(err)
+            {
+                assert.ifError(err);
+              var next = this;
+              assert.response(server, {
+                  url: layergroup_url + "/" + expected_token + ':cb0/0/0/0.png',
+                  method: 'GET',
+                  headers: {host: 'localhost' },
+                  encoding: 'binary'
+              }, {}, function(res) {
+                  assert.equal(res.statusCode, 200, res.body);
+                  assert.equal(res.headers['content-type'], "image/png");
+                  assert.imageBufferIsSimilarToFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
+                      IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
+                          next(err);
+                      }
+                  );
+              });
+            },
+            function finish(err) {
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                keysToDelete['map_cfg|' + expected_token] = 0;
 
+                done(err);
+            }
+          );
+        });
+    }
+    
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/91
     // and https://github.com/CartoDB/Windshaft-cartodb/issues/38
     it("tiles for private tables can be fetched with api_key", function(done) {

test/acceptance/multilayer_server.js

@@ -228,7 +228,9 @@ describe('tests from old api translated to multilayer', function() {
                 },
                 function(res) {
                     var parsed = JSON.parse(res.body);
-                    assert.deepEqual(parsed.errors, [ 'Unexpected token W' ]);
+                    assert.ok(parsed.errors);
+                    assert.equal(parsed.errors.length, 1);
+                    assert.ok(parsed.errors[0].match(/^Unexpected token W/));
 
                     done();
                 }

test/acceptance/named_layers_visibility.js

@@ -0,0 +1,210 @@
+var step = require('step');
+var test_helper = require('../support/test_helper');
+
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var RedisPool = require('redis-mpool');
+var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');
+var mapnik = require('windshaft').mapnik;
+
+var IMAGE_TOLERANCE = 20;
+
+describe('layers visibility for previews', function() {
+    // configure redis pool instance to use in tests
+    var redisPool = new RedisPool(global.environment.redis);
+    var templateMaps = new TemplateMaps(redisPool, {
+        max_user_templates: global.environment.maxUserTemplates
+    });
+
+    var username = 'localhost';
+
+    function createLayer (color, layerId) {
+        var mod;
+
+        if (color === 'red') {
+            mod = 0;
+        } else if (color === 'orange') {
+            mod = 1;
+        } else if (color === 'blue') {
+            mod = 2;
+        } else {
+            mod = 0;
+        }
+
+        return {
+            type: 'mapnik',
+            id: layerId,
+            options: {
+                sql: 'select * from populated_places_simple_reduced where cartodb_id % 3 = ' + mod,
+                cartocss: '#layer { marker-fill: ' + color + '; }',
+                cartocss_version: '2.3.0'
+            }
+        };
+    }
+
+    function createTemplate(scenario) {
+        return {
+            version: '0.0.1',
+            name: scenario.name,
+            auth: {
+                method: 'open'
+            },
+            view: {
+                bounds: {
+                    west: 0,
+                    south: 0,
+                    east: 45,
+                    north: 45
+                },
+                zoom: 4,
+                center: {
+                    lng: 40,
+                    lat: 20
+                },
+                preview_layers: scenario.layerPerview
+            },
+            layergroup: {
+                layers: scenario.layers
+            }
+        };
+    }
+
+    afterEach(function (done) {
+        test_helper.deleteRedisKeys({
+            'user:localhost:mapviews:global': 5
+        }, done);
+    });
+
+    function previewFixture(version) {
+        return './test/fixtures/previews/populated_places_simple_reduced-' + version + '.png';
+    }
+
+    var threeLayerPointDistintColor = [
+        createLayer('red'),
+        createLayer('orange'),
+        createLayer('blue', 'layer2')
+    ];
+
+    var scenarios = [{
+        name: 'preview_layers_red',
+        layerPerview: {
+            '0': true,
+            '1': false,
+            'layer2': false
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_orange',
+        layerPerview: {
+            '0': false,
+            '1': true,
+            'layer2': false
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_blue',
+        layerPerview: {
+            '0': false,
+            '1': false,
+            'layer2': true
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_red_orange',
+        layerPerview: {
+            '0': true,
+            '1': true,
+            'layer2': false
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_red_blue',
+        layerPerview: {
+            '0': true,
+            '1': false,
+            'layer2': true
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_orange_blue',
+        layerPerview: {
+            '0': false,
+            '1': true,
+            'layer2': true
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_red_orange_blue',
+        layerPerview: {
+            '0': true,
+            '1': true,
+            'layer2': true
+        },
+        layers: threeLayerPointDistintColor
+    }, {
+        name: 'preview_layers_all',
+        layerPerview: {},
+        layers: threeLayerPointDistintColor
+    },  {
+        name: 'preview_layers_undefined',
+        layerPerview: undefined,
+        layers: threeLayerPointDistintColor
+    }];
+
+    scenarios.forEach(function (scenario) {
+        it('should filter layers for template: ' + scenario.name, function (done) {
+            step(
+                function addTemplate () {
+                    var next = this;
+                    var template = createTemplate(scenario);
+
+                    templateMaps.addTemplate(username, template, next);
+                },
+                function requestPreview (err) {
+                    assert.ifError(err);
+
+                    var next = this;
+
+                    assert.response(server, {
+                        url: '/api/v1/map/static/named/' + scenario.name + '/640/480.png',
+                        method: 'GET',
+                        headers: {
+                            host: 'localhost'
+                        },
+                        encoding: 'binary'
+                    }, {
+                        status: 200,
+                        headers: {
+                            'content-type': 'image/png'
+                        }
+                    }, function (res, err) {
+                        next(err, res);
+                    });
+                },
+                function checkPreview (err, res) {
+                    assert.ifError(err);
+
+                    var next = this;
+                    var img = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+                    var previewFixturePath = previewFixture(scenario.name);
+
+                    assert.imageIsSimilarToFile(img, previewFixturePath, IMAGE_TOLERANCE, next);
+                },
+                function deleteTemplate(err) {
+                    assert.ifError(err);
+
+                    var next = this;
+
+                    templateMaps.delTemplate(username, scenario.name, next);
+                },
+                function finish (err) {
+                    done(err);
+                }
+            );
+        });
+    });
+
+});

test/acceptance/named_maps_static_view.js

@@ -21,7 +21,7 @@ describe('named maps static view', function() {
 
     var IMAGE_TOLERANCE = 20;
 
-    function createTemplate(view) {
+    function createTemplate(view, layers) {
         return {
             version: '0.0.1',
             name: templateName,
@@ -36,7 +36,7 @@ describe('named maps static view', function() {
             },
             view: view,
             layergroup: {
-                layers: [
+                layers: layers || [
                     {
                         type: 'mapnik',
                         options: {
@@ -198,4 +198,43 @@ describe('named maps static view', function() {
         });
     });
 
+    it('should allow to select the layers to render', function (done) {
+        var view = {
+            bounds: {
+                west: 0,
+                south: 0,
+                east: 45,
+                north: 45
+            }
+        };
+
+        var layers = [
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: '#layer { marker-fill: <%= color %>; }',
+                    cartocss_version: '2.3.0'
+                }
+            },
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select ST_Transform(ST_MakeEnvelope(-45, -45, 45, 45, 4326), 3857) the_geom_webmercator',
+                    cartocss: '#layer { polygon-fill: <%= color %>; }',
+                    cartocss_version: '2.3.0'
+                }
+            }
+        ];
+        templateMaps.addTemplate(username, createTemplate(view, layers), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ layer: 0 }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
 });

test/acceptance/ported/multilayer_error_cases.js

@@ -419,7 +419,9 @@ describe('multilayer error cases', function() {
             },
             function(res) {
                 var parsedBody = JSON.parse(res.body);
-                assert.deepEqual(parsedBody, { errors: ['SyntaxError: Unexpected token {'] });
+                assert.ok(parsedBody.errors);
+                assert.equal(parsedBody.errors.length, 1);
+                assert.ok(parsedBody.errors[0].match(/^SyntaxError: Unexpected token {/));
                 done();
             }
         );

test/acceptance/ported/server_gettile.js

@@ -139,13 +139,15 @@ describe('server_gettile', function() {
             13, 4011, 3088, imageCompareFn('test_table_13_4011_3088_styled_black.png', done));
     });
 
-    // See http://github.com/CartoDB/Windshaft/issues/99
-    it("unused directives are tolerated",  function(done){
-        var style = "#test_table{point-transform: 'scale(100)';}";
-        var sql = "SELECT 1 as cartodb_id, 'SRID=4326;POINT(0 0)'::geometry as the_geom";
-        testClient.getTile(testClient.singleLayerMapConfig(sql, style), 0, 0, 0,
-            imageCompareFn('test_default_mapnik_point.png', done));
-    });
+    if ( semver.satisfies(mapnik.versions.mapnik, '2.3.x') ) {
+        // See http://github.com/CartoDB/Windshaft/issues/99
+        it("unused directives are tolerated",  function(done){
+            var style = "#test_table{point-transform: 'scale(100)';}";
+            var sql = "SELECT 1 as cartodb_id, 'SRID=4326;POINT(0 0)'::geometry as the_geom";
+            testClient.getTile(testClient.singleLayerMapConfig(sql, style), 0, 0, 0,
+                imageCompareFn('test_default_mapnik_point.png', done));
+        });
+    }
 
     // See http://github.com/CartoDB/Windshaft/issues/100
     var test_strictness = function(done) {
@@ -168,51 +170,53 @@ describe('server_gettile', function() {
       // Strictness handling changed in 2.3.x, possibly a bug: see http://github.com/mapnik/mapnik/issues/2301
       it.skip('[skipped due to http://github.com/mapnik/mapnik/issues/2301]' + test_strict_lbl,  test_strictness);
     }
-    else {
+    else  if (!semver.satisfies(mapnik.versions.mapnik, '3.0.x')) {
       it(test_strict_lbl,  test_strictness);
     }
 
-    it('high cpu regression with mapnik <2.3.x', function(done) {
-        var sql = [
-            "SELECT 'my polygon name here' AS name,",
-            "st_envelope(st_buffer(st_transform(",
-            "st_setsrid(st_makepoint(-26.6592894004,49.7990296995),4326),3857),10000000)) AS the_geom",
-            "FROM generate_series(-6,6) x",
-            "UNION ALL",
-            "SELECT 'my marker name here' AS name,",
-            "       st_transform(st_setsrid(st_makepoint(49.6042060319,-49.0522997372),4326),3857) AS the_geom",
-            "FROM generate_series(-6,6) x"
-        ].join(' ');
+    if ( semver.satisfies(mapnik.versions.mapnik, '2.3.x') ) {
 
-        var style = [
-            '#test_table {marker-fill:#ff7;',
-            '    marker-max-error:0.447492761618;',
-            '    marker-line-opacity:0.659371340628;',
-            '    marker-allow-overlap:true;',
-            '    polygon-fill:green;',
-            '    marker-spacing:0.0;',
-            '    marker-width:4.0;',
-            '    marker-height:18.0;',
-            '    marker-opacity:0.942312062822;',
-            '    line-color:green;',
-            '    line-gamma:0.945973211092;',
-            '    line-cap:square;',
-            '    polygon-opacity:0.12576055992;',
-            '    marker-type:arrow;',
-            '    polygon-gamma:0.46354913107;',
-            '    line-dasharray:33,23;',
-            '    line-join:bevel;',
-            '    marker-placement:line;',
-            '    line-width:1.0;',
-            '    marker-line-color:#ff7;',
-            '    line-opacity:0.39403752154;',
-            '    marker-line-width:3.0;',
-            '}'
-        ].join('');
+        it('high cpu regression with mapnik <2.3.x', function(done) {
+            var sql = [
+                "SELECT 'my polygon name here' AS name,",
+                "st_envelope(st_buffer(st_transform(",
+                "st_setsrid(st_makepoint(-26.6592894004,49.7990296995),4326),3857),10000000)) AS the_geom",
+                "FROM generate_series(-6,6) x",
+                "UNION ALL",
+                "SELECT 'my marker name here' AS name,",
+                "       st_transform(st_setsrid(st_makepoint(49.6042060319,-49.0522997372),4326),3857) AS the_geom",
+                "FROM generate_series(-6,6) x"
+            ].join(' ');
 
-        testClient.getTile(testClient.singleLayerMapConfig(sql, style), 13, 4011, 3088, done);
-    });
+            var style = [
+                '#test_table {marker-fill:#ff7;',
+                '    marker-max-error:0.447492761618;',
+                '    marker-line-opacity:0.659371340628;',
+                '    marker-allow-overlap:true;',
+                '    polygon-fill:green;',
+                '    marker-spacing:0.0;',
+                '    marker-width:4.0;',
+                '    marker-height:18.0;',
+                '    marker-opacity:0.942312062822;',
+                '    line-color:green;',
+                '    line-gamma:0.945973211092;',
+                '    line-cap:square;',
+                '    polygon-opacity:0.12576055992;',
+                '    marker-type:arrow;',
+                '    polygon-gamma:0.46354913107;',
+                '    line-dasharray:33,23;',
+                '    line-join:bevel;',
+                '    marker-placement:line;',
+                '    line-width:1.0;',
+                '    marker-line-color:#ff7;',
+                '    line-opacity:0.39403752154;',
+                '    marker-line-width:3.0;',
+                '}'
+            ].join('');
 
+            testClient.getTile(testClient.singleLayerMapConfig(sql, style), 13, 4011, 3088, done);
+        });
+    }
     // https://github.com/CartoDB/Windshaft-cartodb/issues/316
     it('should return errors with better formatting', function(done) {
         var mapConfig = {

test/acceptance/ported/torque.js

@@ -370,7 +370,7 @@ describe('torque', function() {
         {
           var next = this;
           assert.response(server, {
-              url: '/database/windshaft_test/layergroup?dbport=1234567',
+              url: '/database/windshaft_test/layergroup?dbport=54777',
               method: 'POST',
               headers: {'Content-Type': 'application/json' },
               data: JSON.stringify(mapconfig)

test/acceptance/templates.js

@@ -1052,8 +1052,9 @@ describe('template_api', function() {
             'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
           var cc = res.headers['x-cache-channel'];
+          var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
           assert.ok(cc);
-          assert.ok(cc.match, /ciao/, cc);
+          assert.equal(cc, expectedCC);
           // hack simulating restart...
           server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache
           var get_request = {
@@ -1072,8 +1073,9 @@ describe('template_api', function() {
             'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
           var cc = res.headers['x-cache-channel'];
+          var expectedCC = 'test_windshaft_cartodb_user_1_db:public.test_table_private_1';
           assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
-          assert.ok(cc.match, /ciao/, cc);
+          assert.equal(cc, expectedCC);
           return null;
         },
         function deleteTemplate(err)

test/acceptance/turbo-carto/error-cases.js

@@ -50,14 +50,14 @@ describe('turbo-carto error cases', function() {
     });
 
     it('should return invalid number of ramp error', function(done) {
-        this.testClient = new TestClient(makeMapconfig('ramp([pop_max], (8,24,96), (8,24,96,128))'));
+        this.testClient = new TestClient(makeMapconfig('ramp([pop_max], 8, 96, 3, (8,24,96,128))'));
         this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
             assert.ok(!err, err);
 
             assert.ok(layergroup.hasOwnProperty('errors'));
             assert.equal(layergroup.errors.length, 1);
-            assert.ok(layergroup.errors[0].match(/^Failed\sto\sprocess/));
-            assert.ok(layergroup.errors[0].match(/invalid\sramp\slength/i));
+            assert.ok(layergroup.errors[0].match(/^Failed\sto\sprocess/), layergroup.errors[0]);
+            assert.ok(layergroup.errors[0].match(/invalid\sramp\slength/i), layergroup.errors[0]);
 
             done();
         });

test/acceptance/turbo-carto/regressions.js

@@ -287,4 +287,97 @@ describe('turbo-carto regressions', function() {
         });
     });
 
+    var scenarios = [
+        {
+            desc: 'numeric datasource',
+            cartocss: [
+                "#points {",
+                "  marker-fill: ramp([scalerank], colorbrewer(Reds), category);",
+                "}"
+            ].join('\n'),
+            expected: [
+                '#points {',
+                '  marker-fill: #fee5d9;',
+                '  [ scalerank = 6 ] {',
+                '    marker-fill: #fcae91',
+                '  }',
+                '  [ scalerank = 8 ] {',
+                '    marker-fill: #fb6a4a',
+                '  }',
+                '  [ scalerank = 4 ] {',
+                '    marker-fill: #de2d26',
+                '  }',
+                '  [ scalerank = 10 ] {',
+                '    marker-fill: #a50f15',
+                '  }',
+                '}',
+            ].join('\n')
+        },
+        {
+            desc: 'string datasource',
+            cartocss: [
+                "#points {",
+                "  marker-fill: ramp([adm0name], colorbrewer(Reds), category);",
+                "}"
+            ].join('\n'),
+            expected: [
+                '#points {',
+                '  marker-fill: #fee5d9;',
+                '  [ adm0name = "Russia" ] {',
+                '    marker-fill: #fcae91',
+                '  }',
+                '  [ adm0name = "China" ] {',
+                '    marker-fill: #fb6a4a',
+                '  }',
+                '  [ adm0name = "Brazil" ] {',
+                '    marker-fill: #de2d26',
+                '  }',
+                '  [ adm0name = "Canada" ] {',
+                '    marker-fill: #a50f15',
+                '  }',
+                '}',
+            ].join('\n')
+        },
+        {
+            desc: 'numeric manual',
+            cartocss: [
+                "#points {",
+                "  marker-fill: ramp([scalerank], colorbrewer(Reds), (-1, 6, 8, 4, 10), category);",
+                "}"
+            ].join('\n'),
+            expected: [
+                '#points {',
+                '  marker-fill: #fee5d9;',
+                '  [ scalerank = 6 ] {',
+                '    marker-fill: #fcae91',
+                '  }',
+                '  [ scalerank = 8 ] {',
+                '    marker-fill: #fb6a4a',
+                '  }',
+                '  [ scalerank = 4 ] {',
+                '    marker-fill: #de2d26',
+                '  }',
+                '  [ scalerank = 10 ] {',
+                '    marker-fill: #a50f15',
+                '  }',
+                '}',
+            ].join('\n')
+        }
+    ];
+
+    scenarios.forEach(function(scenario) {
+        it('category ramps should use original type: ' + scenario.desc, function(done) {
+            var mapConfig = makeMapconfig('SELECT * FROM populated_places_simple_reduced', scenario.cartocss);
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getLayergroup(function(err, layergroup) {
+                assert.ok(!err, err);
+
+                assert.ok(layergroup.hasOwnProperty('layergroupid'));
+                assert.deepEqual(layergroup.metadata.layers[0].meta.cartocss, scenario.expected);
+
+                done();
+            });
+        });
+    });
+
 });

test/acceptance/widgets/ported/histogram.js

@@ -304,6 +304,37 @@ describe('widgets', function() {
                 });
             });
 
+            it('can use a datetime filtered column with no results', function(done) {
+                this.testClient = new TestClient(histogramsMapConfig({
+                    updated_at: {
+                        type: 'histogram',
+                        options: {
+                            column: 'updated_at'
+                        }
+                    }
+                }));
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [{
+                            updated_at: {
+                                // this will remove all results
+                                max: -1
+                            }
+                        }]
+                    }
+                };
+                this.testClient.getWidget('updated_at', params, function (err, res, histogram) {
+                    assert.ok(!err, err);
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+
+                    assert.equal(histogram.bins.length, 0);
+
+                    done();
+                });
+            });
+
             it('can getTile with datetime filtered column', function(done) {
                 this.testClient = new TestClient(histogramsMapConfig({
                     updated_at: {

test/acceptance/x_cache_channel.js

@@ -1,307 +0,0 @@
-var testHelper = require('../support/test_helper');
-
-var assert = require('../support/assert');
-var qs = require('querystring');
-
-var CartodbWindshaft = require('../../lib/cartodb/server');
-var serverOptions = require('../../lib/cartodb/server_options');
-var server = new CartodbWindshaft(serverOptions);
-server.setMaxListeners(0);
-
-var LayergroupToken = require('../support/layergroup-token');
-
-describe('get requests x-cache-channel', function() {
-
-    var keysToDelete;
-    beforeEach(function() {
-        keysToDelete = {};
-    });
-
-    afterEach(function(done) {
-        testHelper.deleteRedisKeys(keysToDelete, done);
-    });
-
-    var statusOkResponse = {
-        status: 200
-    };
-
-    var mapConfig = {
-        version: '1.3.0',
-        layers: [
-            {
-                options: {
-                    sql: 'select * from test_table limit 2',
-                    cartocss: '#layer { marker-fill:red; }',
-                    cartocss_version: '2.3.0',
-                    attributes: {
-                        id:'cartodb_id',
-                        columns: [
-                            'name',
-                            'address'
-                        ]
-                    }
-                }
-            }
-        ]
-    };
-
-    var layergroupRequest = {
-        url: '/api/v1/map?config=' + encodeURIComponent(JSON.stringify(mapConfig)),
-        method: 'GET',
-        headers: {
-            host: 'localhost'
-        }
-    };
-
-    function getRequest(url, addApiKey, callbackName) {
-        var params = {};
-        if (!!addApiKey) {
-            params.api_key = '1234';
-        }
-        if (!!callbackName) {
-            params.callback = callbackName;
-        }
-
-        return {
-            url: url + '?' + qs.stringify(params),
-            method: 'GET',
-            headers: {
-                host: 'localhost',
-                'Content-Type': 'application/json'
-            }
-        };
-    }
-
-    function validateXCacheChannel(done, expectedCacheChannel) {
-        return function(res, err) {
-            if (err) {
-                return done(err);
-            }
-
-            assert.ok(res.headers['x-cache-channel']);
-            if (expectedCacheChannel) {
-                assert.equal(res.headers['x-cache-channel'], expectedCacheChannel);
-            }
-
-            done();
-        };
-    }
-
-    function noXCacheChannelHeader(done) {
-        return function(res, err) {
-            if (err) {
-                return done(err);
-            }
-
-            assert.ok(
-                !res.headers['x-cache-channel'],
-                'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
-            );
-            done();
-        };
-    }
-
-    function withLayergroupId(callback) {
-        assert.response(
-            server,
-            layergroupRequest,
-            statusOkResponse,
-            function(res, err) {
-                if (err) {
-                    return callback(err);
-                }
-                var layergroupId = JSON.parse(res.body).layergroupid;
-                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
-                callback(null, layergroupId, res);
-            }
-        );
-    }
-
-    describe('header should be present', function() {
-
-        it('/api/v1/map Map instantiation', function(done) {
-            var testFn = validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table');
-            withLayergroupId(function(err, layergroupId, res) {
-                testFn(res);
-            });
-        });
-
-        it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
-            withLayergroupId(function(err, layergroupId) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png'),
-                    validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
-                );
-            });
-        });
-
-        it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
-            withLayergroupId(function(err, layergroupId) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png'),
-                    validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
-                );
-            });
-        });
-
-        it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
-            withLayergroupId(function(err, layergroupId) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png'),
-                    validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
-                );
-            });
-        });
-
-        it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
-            withLayergroupId(function(err, layergroupId) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1'),
-                    validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
-                );
-            });
-        });
-
-        it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
-            withLayergroupId(function(err, layergroupId) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png'),
-                    validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
-                );
-            });
-        });
-
-        it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
-            withLayergroupId(function(err, layergroupId) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png'),
-                    validateXCacheChannel(done, 'test_windshaft_cartodb_user_1_db:public.test_table')
-                );
-            });
-        });
-
-    });
-
-    describe('header should NOT be present', function() {
-
-        it('/', function(done) {
-            assert.response(
-                server,
-                getRequest('/'),
-                statusOkResponse,
-                noXCacheChannelHeader(done)
-            );
-        });
-
-        it('/version', function(done) {
-            assert.response(
-                server,
-                getRequest('/version'),
-                statusOkResponse,
-                noXCacheChannelHeader(done)
-            );
-        });
-
-        it('/health', function(done) {
-            assert.response(
-                server,
-                getRequest('/health'),
-                statusOkResponse,
-                noXCacheChannelHeader(done)
-            );
-        });
-
-        it('/api/v1/map/named list named maps', function(done) {
-            assert.response(
-                server,
-                getRequest('/api/v1/map/named', true),
-                statusOkResponse,
-                noXCacheChannelHeader(done)
-            );
-        });
-
-        describe('with named maps', function() {
-
-            var templateName = 'x_cache';
-
-            beforeEach(function(done) {
-                var template =  {
-                    version: '0.0.1',
-                    name: templateName,
-                    auth: {
-                        method: 'open'
-                    },
-                    layergroup:  mapConfig
-                };
-
-                var namedMapRequest = {
-                    url: '/api/v1/map/named?api_key=1234',
-                    method: 'POST',
-                    headers: {
-                        host: 'localhost',
-                        'Content-Type': 'application/json'
-                    },
-                    data: JSON.stringify(template)
-                };
-
-                assert.response(
-                    server,
-                    namedMapRequest,
-                    statusOkResponse,
-                    function(res, err) {
-                        done(err);
-                    }
-                );
-            });
-
-            afterEach(function(done) {
-                assert.response(
-                    server,
-                    {
-                        url: '/api/v1/map/named/' + templateName + '?api_key=1234',
-                        method: 'DELETE',
-                        headers: {
-                            host: 'localhost'
-                        }
-                    },
-                    {
-                        status: 204
-                    },
-                    function(res, err) {
-                        done(err);
-                    }
-                );
-            });
-
-
-            it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/named/' + templateName, true),
-                    statusOkResponse,
-                    noXCacheChannelHeader(done)
-                );
-            });
-
-            it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
-                assert.response(
-                    server,
-                    getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
-                    statusOkResponse,
-                    noXCacheChannelHeader(done)
-                );
-            });
-
-        });
-
-    });
-
-
-});

test/integration/analysis-backend-limits.js

@@ -0,0 +1,167 @@
+var testHelper = require('../support/test_helper');
+
+var assert = require('assert');
+var redis = require('redis');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
+
+describe('analysis-backend limits', function() {
+
+    var redisClient;
+    var keysToDelete;
+    var user = 'localhost';
+
+    beforeEach(function() {
+        redisClient = redis.createClient(global.environment.redis.port);
+        keysToDelete = {};
+        var redisPool = new RedisPool(global.environment.redis);
+        this.metadataBackend = cartodbRedis({pool: redisPool});
+    });
+
+    afterEach(function(done) {
+        redisClient.quit(function() {
+            testHelper.deleteRedisKeys(keysToDelete, done);
+        });
+    });
+
+    function withAnalysesLimits(limits, callback) {
+        redisClient.SELECT(5, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            var analysesLimitsKey = 'limits:analyses:' + user;
+            redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
+                if (err) {
+                    return callback(err);
+                }
+                keysToDelete[analysesLimitsKey] = 5;
+                return callback();
+            });
+        });
+    }
+
+    it("should use limits from configuration", function(done) {
+        var analysisBackend = new AnalysisBackend(this.metadataBackend, {
+            limits: {
+                moran: { timeout: 5000 },
+                kmeans: { timeout: 5000 }
+            }
+        });
+        analysisBackend.getAnalysesLimits(user, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.ok(result.analyses.moran);
+            assert.equal(result.analyses.moran.timeout, 5000);
+
+            assert.ok(result.analyses.kmeans);
+            assert.equal(result.analyses.kmeans.timeout, 5000);
+
+            done();
+        });
+    });
+
+    it("should use limits from redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend);
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, redis takes priority", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    moran: { timeout: 1000 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    moran: { timeout: 1000 },
+                    kmeans: { timeout: 1000 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                assert.ok(result.analyses.kmeans);
+                assert.equal(result.analyses.kmeans.timeout, 1000);
+
+                done();
+            });
+        });
+    });
+
+    it("should allow to set other limits per analysis via configuration, and keep timeout from redis", function(done) {
+        var self = this;
+        var limits = ['aggregate-intersection', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    'aggregate-intersection': { timeout: 10000, maxNumberOfRows: 1e5 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses['aggregate-intersection']);
+                assert.equal(result.analyses['aggregate-intersection'].timeout, 5000);
+                assert.equal(result.analyses['aggregate-intersection'].maxNumberOfRows, 1e5);
+
+                done();
+            });
+        });
+    });
+
+});

test/integration/mapconfig_named_layers_datasource.js

@@ -145,7 +145,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[0].type, 'cartodb');
                 assert.equal(layers[0].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(0), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(0), {});
 
                 done();
             }
@@ -160,7 +160,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[0].type, 'cartodb');
                 assert.equal(layers[0].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(0), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(0), {});
 
                 assert.equal(layers[1].type, 'cartodb');
                 assert.equal(layers[1].options.sql, wadusTemplateSql);
@@ -181,7 +181,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[0].type, 'cartodb');
                 assert.equal(layers[0].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(0), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(0), {});
 
                 assert.equal(layers[1].type, 'mapnik');
                 assert.equal(layers[1].options.sql, wadusMapnikSql);
@@ -263,7 +263,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[3].type, 'cartodb');
                 assert.equal(layers[3].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(3), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(3), {});
 
                 assert.equal(layers[4].type, 'cartodb');
                 assert.equal(layers[4].options.sql, wadusTemplateSql);
@@ -273,7 +273,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[5].type, 'cartodb');
                 assert.equal(layers[5].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(5), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(5), {});
 
                 assert.equal(layers[6].type, 'mapnik');
                 assert.equal(layers[6].options.sql, wadusMapnikSql);
@@ -298,6 +298,7 @@ describe('named_layers datasources', function() {
             var context = {};
             mapConfigNamedLayersAdapter.getMapConfig(username, testScenario.config, params, context,
                 function(err, mapConfig) {
+                    assert.ifError(err);
                     testScenario.test(err, mapConfig.layers, context.datasource, done);
                 }
             );

test/support/prepare_db.sh

@@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced'
+  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish'
   REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
   CURL_ARGS=""

test/support/sql/analysis_catalog.sql

@@ -26,5 +26,7 @@ CREATE TABLE IF NOT EXISTS
 -- last job modifying the node
     last_modified_by uuid,
 -- store error message for failures
-    last_error_message text
+    last_error_message text,
+-- cached tables involved in the analysis
+    cache_tables regclass[] NOT NULL DEFAULT '{}'
 );

test/support/sql/cdb_analysis_check.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+BEGIN
+END;
+$$ LANGUAGE PLPGSQL;

test/support/sql/cdb_invalidate_varnish.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION CDB_Invalidate_Varnish(table_name TEXT)
+RETURNS void AS
+$$
+BEGIN
+END;
+$$ LANGUAGE PLPGSQL;

test/support/sql/windshaft.test.sql

@@ -634,7 +634,8 @@ GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER;
 --
 GRANT SELECT, UPDATE, INSERT, DELETE ON cdb_analysis_catalog TO :TESTUSER;
 
-create schema cdb_crankshaft;
+DROP EXTENSION IF EXISTS crankshaft;
+CREATE SCHEMA IF NOT EXISTS cdb_crankshaft;
 GRANT USAGE ON SCHEMA cdb_crankshaft TO :TESTUSER;
 CREATE TYPE kmeans_type as (cartodb_id numeric, cluster_no numeric);
 CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters integer,no_init integer default 20)

test/support/test-client.js

@@ -307,6 +307,13 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
         url += '?' + qs.stringify(extraParams);
     }
 
+    var expectedResponse = params.response || {
+        status: 200,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+    };
+
     var layergroupId;
     step(
         function createLayergroup() {
@@ -372,12 +379,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
                         host: 'localhost'
                     }
                 },
-                {
-                    status: 200,
-                    headers: {
-                        'Content-Type': 'application/json; charset=utf-8'
-                    }
-                },
+                expectedResponse,
                 function(res, err) {
                     if (err) {
                         return next(err);

test/unit/cartodb/model/resource-locator.test.js

@@ -0,0 +1,132 @@
+require('../../../support/test_helper');
+
+var assert = require('../../../support/assert');
+var ResourceLocator = require('../../../../lib/cartodb/models/resource-locator');
+
+describe('ResourceLocator.getUrls', function() {
+    var USERNAME = 'username';
+    var RESOURCE = 'wadus';
+    var HTTP_SUBDOMAINS = ['1', '2', '3', '4'];
+    var HTTPS_SUBDOMAINS = ['a', 'b', 'c', 'd'];
+
+    it('should return default urls when no serverMetadata is in environment', function() {
+        var resourceLocator = new ResourceLocator({});
+        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+        assert.ok(urls);
+    });
+
+    var BASIC_ENVIRONMENT = {
+        serverMetadata: {
+            cdn_url: {
+                http: 'cdn.carto.com',
+                https: 'cdn.ssl.carto.com'
+            }
+        }
+    };
+    it('should return default urls when basic http and https domains are provided', function() {
+        var resourceLocator = new ResourceLocator(BASIC_ENVIRONMENT);
+        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+        assert.ok(urls);
+
+        assert.equal(urls.http, ['http://cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
+        assert.equal(urls.https, ['https://cdn.ssl.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
+    });
+
+    var RESOURCE_TEMPLATES_ENVIRONMENT = {
+        serverMetadata: {
+            cdn_url: {
+                http: 'cdn.carto.com',
+                https: 'cdn.ssl.carto.com'
+            }
+        },
+        resources_url_templates: {
+            http: 'http://{{=it.user}}.localhost.lan/api/v1/map',
+            https: 'https://{{=it.user}}.ssl.localhost.lan/api/v1/map'
+        }
+    };
+    it('resources_url_templates should take precedence over http and https domains', function() {
+        var resourceLocator = new ResourceLocator(RESOURCE_TEMPLATES_ENVIRONMENT);
+        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+        assert.ok(urls);
+
+        assert.equal(urls.http, ['http://' + USERNAME + '.localhost.lan', 'api/v1/map', RESOURCE].join('/'));
+        assert.equal(urls.https, ['https://' + USERNAME + '.ssl.localhost.lan', 'api/v1/map', RESOURCE].join('/'));
+    });
+
+    var CDN_TEMPLATES_ENVIRONMENT = {
+        serverMetadata: {
+            cdn_url: {
+                http: 'cdn.carto.com',
+                https: 'cdn.ssl.carto.com',
+                templates: {
+                    http: {
+                        url: "http://{s}.cdn.carto.com",
+                        subdomains: HTTP_SUBDOMAINS
+                    },
+                    https: {
+                        url: "https://cdn_{s}.ssl.cdn.carto.com",
+                        subdomains: HTTPS_SUBDOMAINS
+                    }
+                }
+            }
+        }
+    };
+    it('cdn_url templates should take precedence over http and https domains', function() {
+        var resourceLocator = new ResourceLocator(CDN_TEMPLATES_ENVIRONMENT);
+        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+        assert.ok(urls);
+
+        var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
+        var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
+
+        assert.equal(
+            urls.http,
+            ['http://' + httpSubdomain + '.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
+        );
+        assert.equal(
+            urls.https,
+            ['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
+        );
+    });
+
+    var CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT = {
+        serverMetadata: {
+            cdn_url: {
+                http: 'cdn.carto.com',
+                https: 'cdn.ssl.carto.com',
+                templates: {
+                    http: {
+                        url: "http://{s}.cdn.carto.com",
+                        subdomains: HTTP_SUBDOMAINS
+                    },
+                    https: {
+                        url: "https://cdn_{s}.ssl.cdn.carto.com",
+                        subdomains: HTTPS_SUBDOMAINS
+                    }
+                }
+            }
+        },
+        resources_url_templates: {
+            http: 'http://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map',
+            https: 'https://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map'
+        }
+    };
+    it('should mix cdn_url templates and resources_url_templates', function() {
+        var resourceLocator = new ResourceLocator(CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT);
+        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+        assert.ok(urls);
+
+        var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
+        var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
+
+        assert.equal(
+            urls.http,
+            ['http://' + httpSubdomain + '.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
+        );
+        assert.equal(
+            urls.https,
+            ['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
+        );
+    });
+
+});

test/unit/cartodb/ported/tile_stats.test.js

@@ -26,12 +26,14 @@ describe('tile stats', function() {
         var layergroupController = new LayergroupController();
 
         var reqMock = {
+            profiler: { toJSONString:function() {} },
             params: {
                 format: invalidFormat
             }
         };
         var resMock = {
             status: function() { return this; },
+            set: function() {},
             json: function() {},
             jsonp: function() {},
             send: function() {}
@@ -54,12 +56,14 @@ describe('tile stats', function() {
             }
         });
         var reqMock = {
+            profiler: { toJSONString:function() {} },
             params: {
                 format: validFormat
             }
         };
         var resMock = {
             status: function() { return this; },
+            set: function() {},
             json: function() {},
             jsonp: function() {},
             send: function() {}

test/unit/cartodb/ported/windshaft_server.test.js

@@ -19,7 +19,7 @@ describe('windshaft', function() {
 
     it('can spawn a new server on the global listen port', function(done){
         var ws = cartodbServer(serverOptions);
-        var server = ws.listen(global.environment.windshaft_port, function() {
+        var server = ws.listen(global.environment.port, function() {
             assert.ok(ws);
             server.close(done); /* allow proper tear down */
         });