Release 4.2.0

Allow to request MVT tiles without CartoCSS

.gitignore

@@ -2,10 +2,12 @@ node_modules*
 config.status*
 config/environments/*.js
 .idea
+.vscode
+.nvmrc
 tools/munin/windshaft.conf
 logs/
 pids/
 redis.pid
-test.log
-npm-debug.log
+*.log
 coverage/
+.DS_Store

.jshintignore

@@ -1,4 +1,3 @@
 test/results/
 test/monkey/
 test/benchmark.js
-test/support/

.jshintrc

@@ -40,7 +40,7 @@
     "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
 //    "eqnull"        : false,     // true: Tolerate use of `== null`
 //    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
-//    "esnext"        : false,     // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "esnext"        : true,     // true: Allow ES.next (ES6) syntax (ex: `const`)
 //    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
 //    // (ex: `for each`, multiple try/catch, function expression…)
 //    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`

.travis.yml

@@ -1,37 +1,14 @@
-addons:
-  apt:
-    packages:
-      - pkg-config
-      - libcairo2-dev
-      - libjpeg8-dev
-      - libgif-dev
+sudo: required
+dist: trusty
+
+services:
+  - docker
 
 before_install:
-  - npm install -g npm@2
-  - lsb_release -a
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
-  - sudo apt-get -qq purge postgis* postgresql*
-  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
-  - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.5
-  - sudo apt-add-repository --yes ppa:cartodb/gis
-  - sudo apt-get update
-  - sudo apt-get install -q postgresql-9.5-postgis-2.2
-  - sudo apt-get install -q postgresql-plpython-9.5
-  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.5/main/pg_hba.conf
-  - sudo service postgresql restart
-  - createdb template_postgis
-  - createuser publicuser
-  - psql -c "CREATE EXTENSION postgis" template_postgis
+  - docker pull cartoimages/windshaft-carto-testing
 
-env:
-  - NPROCS=1 JOBS=1 PGUSER=postgres
+script:
+  - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing
 
-language: node_js
-node_js:
-  - "0.10"
+language: generic
 
-notifications:
-  irc:
-    channels:
-      - "irc.freenode.org#cartodb"
-    use_notice: true

CONTRIBUTING.md

@@ -8,4 +8,4 @@ We love pull requests from everyone, see [Contributing to Open Source on GitHub]
 
 ## Submitting Contributions
 
-* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

HOWTO_RELEASE

@@ -1,11 +1,10 @@
 1. Test (make clean all check), fix if broken before proceeding
-2. Ensure proper version in package.json 
+2. Ensure proper version in package.json
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
-5. Commit package.json, npm-shrinwrap.json, NEWS
+4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
+5. Commit package.json, yarn.lock, NEWS
 6. git tag -a Major.Minor.Patch # use NEWS section as content
-7. Announce on cartodb@googlegroups.com
-8. Stub NEWS/package for next version
+7. Stub NEWS/package for next version
 
 Versions:
 

INSTALL.md

@@ -4,11 +4,11 @@
 Make sure that you have the requirements needed. These are
 
 - Core
-  - Node.js >=0.8
-  - npm >=1.2.1 <2.0.0
+  - Node.js >=6.9.x
+  - yarn >=0.27.5 <1.0.0
   - PostgreSQL >8.3.x, PostGIS >1.5.x
   - Redis >2.4.0 (http://www.redis.io)
-  - Mapnik 2.0.1, 2.0.2, 2.1.0, 2.2.0, 2.3.0. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
+  - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
   - Windshaft: check [Windshaft dependencies and installation notes](https://github.com/CartoDB/Windshaft#dependencies)
   - libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support
 
@@ -43,11 +43,11 @@ psql -d template_postgis -c 'CREATE EXTENSION postgis;'
 To fetch and build all node-based dependencies, run:
 
 ```
-npm install
+yarn
 ```
 
-Note that the ```npm install``` step will populate the node_modules/
+Note that the ```yarn``` step will populate the node_modules/
 directory with modules, some of which being compiled on demand. If you
 happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
-```npm install``` again.
+```yarn``` again.

Makefile

@@ -7,7 +7,7 @@ all:
 	@$(SHELL) ./scripts/install.sh
 
 clean:
-	rm -rf node_modules/*
+	rm -rf node_modules/
 
 distclean: clean
 	rm config.status*
@@ -16,7 +16,7 @@ config.status--test:
 	./configure --environment=test
 
 config/environments/test.js: config.status--test
-	./config.status--test 
+	./config.status--test
 
 TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js")
 TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")

NEWS.md

@@ -1,5 +1,747 @@
 # Changelog
 
+## 4.2.0
+Released 2017-12-04
+
+Announcements:
+ - Allow to request MVT tiles without CartoCSS
+ - Upgrades windshaft to [4.1.0](https://github.com/CartoDB/windshaft/releases/tag/4.1.0).
+
+
+## 4.1.1
+Released 2017-11-29
+
+Announcements:
+ - Upgrades turbo-carto to [0.20.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.2).
+
+
+## 4.1.0
+Released 2017-mm-dd
+
+Announcements:
+ - Upgrades windshaft to [4.0.1](https://github.com/CartoDB/windshaft/releases/tag/4.0.1).
+ - Add `categories` query param to define the number of categories to be ranked for aggregation dataviews.
+
+
+## 4.0.1
+Released 2017-10-18
+
+Announcements:
+ - Upgrades camshaft to [0.59.4](https://github.com/CartoDB/camshaft/releases/tag/0.59.4).
+ - Upgrades windshaft to [4.0.0](https://github.com/CartoDB/windshaft/releases/tag/4.0.0).
+ - Split and move `req2params` method to multiple middlewares.
+ - Use express error handler middleware to respond in case of something went wrong.
+ - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params.
+ - Move `LZMA` decompression to its own middleware.
+ - Implement stats middleware removing some duplicated code while sending response.
+
+
+## 4.0.0
+Released 2017-10-04
+
+Backward incompatible changes:
+ - Removes `list` dataview type.
+
+Announcements:
+ - Upgrades body-parser to 1.18.2.
+ - Upgrades express to 4.16.0.
+ - Upgrades debug to 3.1.0.
+ - Upgrades request to 2.83.0.
+ - Upgrades turbo-carto to [0.20.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.1)
+ - Upgrades cartodb-psql to [0.10.2](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.2).
+ - Upgrades camshaft to [0.59.2](https://github.com/CartoDB/camshaft/releases/tag/0.59.2).
+ - Upgrades windshaft to [3.3.3](https://github.com/CartoDB/windshaft/releases/tag/3.3.3).
+ - Upgrades yarn minimum version requirement to v0.27.5
+
+
+## 3.13.0
+Released 2017-10-02
+ - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables.
+
+Bugfixes:
+ - Bounding box parameter ignored in static named maps #735.
+ - camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis
+
+## 3.12.10
+Released 2017-09-18
+ - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2).
+
+## 3.12.9
+Released 2017-09-07
+
+Bug fixes:
+- Do not use distinct when calculating quantiles. #743
+
+## 3.12.8
+Released 2017-09-07
+
+Bug fixes:
+- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962)
+
+## 3.12.7
+Released 2017-09-01
+
+ - Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1).
+
+
+## 3.12.6
+Released 2017-08-31
+
+ - Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0).
+
+
+## 3.12.5
+Released 2017-08-24
+
+ - Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0).
+
+
+## 3.12.4
+Released 2017-08-23
+
+Announcements:
+ - Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0).
+
+## 3.12.3
+Released 2017-08-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8).
+
+## 3.12.2
+Released 2017-08-16
+
+Bug fixes:
+ - Polygon count problems #725.
+
+
+## 3.12.1
+Released 2017-08-13
+ - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1).
+ - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1).
+ - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7).
+
+
+## 3.12.0
+Released 2017-08-10
+
+Announcements:
+ - Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717.
+ - Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0).
+ - Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0).
+
+
+## 3.11.0
+Released 2017-08-08
+
+Announcements:
+ - Allow to override with any aggregation for histograms instantiated w/o aggregation.
+
+Bug fixes:
+ - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series.
+ - Support timestamp with timezones to calculate the number of bins in time-series.
+ - Fixed issue related to name collision while building time-series query.
+
+
+## 3.10.1
+Released 2017-08-04
+
+Bug fixes:
+ - Exclude Infinities & NaNs from ramps #719.
+ - Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720.
+
+
+## 3.10.0
+Released 2017-08-03
+
+Announcements:
+ - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698.
+ - Support special numeric values (±Infinity, NaN) for json responses #706
+
+
+## 3.9.8
+Released 2017-07-21
+
+ - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2).
+
+
+## 3.9.7
+Released 2017-07-20
+
+Bug fixes:
+ - Respond with 204 (No content) when vector tile has no data #712
+
+Announcements:
+ - Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2)
+
+
+## 3.9.6
+Released 2017-07-11
+
+ - Dataviews: support for aggregation in search results #708
+
+
+## 3.9.5
+Released 2017-06-27
+
+ - Dataviews: support special numeric values (±Infinity, NaN) #700
+
+
+## 3.9.4
+Released 2017-06-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
+
+## 3.9.3
+Released 2017-06-16
+
+Announcements:
+ - Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5).
+
+## 3.9.2
+Released 2017-06-16
+
+Announcements:
+ - Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4).
+
+## 3.9.1
+Released 2017-06-06
+
+Announcements:
+ - Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3).
+
+
+## 3.9.0
+Released 2017-05-31
+
+Announcements:
+ - Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1).
+ - Add support to retrieve info about layer stats in map instantiation.
+ - Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2).
+ - Remove promise polyfill from turbo-carto adapter
+
+
+## 3.8.0
+Released 2017-05-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0).
+ - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1)
+
+
+## 3.7.1
+Released 2017-05-18
+
+Bug fixes:
+ - Fix buffersize assignment when is not defined in requested mapconfig.
+
+
+## 3.7.0
+Released 2017-05-18
+
+Announcements:
+- Manage multiple values of buffer-size for different formats
+- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0).
+
+
+## 3.6.6
+Released 2017-05-11
+
+Announcements:
+ - Upgrades camshaft to [0.54.4](https://github.com/CartoDB/camshaft/releases/tag/0.54.4).
+
+
+## 3.6.5
+Released 2017-05-09
+
+Announcements:
+ - Upgrades camshaft to [0.54.3](https://github.com/CartoDB/camshaft/releases/tag/0.54.3).
+
+
+## 3.6.4
+Released 2017-05-05
+
+Announcements:
+ - Upgrade cartodb-psql to [0.8.0](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.8.0).
+ - Upgrades camshaft to [0.54.2](https://github.com/CartoDB/camshaft/releases/tag/0.54.2).
+ - Upgrades windshaft to [3.1.2](https://github.com/CartoDB/windshaft/releases/tag/3.1.2).
+
+
+## 3.6.3
+Released 2017-04-25
+
+Announcements:
+ - Upgrades windshaft to [3.1.1](https://github.com/CartoDB/windshaft/releases/tag/3.1.1).
+
+
+## 3.6.2
+Released 2017-04-24
+
+Announcements:
+ - Upgrades grainstore to [1.6.3](https://github.com/CartoDB/grainstore/releases/tag/1.6.3).
+
+
+## 3.6.1
+Released 2017-04-24
+
+Announcements:
+ - Upgrades camshaft to [0.54.1](https://github.com/CartoDB/camshaft/releases/tag/0.54.1).
+
+
+## 3.6.0
+Released 2017-04-20
+
+Announcements:
+ - Upgrades camshaft to [0.54.0](https://github.com/CartoDB/camshaft/releases/tag/0.54.0).
+
+
+## 3.5.1
+Released 2017-04-11
+
+Announcements:
+ - Upgrades camshaft to [0.53.1](https://github.com/CartoDB/camshaft/releases/tag/0.53.1).
+
+
+## 3.5.0
+Released 2017-04-10
+
+Bug fixes:
+ - Fix invalidation of cache for maps with analyses #638.
+
+Announcements:
+ - Upgrades camshaft to [0.53.0](https://github.com/CartoDB/camshaft/releases/tag/0.53.0).
+
+
+## 3.4.0
+Released 2017-04-03
+
+Announcements:
+ - Upgrades camshaft to [0.51.0](https://github.com/CartoDB/camshaft/releases/tag/0.51.0).
+
+
+## 3.3.0
+Released 2017-04-03
+
+New features:
+ - Static map endpoints allow specifying the layers to render #653.
+
+
+## 3.2.0
+Released 2017-03-30
+
+Announcements:
+ - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0).
+ - Active GC interval.
+
+
+## 3.1.1
+Released 2017-03-23
+
+Bug fixes:
+ - Use crc32 instead of md5 for computing subdomain candidate #642
+
+
+## 3.1.0
+Released 2017-03-22
+
+Features:
+ - Generate URLs for resources based on CDN and template rules
+
+
+## 3.0.2
+Released 2017-03-22
+
+Bug fixes:
+ - Upgrade dependencies
+ - Improve docs: remove mentions to NPM and use yarn instead
+ - Remove script to generate npm-shrinkwrap file
+
+
+## 3.0.1
+Released 2017-03-21
+
+Announcements:
+ - Upgrades windshaft to [3.0.1](https://github.com/CartoDB/windshaft/releases/tag/3.0.1).
+
+
+## 3.0.0
+Released 2017-03-21
+
+Announcements:
+ - Supports Node v6.9.x
+ - Drops support for Node v0.10.x
+ - Upgrades windshaft to 3.0.0
+ - Upgrades cartodb-query-tables to 0.2.0
+ - Upgrades cartodb-redis to 0.13.2
+ - Upgrades redis-mpool to 0.4.1
+
+**Note**: Due to this [issue](https://github.com/npm/npm/issues/15713), Windshaft-cartodb must be installed with `yarn` instead of `npm` providing just a `yarn.lock` to get consistent installs across machines.
+
+## 2.89.0
+Released 2017-03-17
+
+**Deprecation warning**: v2.89.0 is the last release that supports Node v0.10.x. Next mayor release will support Node v6.9.x and further versions.
+
+Announcements:
+ - Upgrades windshaft to [2.8.0](https://github.com/CartoDB/windshaft/releases/tag/2.8.0).
+
+Bug fixes:
+ - Histogram column type discovery query uses non-filtered query #637
+
+
+## 2.88.4
+Released 2017-03-10
+
+Announcements:
+ - Upgrades camshaft to [0.50.3](https://github.com/CartoDB/camshaft/releases/tag/0.50.3).
+
+
+## 2.88.3
+Released 2017-03-02
+
+Bug fixes:
+- Category dataviews now uses the proper aggregation function for the 'Other' category. See https://github.com/CartoDB/Windshaft-cartodb/issues/628
+
+## 2.88.2
+Released 2017-02-23
+
+Announcements:
+ - Upgrades camshaft to [0.50.2](https://github.com/CartoDB/camshaft/releases/tag/0.50.2).
+
+
+## 2.88.1
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.1](https://github.com/CartoDB/camshaft/releases/tag/0.50.1)
+
+
+## 2.88.0
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.0](https://github.com/CartoDB/camshaft/releases/tag/0.50.0).
+ - Upgrades cartodb-psql to [0.7.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.7.1).
+ - Upgrades windshaft to [2.7.0](https://github.com/CartoDB/windshaft/releases/tag/2.7.0).
+
+
+## 2.87.5
+Released 2017-02-02
+
+Bug fixes:
+ - Cast dataview override values to Number or throw error.
+
+
+## 2.87.4
+Released 2017-01-20
+
+Bug fixes:
+ - Be able to not compute NULL categories and null values  in aggregation dataviews #617.
+
+
+## 2.87.3
+Released 2016-12-19
+
+Bug fixes:
+ - Fix overviews-related dataviews problems. See https://github.com/CartoDB/Windshaft-cartodb/pull/604
+
+
+## 2.87.2
+Released 2016-12-19
+
+- Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
+
+
+## 2.87.1
+Released 2016-12-13
+
+Announcements:
+ - Upgrades windshaft to [2.6.4](https://github.com/CartoDB/Windshaft/releases/tag/2.6.4).
+ - Upgrades request dependency.
+ - Regenerate npm-shrinkwrap.json: missing dependency updates.
+
+
+## 2.87.0
+Released 2016-12-12
+
+Enhancements:
+ - Upgrade turbo-carto dependency to version 0.19.0.
+
+## 2.86.1
+Released 2016-12-02
+
+Bug fixes:
+ - Maps with analyses and `sql_wrap` were broken #599.
+
+
+## 2.86.0
+Released 2016-12-02
+
+Announcements:
+ - Upgrades windshaft to [2.6.3](https://github.com/CartoDB/Windshaft/releases/tag/2.6.3).
+
+
+## 2.85.1
+Released 2016-11-30
+
+Announcements:
+ - Upgrades camshaft to [0.48.4](https://github.com/CartoDB/camshaft/releases/tag/0.48.4).
+
+
+## 2.85.0
+Released 2016-11-24
+
+New features:
+ - Allow to set resource URL templates with substitution tokens #594.
+
+
+## 2.84.2
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.3](https://github.com/CartoDB/camshaft/releases/tag/0.48.3).
+
+
+## 2.84.1
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.2](https://github.com/CartoDB/camshaft/releases/tag/0.48.2).
+
+
+## 2.84.0
+Released 2016-11-11
+
+New features:
+ - Analyses limit configuration allows to set other limits than timeout.
+
+
+## 2.83.1
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.1](https://github.com/CartoDB/camshaft/releases/tag/0.48.1).
+
+
+## 2.83.0
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.0](https://github.com/CartoDB/camshaft/releases/tag/0.48.0).
+
+
+## 2.82.0
+Released 2016-11-08
+
+Announcements:
+ - Upgrades camshaft to [0.47.0](https://github.com/CartoDB/camshaft/releases/tag/0.47.0).
+
+
+## 2.81.1
+Released 2016-11-05
+
+Announcements:
+ - Upgrades windshaft to [2.6.2](https://github.com/CartoDB/windshaft/releases/tag/2.6.2).
+ - Upgrades camshaft to [0.46.3](https://github.com/CartoDB/camshaft/releases/tag/0.46.3).
+
+
+## 2.81.0
+Released 2016-11-02
+
+Enhancements:
+ - Returns errors with context when query layer does not retrieve geometry column
+
+Announcements:
+ - Upgrades windshaft to [2.6.1](https://github.com/CartoDB/windshaft/releases/tag/2.6.1).
+ - Upgrades camshaft to [0.46.2](https://github.com/CartoDB/camshaft/releases/tag/0.46.2).
+
+
+## 2.80.2
+Released 2016-10-26
+
+Bug fixes:
+ - Fix order in categories query to get ramps
+
+
+## 2.80.1
+Released 2016-10-25
+
+Announcements:
+ - Upgrades camshaft to [0.46.1](https://github.com/CartoDB/camshaft/releases/tag/0.46.1).
+
+
+## 2.80.0
+Released 2016-10-20
+
+Announcements:
+ - Upgrades camshaft to [0.46.0](https://github.com/CartoDB/camshaft/releases/tag/0.46.0).
+
+New features:
+ - Default analyses limits can be defined in configuration.
+
+
+## 2.79.0
+Released 2016-10-11
+
+New features:
+ - Retrieve analysis limits and pass them into camshaft.
+
+Announcements:
+ - Upgrades turbo-carto to [0.18.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.18.0).
+ - Upgrades camshaft to [0.45.0](https://github.com/CartoDB/camshaft/releases/tag/0.45.0).
+
+
+## 2.78.1
+Released 2016-09-30
+
+Announcements:
+ - Upgrades camshaft to [0.44.2](https://github.com/CartoDB/camshaft/releases/tag/0.44.2).
+
+
+## 2.78.0
+Released 2016-09-29
+
+New features:
+ - Add metadata about processed turbo-carto rules.
+
+Announcements:
+ - Upgrades turbo-carto to [0.17.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.17.1).
+
+
+## 2.77.1
+
+Released 2016-09-28
+
+Announcements:
+ - Upgrades camshaft to [0.44.1](https://github.com/CartoDB/camshaft/releases/tag/0.44.1).
+
+
+## 2.77.0
+
+Released 2016-09-26
+
+Announcements:
+ - Upgrades camshaft to [0.44.0](https://github.com/CartoDB/camshaft/releases/tag/0.44.0).
+ - Adds a new configuration for camshaft: logger stream.
+
+
+## 2.76.0
+
+Released 2016-09-15
+
+New features:
+ - Allow to use `--config /path/to/config.js` to specify configuration file.
+   - Environment will be loaded from config file if `environment` key is present, otherwise it keeps current behaviour.
+
+Bug fixes:
+ - Allow to use absolute paths for log files #570.
+
+
+## 2.75.0
+
+Released 2016-09-14
+
+Announcements:
+ - Upgrades camshaft to [0.43.0](https://github.com/CartoDB/camshaft/releases/tag/0.43.0).
+
+
+## 2.74.1
+
+Released 2016-09-07
+
+Announcements:
+ - Upgrades camshaft to [0.42.1](https://github.com/CartoDB/camshaft/releases/tag/0.42.1).
+
+
+## 2.74.0
+
+Released 2016-09-06
+
+Enhancements:
+ - Layers in previews can be shown or hidden using `preview_layers` property in template map
+
+
+## 2.73.1
+
+Released 2016-09-06
+
+Bug fixes:
+ - Fixes missing column in fixture table `cdb_analysis_catalog`.
+
+
+## 2.73.0
+
+Released 2016-09-06
+
+Announcements:
+ - Upgrades camshaft to [0.42.0](https://github.com/CartoDB/camshaft/releases/tag/0.42.0).
+
+
+## 2.72.0
+
+Released 2016-08-23
+
+Announcements:
+ - Upgrades camshaft to [0.41.0](https://github.com/CartoDB/camshaft/releases/tag/0.41.0).
+
+
+## 2.71.0
+
+Released 2016-08-17
+
+Announcements:
+ - Upgrades windshaft to [2.5.0](https://github.com/CartoDB/windshaft/releases/tag/2.5.0).
+
+
+## 2.70.0
+
+Released 2016-08-16
+
+Announcements:
+ - Upgrades camshaft to [0.40.0](https://github.com/CartoDB/camshaft/releases/tag/0.40.0).
+
+
+## 2.69.1
+
+Released 2016-08-12
+
+Announcements:
+ - Upgrades windshaft to [2.4.2](https://github.com/CartoDB/windshaft/releases/tag/2.4.2).
+
+
+## 2.69.0
+
+Released 2016-08-11
+
+Announcements:
+ - Upgrades camshaft to [0.39.0](https://github.com/CartoDB/camshaft/releases/tag/0.39.0).
+
+
+## 2.68.0
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades turbo-carto to [0.16.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.16.0).
+
+
+## 2.67.1
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades camshaft to [0.38.1](https://github.com/CartoDB/camshaft/releases/tag/0.38.1).
+
+
+## 2.67.0
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades camshaft to [0.38.0](https://github.com/CartoDB/camshaft/releases/tag/0.38.0).
+
+
 ## 2.66.2
 
 Released 2016-07-20

README.md

@@ -32,14 +32,14 @@ Upgrading
 Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following:
 
 ```
-rm -rf node_modules; npm install
+rm -rf node_modules; yarn
 ```
 
 Run
 ---
 
 ```
-node app.js <env> 
+node app.js <env>
 ```
 
 Where <env> is the name of a configuration file under config/environments/.
@@ -71,12 +71,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md).
 ### Developing with a custom windshaft version
 
 If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
-to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
+to use `yarn link`. You can read more about it at [yarn-link: Symlink a package folder](https://yarnpkg.com/en/docs/cli/link).
 
 **Quick start**:
 
 ```shell
-~/windshaft-directory $ npm install
-~/windshaft-directory $ npm link
-~/windshaft-cartodb-directory $ npm link windshaft
+~/windshaft-directory $ yarn
+~/windshaft-directory $ yarn link
+~/windshaft-cartodb-directory $ yarn link windshaft
 ```

app.js

@@ -2,23 +2,44 @@ var http = require('http');
 var https = require('https');
 var path = require('path');
 var fs = require('fs');
-
 var _ = require('underscore');
-
-var ENVIRONMENT;
-if ( process.argv[2] ) {
-    ENVIRONMENT = process.argv[2];
-} else if ( process.env.NODE_ENV ) {
-    ENVIRONMENT = process.env.NODE_ENV;
-} else {
-    ENVIRONMENT = 'development';
-}
+var semver = require('semver');
 
 // jshint undef:false
 var log = console.log.bind(console);
 var logError = console.error.bind(console);
 // jshint undef:true
 
+var nodejsVersion = process.versions.node;
+if (!semver.satisfies(nodejsVersion, '>=6.9.0')) {
+    logError(`Node version ${nodejsVersion} is not supported, please use Node.js 6.9 or higher.`);
+    process.exit(1);
+}
+
+var argv = require('yargs')
+    .usage('Usage: $0 <environment> [options]')
+    .help('h')
+    .example(
+    '$0 production -c /etc/sql-api/config.js',
+    'start server in production environment with /etc/sql-api/config.js as config file'
+)
+    .alias('h', 'help')
+    .alias('c', 'config')
+    .nargs('c', 1)
+    .describe('c', 'Load configuration from path')
+    .argv;
+
+var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
+var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
+if (!fs.existsSync(configurationFile)) {
+    logError('Configuration file "%s" does not exist', configurationFile);
+    process.exit(1);
+}
+
+global.environment = require(configurationFile);
+var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.environment.environment;
+process.env.NODE_ENV = ENVIRONMENT;
+
 var availableEnvironments = {
     production: true,
     staging: true,
@@ -33,16 +54,6 @@ if (!availableEnvironments[ENVIRONMENT]){
 }
 
 process.env.NODE_ENV = ENVIRONMENT;
-
-// set environment specific variables
-global.environment = require('./config/environments/' + ENVIRONMENT);
-
-global.log4js = require('log4js');
-var log4js_config = {
-  appenders: [],
-  replaceConsole: true
-};
-
 if (global.environment.uv_threadpool_size) {
     process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
 }
@@ -58,25 +69,31 @@ var agentOptions = _.defaults(global.environment.httpAgent || {}, {
 http.globalAgent = new http.Agent(agentOptions);
 https.globalAgent = new https.Agent(agentOptions);
 
+
+global.log4js = require('log4js');
+var log4jsConfig = {
+    appenders: [],
+    replaceConsole: true
+};
+
 if ( global.environment.log_filename ) {
-  var logdir = path.dirname(global.environment.log_filename);
-  // See cwd inlog4js.configure call below
-  logdir = path.resolve(__dirname, logdir);
-  if ( ! fs.existsSync(logdir) ) {
-    logError("Log filename directory does not exist: " + logdir);
-    process.exit(1);
-  }
-  log("Logs will be written to " + global.environment.log_filename);
-  log4js_config.appenders.push(
-    { type: "file", filename: global.environment.log_filename }
-  );
+    var logFilename = path.resolve(global.environment.log_filename);
+    var logDirectory = path.dirname(logFilename);
+    if (!fs.existsSync(logDirectory)) {
+        logError("Log filename directory does not exist: " + logDirectory);
+        process.exit(1);
+    }
+    log("Logs will be written to " + logFilename);
+    log4jsConfig.appenders.push(
+        { type: "file", absolute: true, filename: logFilename }
+    );
 } else {
-  log4js_config.appenders.push(
-    { type: "console", layout: { type:'basic' } }
-  );
+    log4jsConfig.appenders.push(
+        { type: "console", layout: { type:'basic' } }
+    );
 }
 
-global.log4js.configure(log4js_config, { cwd: __dirname });
+global.log4js.configure(log4jsConfig);
 global.logger = global.log4js.getLogger();
 
 global.environment.api_hostname = require('os').hostname().split('.')[0];
@@ -99,6 +116,8 @@ var listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, b
 var version = require("./package").version;
 
 listener.on('listening', function() {
+    log("Using Node.js %s", process.version);
+    log('Using configuration file "%s"', configurationFile);
     log(
         "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
         version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
@@ -114,7 +133,7 @@ setInterval(function() {
 
 process.on('SIGHUP', function() {
     global.log4js.clearAndShutdownAppenders(function() {
-        global.log4js.configure(log4js_config);
+        global.log4js.configure(log4jsConfig);
         global.logger = global.log4js.getLogger();
         log('Log files reloaded');
     });
@@ -123,3 +142,17 @@ process.on('SIGHUP', function() {
 process.on('uncaughtException', function(err) {
     global.logger.error('Uncaught exception: ' + err.stack);
 });
+
+if (global.gc) {
+    var gcInterval = Number.isFinite(global.environment.gc_interval) ?
+        global.environment.gc_interval :
+        10000;
+
+    if (gcInterval > 0) {
+        setInterval(function gcForcedCycle() {
+            var start = Date.now();
+            global.gc();
+            global.statsClient.timing('windshaft.gc', Date.now() - start);
+        }, gcInterval);
+    }
+}

config/environments/development.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.localhost'
@@ -23,6 +26,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -62,6 +80,7 @@ var config = {
         extent: "-180,-90,180,90",
         srid: 4326,
         */
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -88,6 +107,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -95,6 +128,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -209,6 +246,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -289,8 +338,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
-
+        layerStats: true
     }
 };
 

config/environments/production.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -23,6 +26,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -56,6 +74,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         /*
          * Set persist_connection to false if you want
@@ -82,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -89,6 +122,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -203,6 +240,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -289,7 +338,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: false
+        layerStats: false
     }
 };
 

config/environments/staging.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -23,6 +26,21 @@ var config = {
     // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -56,6 +74,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -82,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -89,6 +122,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -203,6 +240,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -289,7 +338,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
+        layerStats: true
     }
 };
 

config/environments/test.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '(.*)'
@@ -23,6 +26,20 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -56,6 +73,7 @@ var config = {
         host: '127.0.0.1',
         port: 5432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -82,6 +100,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -89,6 +121,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -204,6 +240,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -284,7 +332,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
+        layerStats: true
     }
 };
 

docker-test.sh

@@ -0,0 +1,11 @@
+export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres
+
+npm install -g yarn@0.27.5
+yarn
+
+ /etc/init.d/postgresql start
+
+createdb template_postgis && createuser publicuser
+psql -c "CREATE EXTENSION postgis" template_postgis
+
+POSTGIS_VERSION=2.4 npm test

docs/Map-API.md

@@ -17,3 +17,4 @@ You can create two types of maps with the Maps API:
 * [Anonymous Maps](anonymous_maps.md)
 * [Named Maps](named_maps.md)
 * [Static Maps API](static_maps_api.md)
+* [MapConfig File Format]([local file in the docs repo](https://github.com/CartoDB/docs/blob/master/_app/_mapsapi/06-mapconfig.md))

docs/MapConfig-Dataviews-extension.md

@@ -8,50 +8,13 @@ This specification describes an extension for
 
 This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`.
 
-It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms.
+It makes possible to get tabular data from analysis nodes: aggregated lists, aggregations, and histograms.
 
 ## 2.1. Dataview types
 
-### List
-
-A list is a simple result set per row where is possible to retrieve several columns from the original layer query.
-
-Definition
-```
-{
-    // REQUIRED
-    // string, `type` the list type
-    “type”: “list”,
-    // REQUIRED
-    // object, `options` dataview params
-    “options”: {
-        // REQUIRED
-        // array, `columns` to select for the list
-        “columns”: [“name”, “description”]
-    }
-}
-```
-
-Expected output
-```
-{
-  "type": "list",
-  "rows": [
-    {
-      "{columnName1}": "val1",
-      "{columnName2}": 100
-    },
-    {
-      "{columnName1}": "val2",
-      "{columnName2}": 200
-    }
-  ]
-}
-```
-
 ### Aggregation
 
-An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function.
+An aggregation is a list with aggregated results by a column and a given aggregation function.
 
 Definition
 ```

docs/anonymous_maps.md

@@ -70,21 +70,203 @@ curl 'https://{username}.carto.com/api/v1/map' -H 'Content-Type: application/jso
 }
 ```
 
-### Retrieve resources from the layergroup
+## Map Tile Rendering
+
+Map tiles create the graphical representation of your map in a web browser. The performance rendering of map tiles is dependent on the type of geospatial data model (raster or vector) that you are using.
+
+- **Raster**: Generates map tiles based on a grid of pixels to represent your data. Each cell is a fixed size and contains values for particular map features. On the server-side, each request queries a dataset to retrieve data for each map tile. The grid size of map tiles can often lead to graphic quality issues.
+
+- **Vector**: Generates map tiles based on pre-defined coordinates to represent your data, similar to how basemap image tiles are rendered. On the client-side, map tiles represent real-world geometries of a map. Depending on the coordinates, vertices are used to connect the data and display points, lines, or polygons for the map tiles.
+
+## Retrieve resources from the layergroup
 
 When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection.
 
-#### Mapnik tiles
+### Mapnik tiles
 
-These tiles will get just the Mapnik layers. To get individual layers, see the following section.
+These raster tiles retrieve just the Mapnik layers. See [individual layers](#individual-layers) for details about how to retrieve other layers.
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 ```
 
-#### Individual layers
+### Mapbox Vector Tiles (MVT)
 
-The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually in different formats depending on the layer type.
+[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server.
+
+CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files.
+
+The following examples describe how to fetch MVT tiles with a cURL request.
+
+#### MVT and Windshaft
+
+CARTO uses Windshaft as the map tiler library to render multilayer maps with the Maps API. You can use Windshaft to request MVT using the same layer type that is used for requesting raster tiles (Mapnik layer). Simply change the file format `.mvt` in the URL.
+
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/:layer/{z}/{x}/{y}.mvt
+```
+
+The following example instantiates an anonymous map with layer options:
+
+```bash
+{
+  user_name: 'mycartodbuser',
+  sublayers: [{
+    sql: "SELECT * FROM table_name";
+    cartocss: '#layer { marker-fill: #F0F0F0; }'
+  }],
+  maps_api_template: 'https://{user}.cartodb.com' // Optional
+}
+```
+
+**Note**: If no layer type is specified, Mapnik tiles are used by default. To access MVT tiles, specify `https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt` as the `maps_api_template` variable.
+
+**Tip:** If you are using [Named Maps](https://carto.com/docs/carto-engine/maps-api/named-maps/) to instantiate a layer, indicate the MVT file format and layer in the response:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/named/:templateId/:layer/{z}/{x}/{y}.mvt
+```
+
+For all layers in a Named Map, you must indicate Mapnik as the layer filter:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/named/:templateId/mapnik/{z}/{x}/{y}.mvt
+```
+
+#### Layergroup Filter for MVT Tiles
+
+To filter layers using Windshaft, use the following request where layers are numbered:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/0,1,2/{z}/{x}/{y}.mvt
+```
+
+To request all layers, remove the layergroup filter parameter:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt
+```
+
+To filter a specific layer:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/2/{z}/{x}/{y}.mvt
+```
+
+#### Example 1: MVT Tiles with Windshaft, CARTO.js, and MapboxGL
+
+1) Import the required libraries:
+
+```bash
+<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.js'></script>
+<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.css' rel='stylesheet' />
+<script src="http://libs.cartocdn.com/cartodb.js/v3/3.15/cartodb.core.js"></script>
+```
+
+2) Configure Map Client:
+
+```bash
+mapboxgl.accessToken = '{yourMapboxToken}';
+```
+
+3) Create Map Object (Mapbox):
+
+```bash
+var map = new mapboxgl.Map({
+container: 'map',
+zoom: 1,
+minZoom: 0,
+maxZoom: 18,
+center: [30, 0]
+});
+```
+
+4) Define Layer Options (CARTO):
+
+```bash
+var layerOptions = {
+user_name: "{username}",
+sublayers: [{
+sql: "SELECT * FROM {table_name}",
+cartocss: "...",
+     }]
+};
+```
+
+5) Request Tiles (from CARTO) and Set to Map Object (Mapbox):
+
+**Note:** By default, [CARTO core functions](https://carto.com/docs/carto-engine/carto-js/core-api/) retrieve URLs for fully rendered tiles. You must replace the default format (.png) with the MVT format (.mvt).
+
+
+```bash
+cartodb.Tiles.getTiles(layerOptions, function(result, err) {
+var tiles = result.tiles.map(function(tileUrl) {
+return tileUrl
+.replace('{s}', 'a')
+.replace(/\.png/, '.mvt');
+});
+map.setStyle(simpleStyle(tiles));
+});
+```
+
+#### Example 2: MVT Libraries with Windshaft and MapboxGL
+
+When you are not including CARTO.js to implement MVT tiles, you must use the `map.setStyle` parameter to specify vector map rendering.
+
+1) Import the required libraries:
+
+```bash
+<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.js'></script>
+<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.css' rel='stylesheet'/>
+```
+
+2) Configure Map Client:
+
+```bash
+mapboxgl.accessToken = '{yourMapboxToken}';
+```
+
+3) Create Map Object (Mapbox):
+
+```bash
+var map = new mapboxgl.Map({
+container: 'map',
+zoom: 1,
+minZoom: 0,
+maxZoom: 18,
+center: [30, 0]
+});
+```
+
+4) Set the Style
+
+```bash
+map.setStyle({
+    "version": 7,
+    "glyphs": "...",
+    "constants": {...}, 
+    "sources": {
+      "cartodb": {
+        "type": "vector",
+        "tiles": [ "http://{username}.cartodb.com/api/v1/map/named/templateId/mapnik/{z}/{x}/{y}.mvt"
+        ],
+        "maxzoom": 18
+      }
+    },
+    "layers": [{...}]
+});
+```
+
+**Tip:** If you are using MapboxGL, see the following resource for additional information.
+
+- [MapboxGL API Reference](https://www.mapbox.com/mapbox-gl-js/api/)
+- [MapboxGL Style Specifications](https://www.mapbox.com/mapbox-gl-js/style-spec/)
+- [Example of MapboxGL Implementation](https://www.mapbox.com/mapbox-gl-js/examples/)
+
+### Individual layers
+
+The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually, in different formats, depending on the layer type.
 
 Individual layers can be accessed using that 0-based index. For UTF grid tiles:
 
@@ -100,19 +282,19 @@ If the MapConfig had a Torque layer at index 1 it could be possible to request i
 https://{username}.carto.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json
 ```
 
-#### Attributes defined in `attributes` section
+### Attributes defined in `attributes` section
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id}
 ```
 
-Which returns JSON with the attributes defined, like:
+Which returns JSON with the attributes defined, such as:
 
 ```javascript
 { "c": 1, "d": 2 }
 ```
 
-#### Blending and layer selection
+### Blending and layer selection
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png
@@ -141,13 +323,7 @@ https://{username}.carto.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
 Some notes about filtering:
 
   - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors.
-  - Once a Mapnik layer is selected, all Mapnik layers will get blended. As this may change in the future **it is
-  recommended** to always select all Mapnik layers if you want to select at least one so you will get a consistent
-  behavior in the future.
-  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this
-  may change in the future **it is recommended** to always select the layers in ascending order so you will get a
-  consistent behavior in the future.
-
+  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this may change in the future, **it is recommended** to always select the layers in ascending order so that you will always get consistent behavior.
 
 ## Create JSONP
 
@@ -188,7 +364,6 @@ callback({
 })
 ```
 
-
 ## Remove
 
 Anonymous Maps cannot be removed by an API call. They will expire after about five minutes, or sometimes longer. If an Anonymous Map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.

docs/general_concepts.md

@@ -22,6 +22,6 @@ Errors are reported using standard HTTP codes and extended information encoded i
 
 If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
 
-## CORS support
+## CORS Support
 
 All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method.

docs/named_maps.md

@@ -1,10 +1,10 @@
 # Named Maps
 
-Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private). 
+Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private).
 
-The Named Map workflow consists of uploading a MapConfig file to CARTO servers, to select data from your CARTO user database by using SQL, and specifying the CartoCSS for your map. 
+The Named Map workflow consists of uploading a MapConfig file to CARTO servers, to select data from your CARTO user database by using SQL, and specifying the CartoCSS for your map.
 
-The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps. 
+The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps.
 
 **Tip:** You can also use a Named Map that you created (which is defined by its `name`), to create a map using CARTO.js. This is achieved by adding the [`namedmap` type](http://docs.carto.com/carto-engine/carto-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
 
@@ -84,6 +84,10 @@ The `name` argument defines how to name this "template_name".json. Note that the
       "south": -45,
       "east": 45,
       "north": 45
+    },
+    "preview_layers": {
+      "0": true,
+      "layer1": false
     }
   }
 }
@@ -95,22 +99,22 @@ Params | Description
 --- | ---
 name | There can only be _one_ template with the same name for any user. Valid names start with a letter or a number, and only contain letters, numbers, dashes (-), or underscores (_). _This is specific to the name of your Named Map that is specified in the `name` property of the template file_.
 
-auth | 
+auth |
 --- | ---
 |_ method | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
 |_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.carto.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
 placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
 layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.carto.com/carto-engine/maps-api/mapconfig/) for more information.
-view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box. Also it is possible to choose which layers are visible or not with `preview_layers` indicating its visibility by layer index or id (visible by default).
 --- | ---
 |_ zoom | The zoom level to use
 
-|_ center | 
+|_ center |
 --- | ---
 |_ |_ lng | The longitude to use for the center
 |_ |_ lat | The latitude to use for the center
 
-|_ bounds | 
+|_ bounds |
 --- | ---
 |_ |_ west | LowerCorner longitude for the bounding box, in decimal degrees (aka most western)
 |_ |_ south | LowerCorner latitude for the bounding box, in decimal degrees (aka most southern)
@@ -120,7 +124,7 @@ view (optional) | extra keys to specify the view area for the map. It can be use
 
 ### Placeholder Format
 
-Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergoup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
+Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergroup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
 
 Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced inside the Named Maps API.
 
@@ -160,7 +164,7 @@ curl -X POST \
 
 #### Response
 
-The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL. 
+The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL.
 
 ```javascript
 {
@@ -516,18 +520,18 @@ If you are creating a Torque layer in a Named Map without using the Torque.js li
     })
     .addTo(map)
     .done(function(layer) {
-  
+
     });
 }
 ```
 
 #### Examples of Named Maps created with CARTO.js
 
-- [Named Map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+- [Named Map selectors with interaction](http://bl.ocks.org/andy-esch/515a8af1f99d5e690484)
 
-- [Named Map with interactivity](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+- [Named Map with interactivity](http://bl.ocks.org/andy-esch/d1a45b8ff5e7bd90cd68)
 
-- [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)
+- [Toggling sublayers in a Named Map](http://bl.ocks.org/andy-esch/c1a0f4913610eec53cd3)
 
 ## Fetching XYZ Tiles for Named Maps
 
@@ -535,7 +539,7 @@ Optionally, authenticated users can fetch projected tiles (XYZ tiles or Mapnik R
 
 ### Fetch XYZ Tiles Directly with a URL
 
-Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map. 
+Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map.
 
 To call a template_id in a URL:
 

docs/static_maps_api.md

@@ -146,10 +146,15 @@ It is important to note that generated images are cached from the live data refe
 
 ### Limits
 
-* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
-* Image resolution by default is set to 72 DPI
-* JPEG quality by default is 85%
-* Timeout limits for generating static maps are the same across the CARTO Editor and CARTO Engine. It is important to ensure timely processing of queries.
+* While images can encompass an entirety of a map, the limit for pixel range is 8192 x 8192.
+* Image resolution is set to 72 DPI
+* JPEG quality is 85%
+* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
+* If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code:
+
+{% highlight javascript %}
+attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors, &copy; <a href="https://carto.com/attributions">CARTO</a>
+{% endhighlight %}
 
 ## Examples
 

lib/cartodb/api/auth_api.js

@@ -19,22 +19,22 @@ function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
 
 module.exports = AuthApi;
 
-// Check if a request is authorized by a signer
+// Check if the user is authorized by a signer
 //
-// @param req express request object
+// @param res express response object
 // @param callback function(err, signed_by) signed_by will be
 //                 null if the request is not signed by anyone
 //                 or will be a string cartodb username otherwise.
 //
-AuthApi.prototype.authorizedBySigner = function(req, callback) {
-    if ( ! req.params.token || ! req.params.signer ) {
+AuthApi.prototype.authorizedBySigner = function(res, callback) {
+    if ( ! res.locals.token || ! res.locals.signer ) {
         return callback(null, false); // no signer requested
     }
 
     var self = this;
 
-    var layergroup_id = req.params.token;
-    var auth_token = req.params.auth_token;
+    var layergroup_id = res.locals.token;
+    var auth_token = res.locals.auth_token;
 
     this.mapStore.load(layergroup_id, function(err, mapConfig) {
         if (err) {
@@ -84,30 +84,29 @@ AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
  * Check access authorization
  *
  * @param req - standard req object. Importantly contains table and host information
+ * @param res - standard res object. Contains the auth parameters in locals
  * @param callback function(err, allowed) is access allowed not?
  */
-AuthApi.prototype.authorize = function(req, callback) {
+AuthApi.prototype.authorize = function(req, res, callback) {
     var self = this;
-    var user = req.context.user;
+    var user = res.locals.user;
 
     step(
         function () {
             self.authorizedByAPIKey(user, req, this);
         },
         function checkApiKey(err, authorized){
-            if (req.profiler) {
-                req.profiler.done('authorizedByAPIKey');
-            }
+            req.profiler.done('authorizedByAPIKey');
             assert.ifError(err);
 
             // if not authorized by api_key, continue
             if (!authorized)  {
                 // not authorized by api_key, check if authorized by signer
-                return self.authorizedBySigner(req, this);
+                return self.authorizedBySigner(res, this);
             }
 
             // authorized by api key, login as the given username and stop
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
                 callback(err, true); // authorized (or error)
             });
         },
@@ -122,7 +121,7 @@ AuthApi.prototype.authorize = function(req, callback) {
                 // if no signer name was given, let dbparams and
                 // PostgreSQL do the rest.
                 //
-                if ( ! req.params.signer ) {
+                if ( ! res.locals.signer ) {
                     return callback(null, true); // authorized so far
                 }
 
@@ -130,10 +129,8 @@ AuthApi.prototype.authorize = function(req, callback) {
                 return callback(null, false);
             }
 
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
-                if (req.profiler) {
-                    req.profiler.done('setDBAuth');
-                }
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
+                req.profiler.done('setDBAuth');
                 callback(err, true); // authorized (or error)
             });
         }

lib/cartodb/api/user_limits_api.js

@@ -1,3 +1,5 @@
+var step = require('step');
+
 /**
  *
  * @param metadataBackend
@@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) {
 
 module.exports = UserLimitsApi;
 
-UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
+UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
     var self = this;
-    this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
+
+    var limits = {
+        cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
+        render: self.options.limits.render || 0
+    };
+
+    self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) {
         if (err) {
             return callback(err);
         }
 
-        return callback(null, {
-            cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
-            render: renderLimit || self.options.limits.render || 0
-        });
+        if (timeoutRenderLimit && timeoutRenderLimit.render) {
+            if (Number.isFinite(timeoutRenderLimit.render)) {
+                limits.render = timeoutRenderLimit.render;
+            }
+        }
+
+        return callback(null, limits);
     });
 };
+
+UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
+    var self = this;
+
+    step(
+        function isAuthorized() {
+            var next = this;
+
+            if (!apiKey) {
+                return next(null, false);
+            }
+
+            self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
+                if (err) {
+                    return next(err);
+                }
+
+                return next(null, userApiKey === apiKey);
+            });
+        },
+        function getUserTimeoutRenderLimits(err, authorized) {
+            var next = this;
+
+            if (err) {
+                return next(err);
+            }
+
+            self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
+                if (err) {
+                    return next(err);
+                }
+
+                next(null, {
+                    render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
+                });
+            });
+        },
+        callback
+    );
+};

lib/cartodb/backends/analysis.js

@@ -1,19 +1,93 @@
-var camshaft = require('camshaft');
+'use strict';
 
-function AnalysisBackend(options) {
-    var batchConfig = options.batch || {};
+var _ = require('underscore');
+var camshaft = require('camshaft');
+var fs = require('fs');
+
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.options.limits = this.options.limits || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
+}
+
+module.exports = AnalysisBackend;
+
+AnalysisBackend.prototype.setBatchConfig = function (options) {
+    var batchConfig = options || {};
     batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job';
     batchConfig.inlineExecution = batchConfig.inlineExecution || false;
     batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan';
     this.batchConfig = batchConfig;
-}
+};
 
-module.exports = AnalysisBackend;
+AnalysisBackend.prototype.setLoggerConfig = function (options) {
+    this.loggerConfig = options || {};
+
+    if (this.loggerConfig.filename) {
+        this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+
+        process.on('SIGHUP', function () {
+            if (this.stream) {
+                this.stream.destroy();
+            }
+
+            this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+        }.bind(this));
+    }
+};
 
 AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) {
     analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
     analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
     analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    analysisConfiguration.logger = {
+        stream: this.stream ? this.stream : process.stdout
+    };
+
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+
+    var analysesLimits = {
+        analyses: {
+            // buffer: {
+            //     timeout: 1000,
+            //     maxNumberOfRows: 1e6
+            // }
+        }
+    };
+
+    Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) {
+        analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]);
+    });
+
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        // analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 }
+        analysesTimeouts = analysesTimeouts || {};
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = _.defaults(
+                {
+                    timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+                },
+                analysesLimits.analyses[analysisType]
+            );
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/dataview.js

@@ -43,50 +43,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             ownFilter = !!ownFilter;
 
             var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
-            var sourceId =  dataviewDefinition.source.id; // node.id
-            var layer = _.find(mapConfig.obj().layers, function(l) {
-                return l.options.source && (l.options.source.id === sourceId);
-            });
-            var queryRewriteData = layer && layer.options.query_rewrite_data;
-            if (queryRewriteData && dataviewDefinition.node.type === 'source') {
-                queryRewriteData = _.extend({}, queryRewriteData, {
-                    filters: dataviewDefinition.node.filters,
-                    unfiltered_query: dataviewDefinition.sql.own_filter_on
-                });
-            }
-
             if (params.bbox) {
                 var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
                 query = bboxFilter.sql(query);
-                if ( queryRewriteData ) {
-                    var bbox_filter_definition = {
-                        type: 'bbox',
-                        options: {
-                            column: 'the_geom_webmercator',
-                            srid: 3857
-                        },
-                        params: {
-                            bbox: params.bbox
-                        }
-                    };
-                    queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
-                }
             }
 
+            var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
+
             var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
                 overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
             );
 
-            var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
-                function castNumbers(overrides, val, k) {
-                    overrides[k] = Number.isFinite(+val) ? +val : val;
-                    return overrides;
-                },
-                {ownFilter: ownFilter}
-            );
-
             var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
-            dataview.getResult(pg, overrideParams, this);
+            dataview.getResult(pg, getOverrideParams(params, ownFilter), this);
         },
         function returnCallback(err, result) {
             return callback(err, result);
@@ -94,9 +63,57 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
     );
 };
 
-DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
-    var dataviewName = params.dataviewName;
+function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
+    var sourceId = dataviewDefinition.source.id; // node.id
+    var layer = _.find(mapConfig.obj().layers, function(l) {
+        return l.options.source && (l.options.source.id === sourceId);
+    });
+    var queryRewriteData = layer && layer.options.query_rewrite_data;
+    if (queryRewriteData && dataviewDefinition.node.type === 'source') {
+        queryRewriteData = _.extend({}, queryRewriteData, {
+            filters: dataviewDefinition.node.filters,
+            unfiltered_query: dataviewDefinition.sql.own_filter_on
+        });
+    }
 
+    if (params.bbox && queryRewriteData) {
+        var bbox_filter_definition = {
+            type: 'bbox',
+            options: {
+                column: 'the_geom_webmercator',
+                srid: 3857
+            },
+            params: {
+                bbox: params.bbox
+            }
+        };
+        queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
+    }
+
+    return queryRewriteData;
+}
+
+function getOverrideParams(params, ownFilter) {
+    var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset', 'categories'),
+        function castNumbers(overrides, val, k) {
+            if (!Number.isFinite(+val)) {
+                throw new Error('Invalid number format for parameter \'' + k + '\'');
+            }
+            overrides[k] = +val;
+            return overrides;
+        },
+        {ownFilter: ownFilter}
+    );
+
+    // validation will be delegated to the proper dataview
+    if (params.aggregation !== undefined) {
+        overrideParams.aggregation = params.aggregation;
+    }
+
+    return overrideParams;
+}
+
+DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) {
     step(
         function getMapConfig() {
             mapConfigProvider.getMapConfig(this);

lib/cartodb/backends/layer-stats/empty-layer-stats.js

@@ -0,0 +1,16 @@
+function EmptyLayerStats(types) {
+    this._types = types || {};
+}
+
+EmptyLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+EmptyLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    setImmediate(function() {
+        callback(null, {});
+    });
+};
+
+module.exports = EmptyLayerStats;

lib/cartodb/backends/layer-stats/factory.js

@@ -0,0 +1,23 @@
+var LayerStats = require('./layer-stats');
+var EmptyLayerStats = require('./empty-layer-stats');
+var MapnikLayerStats = require('./mapnik-layer-stats');
+var TorqueLayerStats = require('./torque-layer-stats');
+
+module.exports = function LayerStatsFactory(type) {
+    var layerStatsIterator = [];
+    var selectedType = type || 'ALL';
+
+    if (selectedType === 'ALL') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
+        layerStatsIterator.push(new MapnikLayerStats());
+        layerStatsIterator.push(new TorqueLayerStats());
+    } else if (selectedType === 'mapnik') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
+        layerStatsIterator.push(new MapnikLayerStats());
+    } else if (selectedType === 'torque') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
+        layerStatsIterator.push(new TorqueLayerStats());
+    }
+
+    return new LayerStats(layerStatsIterator);
+};

lib/cartodb/backends/layer-stats/layer-stats.js

@@ -0,0 +1,45 @@
+var queue = require('queue-async');
+
+function LayerStats(layerStatsIterator) {
+    this.layerStatsIterator = layerStatsIterator;
+}
+
+LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
+    var self = this;
+    var stats = [];
+
+    if (!mapConfig.getLayers().length) {
+        return callback(null, stats);
+    }
+    var metaQueue = queue(mapConfig.getLayers().length);
+    mapConfig.getLayers().forEach(function (layer, layerId) {
+        var layerType = mapConfig.layerType(layerId);
+
+        for (var i = 0; i < self.layerStatsIterator.length; i++) {
+            if (self.layerStatsIterator[i].is(layerType)) {
+                var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
+                metaQueue.defer(getStats, layer, dbConnection);
+                break;
+            }
+        }
+    });
+
+    metaQueue.awaitAll(function (err, results) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!results) {
+            return callback(null, null);
+        }
+
+        mapConfig.getLayers().forEach(function (layer, layerIndex) {
+            stats[layerIndex] = results[layerIndex];
+        });
+
+        return callback(err, stats);
+    });
+
+};
+
+module.exports = LayerStats;

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -0,0 +1,28 @@
+var queryUtils = require('../../utils/query-utils');
+
+function MapnikLayerStats () {
+    this._types = {
+        mapnik: true,
+        cartodb: true
+    };
+}
+
+MapnikLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+MapnikLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
+    // This query would gather stats for postgresql table if not exists
+    dbConnection.query(queryRowCountSql, function (err, res) {
+        if (err) {
+            return callback(null, {estimatedFeatureCount: -1});
+        } else {
+            // We decided that the relation is 1 row == 1 feature
+            return callback(null, {estimatedFeatureCount: res.rows[0].rows});
+        }
+    });
+};
+
+module.exports = MapnikLayerStats;

lib/cartodb/backends/layer-stats/torque-layer-stats.js

@@ -0,0 +1,16 @@
+function TorqueLayerStats() {
+    this._types = {
+        torque: true
+    };
+}
+
+TorqueLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+TorqueLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    return callback(null, {});
+};
+
+module.exports = TorqueLayerStats;

lib/cartodb/backends/stats.js

@@ -0,0 +1,16 @@
+var layerStats = require('./layer-stats/factory');
+
+function StatsBackend() {
+}
+
+module.exports = StatsBackend;
+
+StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
+    var enabledFeatures = global.environment.enabledFeatures;
+    var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
+    if (layerStatsEnabled) {
+        layerStats().getStats(mapConfig, dbConnection, callback);
+    } else {
+        return callback(null, []);
+    }
+};

lib/cartodb/backends/template_maps.js

@@ -55,11 +55,9 @@ util.inherits(TemplateMaps, EventEmitter);
 module.exports = TemplateMaps;
 
 
-var o = TemplateMaps.prototype;
-
 //--------------- PRIVATE METHODS --------------------------------
 
-o._userTemplateLimit = function() {
+TemplateMaps.prototype._userTemplateLimit = function() {
   return this.opts.max_user_templates || 0;
 };
 
@@ -70,7 +68,7 @@ o._userTemplateLimit = function() {
  * @param redisArgs - the arguments for the redis function in an array
  * @param callback - function to pass results too.
  */
-o._redisCmd = function(redisFunc, redisArgs, callback) {
+TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
   var redisClient;
   var that = this;
   var db = that.db_signatures;
@@ -97,7 +95,7 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
 var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
 // jshint maxcomplexity:15
-o._checkInvalidTemplate = function(template) {
+TemplateMaps.prototype._checkInvalidTemplate = function(template) {
   if ( template.version !== '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
   }
@@ -200,7 +198,7 @@ function templateDefaults(template) {
 // @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
-o.addTemplate = function(owner, template, callback) {
+TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
     var self = this;
 
     template = templateDefaults(template);
@@ -258,7 +256,7 @@ o.addTemplate = function(owner, template, callback) {
 //
 // @param callback function(err)
 //
-o.delTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function deleteTemplate() {
@@ -297,7 +295,8 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //
 // @param callback function(err)
 //
-o.updTemplate = function(owner, tpl_id, template, callback) {
+TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
+
     var self = this;
 
     template = templateDefaults(template);
@@ -356,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
 //
-o.listTemplates = function(owner, callback) {
+TemplateMaps.prototype.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
 
@@ -370,7 +369,7 @@ o.listTemplates = function(owner, callback) {
 // @param callback function(err, template)
 //        Return full template definition
 //
-o.getTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function getTemplate() {
@@ -386,7 +385,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
     );
 };
 
-o.isAuthorized = function(template, authTokens) {
+TemplateMaps.prototype.isAuthorized = function(template, authTokens) {
     if (!template) {
         return false;
     }
@@ -431,14 +430,18 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
     _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
 
 function _replaceVars (str, params) {
-  //return _.template(str, params); // lazy way, possibly dangerous
-  // Construct regular expressions for each param
+    // Construct regular expressions for each param
     Object.keys(params).forEach(function(k) {
         str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
     });
     return str;
 }
-o.instance = function(template, params) {
+
+function isObject(val) {
+    return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val));
+}
+
+TemplateMaps.prototype.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
   Object.keys(phold).forEach(function(k) {
@@ -475,6 +478,13 @@ o.instance = function(template, params) {
 
   // NOTE: we're deep-cloning the layergroup here
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
+
+  if (layergroup.buffersize && isObject(layergroup.buffersize)) {
+      Object.keys(layergroup.buffersize).forEach(function(k) {
+        layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10);
+      });
+  }
+
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
 
@@ -500,7 +510,7 @@ o.instance = function(template, params) {
 };
 
 // Return a fingerPrint of the object
-o.fingerPrint = function(template) {
+TemplateMaps.prototype.fingerPrint = function(template) {
   return crypto.createHash('md5')
     .update(JSON.stringify(template))
     .digest('hex')

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -6,16 +6,25 @@ dot.templateSettings.strip = false;
 function createTemplate(method) {
     return dot.template([
         'SELECT',
+        'min({{=it._column}}) min_val,',
+        'max({{=it._column}}) max_val,',
+        'avg({{=it._column}}) avg_val,',
         method,
-        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL',
+        'AND',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float'
     ].join('\n'));
 }
 
 var methods = {
-    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles',
     equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
-    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
-    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+    jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails'
 };
 
 var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {
@@ -29,7 +38,7 @@ methodTemplates.category = dot.template([
     '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
     '  FROM ({{=it._sql}}) _cdb_aggregation_all',
     '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
+    '  ORDER BY 2 DESC, 1 ASC',
     '),',
     'agg_categories AS (',
     '  SELECT category',
@@ -74,11 +83,14 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             return callback(err);
         }
 
-        resultSet = resultSet || {};
-        var result = resultSet.rows || [];
-
+        var result = getResult(resultSet);
         var strategy = method2strategy[methodName];
-        var ramp = result[0][methodName] || [];
+        var ramp = result[methodName] || [];
+        var stats = {
+            min_val: result.min_val,
+            max_val: result.max_val,
+            avg_val: result.avg_val
+        };
         // Skip null values from ramp
         // Generated turbo-carto won't be correct, but better to keep it working than failing
         // TODO fix cartodb-postgres extension quantification functions
@@ -89,8 +101,16 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             });
         }
 
-        return callback(null, { ramp: ramp, strategy: strategy });
+        return callback(null, { ramp: ramp, strategy: strategy, stats: stats });
     }, true); // use read-only transaction
 };
 
+function getResult(resultSet) {
+    resultSet = resultSet || {};
+    var result = resultSet.rows || [];
+    result = result[0] || {};
+
+    return result;
+}
+
 module.exports = PostgresDatasource;

lib/cartodb/controllers/analyses.js

@@ -1,152 +1,150 @@
-var step = require('step');
-var assert = require('assert');
-var dot = require('dot');
-dot.templateSettings.strip = false;
 var PSQL = require('cartodb-psql');
-
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-function AnalysesController(authApi, pgConnection) {
-    BaseController.call(this, authApi, pgConnection);
+function AnalysesController(prepareContext) {
+    this.prepareContext = prepareContext;
 }
 
-util.inherits(AnalysesController, BaseController);
-
 module.exports = AnalysesController;
 
-AnalysesController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig + '/analyses/catalog', cors(), userMiddleware, this.catalog.bind(this));
-};
-
-AnalysesController.prototype.sendResponse = function(req, res, resource) {
-    res.set('Cache-Control', 'public,max-age=10,must-revalidate');
-    this.send(req, res, resource, 200);
-};
-
-AnalysesController.prototype.catalog = function(req, res) {
-    var self = this;
-    var username = req.context.user;
-
-    step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function catalogQuery(err) {
-            assert.ifError(err);
-            var pg = new PSQL(dbParamsFromReqParams(req.params));
-            getMetadata(username, pg, this);
-        },
-        function prepareResponse(err, results) {
-            assert.ifError(err);
-
-            var analysisIdToTable = results.tables.reduce(function(analysisIdToTable, table) {
-                var analysisId = table.relname.split('_')[2];
-                if (analysisId && analysisId.length === 40) {
-                    analysisIdToTable[analysisId] = table;
-                }
-                return analysisIdToTable;
-            }, {});
-
-            var catalogWithTables = results.catalog.map(function(analysis) {
-                if (analysisIdToTable.hasOwnProperty(analysis.node_id)) {
-                    analysis.table = analysisIdToTable[analysis.node_id];
-                }
-                return analysis;
-            });
-
-            return catalogWithTables.sort(function(analysisA, analysisB) {
-                if (!!analysisA.table && !!analysisB.table) {
-                    return analysisB.table.size - analysisA.table.size;
-                }
-
-                if (!!analysisA.table) {
-                    return -1;
-                }
-
-                if (!!analysisB.table) {
-                    return 1;
-                }
-
-                return -1;
-            });
-        },
-        function sendResponse(err, catalogWithTables) {
-            if (err) {
-                if (err.message.match(/permission\sdenied/)) {
-                    err = new Error('Unauthorized');
-                    err.http_status = 401;
-                }
-                self.sendError(req, res, err);
-            } else {
-                self.sendResponse(req, res, { catalog: catalogWithTables });
-            }
-        }
+AnalysesController.prototype.register = function (app) {
+    app.get(
+        `${app.base_url_mapconfig}/analyses/catalog`,
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.createPGClient(),
+        this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),
+        this.getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }),
+        this.prepareResponse(),
+        this.setCacheControlHeader(),
+        this.sendResponse(),
+        this.unathorizedError()
     );
 };
 
-var catalogQueryTpl = dot.template(
-    'SELECT analysis_def->>\'type\' as type, * FROM cartodb.cdb_analysis_catalog WHERE username = \'{{=it._username}}\''
-);
-
-var tablesQueryTpl = dot.template([
-    "WITH analysis_tables AS (",
-    "    SELECT",
-    "        n.nspname AS nspname,",
-    "        c.relname AS relname,",
-    "        pg_total_relation_size(",
-    "            format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))",
-    "        ) AS size,",
-    "        format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name",
-    "    FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n",
-    "    WHERE c.relnamespace = n.oid",
-    "    AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'",
-    "    AND n.nspname IN ('{{=it._username}}', 'public')",
-    ")",
-    "SELECT *, pg_size_pretty(size) as size_pretty",
-    "FROM analysis_tables",
-    "ORDER BY size DESC"
-].join('\n'));
-
-
-function getMetadata(username, pg, callback) {
-    var results = {
-        catalog: [],
-        tables: []
+AnalysesController.prototype.createPGClient = function () {
+    return function createPGClientMiddleware (req, res, next) {
+        res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals));
+        next();
     };
-    step(
-        function getCatalog() {
-            pg.query(catalogQueryTpl({_username: username}), this, true); // use read-only transaction
-        },
-        function handleCatalog(err, resultSet) {
-            assert.ifError(err);
-            resultSet = resultSet || {};
-            results.catalog = resultSet.rows || [];
-            this();
-        },
-        function getTables(err) {
-            assert.ifError(err);
-            pg.query(tablesQueryTpl({_username: username}), this, true); // use read-only transaction
-        },
-        function handleTables(err, resultSet) {
-            assert.ifError(err);
-            resultSet = resultSet || {};
-            results.tables = resultSet.rows || [];
-            this();
-        },
-        function finish(err) {
+};
+
+AnalysesController.prototype.getDataFromQuery = function ({ queryTemplate, key }) {
+    const readOnlyTransactionOn = true;
+
+    return function getCatalogMiddleware(req, res, next) {
+        const { pg, user } = res.locals;
+        const sql = queryTemplate({ _username: user });
+
+        pg.query(sql, (err, resultSet = {}) => {
             if (err) {
-                return callback(err);
+                return next(err);
             }
 
-            return callback(null, results);
-        }
-    );
-}
+            res.locals[key] = resultSet.rows || [];
 
+            next();
+        }, readOnlyTransactionOn);
+    };
+};
+
+AnalysesController.prototype.prepareResponse = function () {
+    return function prepareResponseMiddleware (req, res, next) {
+        const { catalog, tables } = res.locals;
+
+        const analysisIdToTable = tables.reduce((analysisIdToTable, table) => {
+            const analysisId = table.relname.split('_')[2];
+
+            if (analysisId && analysisId.length === 40) {
+                analysisIdToTable[analysisId] = table;
+            }
+
+            return analysisIdToTable;
+        }, {});
+
+        const analysisCatalog = catalog.map(analysis => {
+            if (analysisIdToTable.hasOwnProperty(analysis.node_id)) {
+                analysis.table = analysisIdToTable[analysis.node_id];
+            }
+
+            return analysis;
+        })
+        .sort((analysisA, analysisB) => {
+            if (!!analysisA.table && !!analysisB.table) {
+                return analysisB.table.size - analysisA.table.size;
+            }
+
+            if (!!analysisA.table) {
+                return -1;
+            }
+
+            if (!!analysisB.table) {
+                return 1;
+            }
+
+            return -1;
+        });
+
+        res.body = { catalog: analysisCatalog };
+
+        next();
+    };
+};
+
+AnalysesController.prototype.setCacheControlHeader = function () {
+    return function setCacheControlHeaderMiddleware (req, res, next) {
+        res.set('Cache-Control', 'public,max-age=10,must-revalidate');
+        next();
+    };
+};
+
+AnalysesController.prototype.sendResponse = function() {
+    return function sendResponseMiddleware (req, res) {
+        res.status(200);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(res.body);
+        } else {
+            res.json(res.body);
+        }
+    };
+};
+
+AnalysesController.prototype.unathorizedError = function () {
+    return function unathorizedErrorMiddleware(err, req, res, next) {
+        if (err.message.match(/permission\sdenied/)) {
+            err = new Error('Unauthorized');
+            err.http_status = 401;
+        }
+
+        next(err);
+    };
+};
+
+const catalogQueryTpl = ctx => `
+    SELECT analysis_def->>'type' as type, * FROM cdb_analysis_catalog WHERE username = '${ctx._username}'
+`;
+
+var tablesQueryTpl = ctx => `
+    WITH analysis_tables AS (
+        SELECT
+            n.nspname AS nspname,
+            c.relname AS relname,
+            pg_total_relation_size(
+                format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))
+            ) AS size,
+            format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name
+        FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n
+        WHERE c.relnamespace = n.oid
+        AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'
+        AND n.nspname IN ('${ctx._username}', 'public')
+    )
+    SELECT *, pg_size_pretty(size) as size_pretty
+    FROM analysis_tables
+    ORDER BY size DESC
+`;
 
 function dbParamsFromReqParams(params) {
     var dbParams = {};

lib/cartodb/controllers/base.js

@@ -1,295 +0,0 @@
-var assert = require('assert');
-
-var _ = require('underscore');
-var step = require('step');
-var debug = require('debug')('windshaft:cartodb');
-
-var LZMA = require('lzma').LZMA;
-var lzmaWorker = new LZMA();
-
-// Whitelist query parameters and attach format
-var REQUEST_QUERY_PARAMS_WHITELIST = [
-    'config',
-    'map_key',
-    'api_key',
-    'auth_token',
-    'callback',
-    'zoom',
-    'lon',
-    'lat',
-    // widgets & filters
-    'filters', // json
-    'own_filter', // 0, 1
-    'bbox', // w,s,e,n
-    'bins', // number
-    'start', // number
-    'end', // number
-    'column_type', // string
-    // widgets search
-    'q'
-];
-
-function BaseController(authApi, pgConnection) {
-    this.authApi = authApi;
-    this.pgConnection = pgConnection;
-}
-
-module.exports = BaseController;
-
-// jshint maxcomplexity:9
-/**
- * Whitelist input and get database name & default geometry type from
- * subdomain/user metadata held in CartoDB Redis
- * @param req - standard express request obj. Should have host & table
- * @param callback
- */
-BaseController.prototype.req2params = function(req, callback){
-    var self = this;
-
-    if ( req.query.lzma ) {
-
-        // Decode (from base64)
-        var lzma = new Buffer(req.query.lzma, 'base64')
-            .toString('binary')
-            .split('')
-            .map(function(c) {
-                return c.charCodeAt(0) - 128;
-            });
-
-
-        // Decompress
-        lzmaWorker.decompress(
-            lzma,
-            function(result) {
-                if (req.profiler) {
-                    req.profiler.done('lzma');
-                }
-                try {
-                    delete req.query.lzma;
-                    _.extend(req.query, JSON.parse(result));
-                    self.req2params(req, callback);
-                } catch (err) {
-                    req.profiler.done('req2params');
-                    callback(new Error('Error parsing lzma as JSON: ' + err));
-                }
-            }
-        );
-        return;
-    }
-
-    req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
-    req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
-
-    var user = req.context.user;
-
-    if ( req.params.token ) {
-        // Token might match the following patterns:
-        // - {user}@{tpl_id}@{token}:{cache_buster}
-        var tksplit = req.params.token.split(':');
-        req.params.token = tksplit[0];
-        if ( tksplit.length > 1 ) {
-            req.params.cache_buster= tksplit[1];
-        }
-        tksplit = req.params.token.split('@');
-        if ( tksplit.length > 1 ) {
-            req.params.signer = tksplit.shift();
-            if ( ! req.params.signer ) {
-                req.params.signer = user;
-            }
-            else if ( req.params.signer !== user ) {
-                var err = new Error(
-                        'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"'
-                );
-                err.http_status = 403;
-                req.profiler.done('req2params');
-                callback(err);
-                return;
-            }
-            if ( tksplit.length > 1 ) {
-                /*var template_hash = */tksplit.shift(); // unused
-            }
-            req.params.token = tksplit.shift();
-        }
-    }
-
-    // bring all query values onto req.params object
-    _.extend(req.params, req.query);
-
-    if (req.profiler) {
-        req.profiler.done('req2params.setup');
-    }
-
-    step(
-        function getPrivacy(){
-            self.authApi.authorize(req, this);
-        },
-        function validateAuthorization(err, authorized) {
-            if (req.profiler) {
-                req.profiler.done('authorize');
-            }
-            assert.ifError(err);
-            if(!authorized) {
-                err = new Error("Sorry, you are unauthorized (permission denied)");
-                err.http_status = 403;
-                throw err;
-            }
-            return null;
-        },
-        function getDatabase(err){
-            assert.ifError(err);
-            self.pgConnection.setDBConn(user, req.params, this);
-        },
-        function finishSetup(err) {
-            if ( err ) {
-                req.profiler.done('req2params');
-                return callback(err, req);
-            }
-
-            // Add default database connection parameters
-            // if none given
-            _.defaults(req.params, {
-                dbuser: global.environment.postgres.user,
-                dbpassword: global.environment.postgres.password,
-                dbhost: global.environment.postgres.host,
-                dbport: global.environment.postgres.port
-            });
-
-            req.profiler.done('req2params');
-            callback(null, req);
-        }
-    );
-};
-// jshint maxcomplexity:6
-
-// jshint maxcomplexity:9
-BaseController.prototype.send = function(req, res, body, status, headers) {
-    if (req.params.dbhost) {
-        res.set('X-Served-By-DB-Host', req.params.dbhost);
-    }
-
-    if (req.profiler) {
-        res.set('X-Tiler-Profiler', req.profiler.toJSONString());
-    }
-
-    if (headers) {
-        res.set(headers);
-    }
-
-    res.status(status);
-
-    if (!Buffer.isBuffer(body) && typeof body === 'object') {
-        if (req.query && req.query.callback) {
-            res.jsonp(body);
-        } else {
-            res.json(body);
-        }
-    } else {
-        res.send(body);
-    }
-
-    if (req.profiler) {
-        try {
-            // May throw due to dns, see
-            // See http://github.com/CartoDB/Windshaft/issues/166
-            req.profiler.sendStats();
-        } catch (err) {
-            debug("error sending profiling stats: " + err);
-        }
-    }
-};
-// jshint maxcomplexity:6
-
-BaseController.prototype.sendError = function(req, res, err, label) {
-    var allErrors = Array.isArray(err) ? err : [err];
-    label = label || 'UNKNOWN';
-    err = allErrors[0] || new Error(label);
-    allErrors[0] = err;
-
-    var statusCode = findStatusCode(err);
-
-    debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
-
-    // If a callback was requested, force status to 200
-    if (req.query && req.query.callback) {
-        statusCode = 200;
-    }
-
-    var errorResponseBody = {
-        errors: allErrors.map(errorMessage),
-        errors_with_context: allErrors.map(errorMessageWithContext)
-    };
-
-    this.send(req, res, errorResponseBody, statusCode);
-};
-
-function stripConnectionInfo(message) {
-    // Strip connection info, if any
-    return message
-        // See https://github.com/CartoDB/Windshaft/issues/173
-        .replace(/Connection string: '[^']*'\n\s/im, '')
-        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
-        .replace(/is the server.*encountered/im, 'encountered');
-}
-
-function errorMessage(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    return stripConnectionInfo(message);
-}
-
-function errorMessageWithContext(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    var error = {
-        type: err.type || 'unknown',
-        message: stripConnectionInfo(message),
-    };
-
-    for (var prop in err) {
-        // type & message are properties from Error's prototype and will be skipped
-        if (err.hasOwnProperty(prop)) {
-            error[prop] = err[prop];
-        }
-    }
-
-    return error;
-}
-module.exports.errorMessage = errorMessage;
-
-function findStatusCode(err) {
-    var statusCode;
-    if ( err.http_status ) {
-        statusCode = err.http_status;
-    } else {
-        statusCode = statusFromErrorMessage('' + err);
-    }
-    return statusCode;
-}
-module.exports.findStatusCode = findStatusCode;
-
-function statusFromErrorMessage(errMsg) {
-    // Find an appropriate statusCode based on message
-    // jshint maxcomplexity:7
-    var statusCode = 400;
-    if ( -1 !== errMsg.indexOf('permission denied') ) {
-        statusCode = 403;
-    }
-    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
-        statusCode = 403;
-    }
-    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
-        statusCode = 400;
-    }
-    else if ( -1 !== errMsg.indexOf('does not exist') ) {
-        if ( -1 !== errMsg.indexOf(' role ') ) {
-            statusCode = 403; // role 'xxx' does not exist
-        } else if ( errMsg.match(/function .* does not exist/) ) {
-            statusCode = 400; // invalid SQL (SQL function does not exist)
-        } else {
-            statusCode = 404;
-        }
-    }
-    return statusCode;
-}

lib/cartodb/controllers/layergroup.js

@@ -1,11 +1,10 @@
 var assert = require('assert');
 var step = require('step');
 
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
+var vectorError = require('../middleware/vector-error');
 
 var DataviewBackend = require('../backends/dataview');
 var AnalysisStatusBackend = require('../backends/analysis-status');
@@ -27,10 +26,8 @@ var QueryTables = require('cartodb-query-tables');
  * @param {AnalysisBackend} analysisBackend
  * @constructor
  */
-function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
+function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
                               surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
-    BaseController.call(this, authApi, pgConnection);
-
     this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
@@ -42,75 +39,142 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
 
     this.dataviewBackend = new DataviewBackend(analysisBackend);
     this.analysisStatusBackend = new AnalysisStatusBackend();
-}
 
-util.inherits(LayergroupController, BaseController);
+    this.prepareContext = prepareContext;
+}
 
 module.exports = LayergroupController;
 
-
 LayergroupController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y@:scale_factor?x.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.layer.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        validateLayerRouteMiddleware,
+        this.prepareContext,
+        this.layer.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/attributes/:fid', cors(), userMiddleware,
-        this.attributes.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/attributes/:fid',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.attributes.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
-        this.center.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.center.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
-        this.bbox.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.bbox.bind(this)
+    );
 
     // Undocumented/non-supported API endpoint methods.
     // Use at your own peril.
-    app.get(app.base_url_mapconfig +
-            '/:token/dataview/:dataviewName', cors(), userMiddleware,
-        this.dataview.bind(this));
-    app.get(app.base_url_mapconfig +
-            '/:token/:layer/widget/:dataviewName', cors(), userMiddleware,
-        this.dataview.bind(this));
 
-    app.get(app.base_url_mapconfig +
-            '/:token/dataview/:dataviewName/search', cors(), userMiddleware,
-        this.dataviewSearch.bind(this));
-    app.get(app.base_url_mapconfig +
-            '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware,
-        this.dataviewSearch.bind(this));
+    var allowedDataviewQueryParams = [
+        'filters', // json
+        'own_filter', // 0, 1
+        'bbox', // w,s,e,n
+        'start', // number
+        'end', // number
+        'column_type', // string
+        'bins', // number
+        'aggregation', //string
+        'offset', // number
+        'q', // widgets search
+        'categories', // number
+    ];
 
-    app.get(app.base_url_mapconfig +
-        '/:token/analysis/node/:nodeId', cors(), userMiddleware,
-        this.analysisNodeStatus.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/dataview/:dataviewName',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataview.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataview.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataviewSearch.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataviewSearch.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/analysis/node/:nodeId',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.analysisNodeStatus.bind(this)
+    );
 };
 
-LayergroupController.prototype.analysisNodeStatus = function(req, res) {
+LayergroupController.prototype.analysisNodeStatus = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveNodeStatus(err) {
-            assert.ifError(err);
-            self.analysisStatusBackend.getNodeStatus(req.params, this);
+        function retrieveNodeStatus() {
+            self.analysisStatusBackend.getNodeStatus(res.locals, this);
         },
         function finish(err, nodeStatus, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET NODE STATUS');
+                err.label = 'GET NODE STATUS';
+                next(err);
             } else {
                 self.sendResponse(req, res, nodeStatus, 200, {
                     'Cache-Control': 'public,max-age=5',
@@ -121,54 +185,50 @@ LayergroupController.prototype.analysisNodeStatus = function(req, res) {
     );
 };
 
-LayergroupController.prototype.dataview = function(req, res) {
+LayergroupController.prototype.dataview = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveDataview(err) {
-            assert.ifError(err);
-
+        function retrieveDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+            );
+            self.dataviewBackend.getDataview(
+                mapConfigProvider,
+                res.locals.user,
+                res.locals,
+                this
             );
-            self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this);
         },
         function finish(err, dataview, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW');
+                err.label = 'GET DATAVIEW';
+                next(err);
             } else {
                 self.sendResponse(req, res, dataview, 200);
             }
         }
     );
-
 };
 
-LayergroupController.prototype.dataviewSearch = function(req, res) {
+LayergroupController.prototype.dataviewSearch = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function searchDataview(err) {
-            assert.ifError(err);
-
+        function searchDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this);
+            self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this);
         },
         function finish(err, searchResult, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW SEARCH');
+                err.label = 'GET DATAVIEW SEARCH';
+                next(err);
             } else {
                 self.sendResponse(req, res, searchResult, 200);
             }
@@ -177,28 +237,24 @@ LayergroupController.prototype.dataviewSearch = function(req, res) {
 
 };
 
-LayergroupController.prototype.attributes = function(req, res) {
+LayergroupController.prototype.attributes = function(req, res, next) {
     var self = this;
 
     req.profiler.start('windshaft.maplayer_attribute');
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveFeatureAttributes(err) {
-            assert.ifError(err);
-
+        function retrieveFeatureAttributes() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.attributesBackend.getFeatureAttributes(mapConfigProvider, req.params, false, this);
+            self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this);
         },
         function finish(err, tile, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET ATTRIBUTES');
+                err.label = 'GET ATTRIBUTES';
+                next(err);
             } else {
                 self.sendResponse(req, res, tile, 200);
             }
@@ -208,49 +264,48 @@ LayergroupController.prototype.attributes = function(req, res) {
 };
 
 // Gets a tile for a given token and set of tile ZXY coords. (OSM style)
-LayergroupController.prototype.tile = function(req, res) {
+LayergroupController.prototype.tile = function(req, res, next) {
     req.profiler.start('windshaft.map_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
 // Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
 LayergroupController.prototype.layer = function(req, res, next) {
-    if (req.params.token === 'static') {
-        return next();
-    }
     req.profiler.start('windshaft.maplayer_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
-LayergroupController.prototype.tileOrLayer = function (req, res) {
+LayergroupController.prototype.tileOrLayer = function (req, res, next) {
     var self = this;
 
     step(
-        function mapController$prepareParams() {
-            self.req2params(req, this);
-        },
-        function mapController$getTileOrGrid(err) {
-            assert.ifError(err);
+        function mapController$getTileOrGrid() {
             self.tileBackend.getTile(
-                new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                 req.params, this
             );
         },
         function mapController$finalize(err, tile, headers, stats) {
             req.profiler.add(stats);
-            self.finalizeGetTileOrGrid(err, req, res, tile, headers);
+            self.finalizeGetTileOrGrid(err, req, res, tile, headers, next);
         }
     );
 };
 
+function getStatusCode(tile, format){
+    return tile.length===0 && format==='mvt'? 204:200;
+}
+
 // This function is meant for being called as the very last
 // step by all endpoints serving tiles or grids
-LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) {
+LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
     var supportedFormats = {
         grid_json: true,
         json_torque: true,
         torque_json: true,
-        png: true
+        png: true,
+        png32: true,
+        mvt: true
     };
 
     var formatStat = 'invalid';
@@ -272,52 +327,50 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
         }
         err.message = errMsg;
 
-        this.sendError(req, res, err, 'TILE RENDER');
+        err.label = 'TILE RENDER';
+        next(err);
+
         global.statsClient.increment('windshaft.tiles.error');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
     } else {
-        this.sendResponse(req, res, tile, 200, headers);
+        this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
         global.statsClient.increment('windshaft.tiles.success');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
     }
 };
 
-LayergroupController.prototype.bbox = function(req, res) {
+LayergroupController.prototype.bbox = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, {
         west: +req.params.west,
         north: +req.params.north,
         east: +req.params.east,
         south: +req.params.south
-    });
+    }, null, next);
 };
 
-LayergroupController.prototype.center = function(req, res) {
+LayergroupController.prototype.center = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
         lng: +req.params.lng,
         lat: +req.params.lat
-    });
+    }, next);
 };
 
-LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) {
+LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.layer = 'all';
-    req.params.format = 'png';
+    req.params.format = req.params.format || 'png';
+    res.locals.layer = res.locals.layer || 'all';
 
     var self = this;
 
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getImage(err) {
-            assert.ifError(err);
+        function getImage() {
             if (center) {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom, center, this);
             } else {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom /* bounds */, this);
             }
         },
@@ -326,7 +379,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'STATIC_MAP');
+                err.label = 'STATIC_MAP';
+                next(err);
             } else {
                 res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
                 self.sendResponse(req, res, image, 200);
@@ -338,22 +392,24 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
 LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
     var self = this;
 
+    req.profiler.done('res');
+
     res.set('Cache-Control', 'public,max-age=31536000');
 
     // Set Last-Modified header
     var lastUpdated;
-    if (req.params.cache_buster) {
+    if (res.locals.cache_buster) {
         // Assuming cache_buster is a timestamp
-        lastUpdated = new Date(parseInt(req.params.cache_buster));
+        lastUpdated = new Date(parseInt(res.locals.cache_buster));
     } else {
         lastUpdated = new Date();
     }
     res.set('Last-Modified', lastUpdated.toUTCString());
 
-    var dbName = req.params.dbname;
+    var dbName = res.locals.dbname;
     step(
         function getAffectedTables() {
-            self.getAffectedTables(req.context.user, dbName, req.params.token, this);
+            self.getAffectedTables(res.locals.user, dbName, res.locals.token, this);
         },
         function sendResponse(err, affectedTables) {
             req.profiler.done('affectedTables');
@@ -364,10 +420,24 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 res.set('X-Cache-Channel', affectedTables.getCacheChannel());
                 self.surrogateKeysCache.tag(res, affectedTables);
             }
-            self.send(req, res, body, status, headers);
+
+            if (headers) {
+                res.set(headers);
+            }
+
+            res.status(status);
+
+            if (!Buffer.isBuffer(body) && typeof body === 'object') {
+                if (req.query && req.query.callback) {
+                    res.jsonp(body);
+                } else {
+                    res.json(body);
+                }
+            } else {
+                res.send(body);
+            }
         }
     );
-
 };
 
 LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
@@ -386,13 +456,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 function getSQL(err, mapConfig) {
                     assert.ifError(err);
 
-                    var queries = mapConfig.getLayers()
-                        .map(function(lyr) {
-                            return lyr.options.sql;
-                        })
-                        .filter(function(sql) {
-                            return !!sql;
-                        });
+                    var queries = [];
+                    mapConfig.getLayers().forEach(function(layer) {
+                        queries.push(layer.options.sql);
+                        if (layer.options.affected_tables) {
+                            layer.options.affected_tables.map(function(table) {
+                                queries.push('SELECT * FROM ' + table + ' LIMIT 0');
+                            });
+                        }
+                    });
 
                     return queries.length ? queries.join(';') : null;
                 },
@@ -427,3 +499,12 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
         callback
     );
 };
+
+
+function validateLayerRouteMiddleware(req, res, next) {
+    if (req.params.token === 'static') {
+        return next('route');
+    }
+
+    next();
+}

lib/cartodb/controllers/map.js

@@ -1,11 +1,8 @@
 var _ = require('underscore');
-var assert = require('assert');
-var step = require('step');
 var windshaft = require('windshaft');
 var QueryTables = require('cartodb-query-tables');
 
-var util = require('util');
-var BaseController = require('./base');
+var ResourceLocator = require('../models/resource-locator');
 
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
@@ -28,13 +25,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @param {MapConfigAdapter} mapConfigAdapter
+ * @param {StatsBackend} statsBackend
  * @constructor
  */
-function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
-
-    BaseController.call(this, authApi, pgConnection);
-
+function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend,
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
+                       statsBackend) {
     this.pgConnection = pgConnection;
     this.templateMaps = templateMaps;
     this.mapBackend = mapBackend;
@@ -44,285 +40,348 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.mapConfigAdapter = mapConfigAdapter;
-}
+    this.resourceLocator = new ResourceLocator(global.environment);
 
-util.inherits(MapController, BaseController);
+    this.statsBackend = statsBackend;
+    this.prepareContext = prepareContext;
+}
 
 module.exports = MapController;
 
-
 MapController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig, cors(), userMiddleware, this.createGet.bind(this));
-    app.post(app.base_url_mapconfig, cors(), userMiddleware, this.createPost.bind(this));
-    app.get(app.base_url_templated + '/:template_id/jsonp', cors(), userMiddleware, this.jsonp.bind(this));
-    app.post(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.instantiate.bind(this));
+    const { base_url_mapconfig, base_url_templated } = app;
+    const useTemplate = true;
+
+    app.get(base_url_mapconfig, this.composeCreateMapMiddleware());
+    app.post(base_url_mapconfig, this.composeCreateMapMiddleware());
+    app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate));
+    app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate));
     app.options(app.base_url_mapconfig, cors('Content-Type'));
 };
 
-MapController.prototype.createGet = function(req, res){
-    req.profiler.start('windshaft.createmap_get');
+MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) {
+    const isTemplateInstantiation = useTemplate;
+    const useTemplateHash = useTemplate;
+    const includeQuery = !useTemplate;
+    const label =  useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP';
+    const addContext = !useTemplate;
 
-    this.create(req, res, function createGet$prepareConfig(err, req) {
-        assert.ifError(err);
-        if ( ! req.params.config ) {
-            throw new Error('layergroup GET needs a "config" parameter');
-        }
-        return JSON.parse(req.params.config);
-    });
+    return [
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.initProfiler(isTemplateInstantiation),
+        this.checkJsonContentType(),
+        useTemplate ? this.checkInstantiteLayergroup() : this.checkCreateLayergroup(),
+        useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(),
+        useTemplate ? this.instantiateLayergroup() : this.createLayergroup(),
+        this.incrementMapViewCount(),
+        this.augmentLayergroupData(),
+        this.getAffectedTables(),
+        this.setCacheChannel(),
+        this.setLastModified(),
+        this.setLastUpdatedTimeToLayergroup(),
+        this.setCacheControl(),
+        this.setLayerStats(),
+        this.setLayergroupIdHeader(useTemplateHash),
+        this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(),
+        this.setAnalysesMetadataToLayergroup(includeQuery),
+        this.setTurboCartoMetadataToLayergroup(),
+        this.setSurrogateKeyHeader(),
+        this.sendResponse(),
+        this.augmentError({ label, addContext })
+    ];
 };
 
-MapController.prototype.createPost = function(req, res) {
-    req.profiler.start('windshaft.createmap_post');
+MapController.prototype.initProfiler = function (isTemplateInstantiation) {
+    const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
 
-    this.create(req, res, function createPost$prepareConfig(err, req) {
-        assert.ifError(err);
-        if (!req.is('application/json')) {
-            throw new Error('layergroup POST data must be of type application/json');
-        }
-        return req.body;
-    });
+    return function initProfilerMiddleware (req, res, next) {
+        req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`);
+        req.profiler.done(`${operation}.initProfilerMiddleware`);
+        next();
+    };
 };
 
-MapController.prototype.instantiate = function(req, res) {
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.instance_template_post');
-    }
-
-    this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
-        if (!req.is('application/json')) {
-            return callback(new Error('Template POST data must be of type application/json'));
+MapController.prototype.checkJsonContentType = function () {
+    return function checkJsonContentTypeMiddleware(req, res, next) {
+        if (req.method === 'POST' && !req.is('application/json')) {
+            return next(new Error('POST data must be of type application/json'));
         }
-        return callback(null, req.body);
-    });
+
+        req.profiler.done('checkJsonContentTypeMiddleware');
+
+        next();
+    };
 };
 
-MapController.prototype.jsonp = function(req, res) {
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.instance_template_get');
-    }
+MapController.prototype.checkInstantiteLayergroup = function () {
+    return function checkInstantiteLayergroupMiddleware(req, res, next) {
+        if (req.method === 'GET') {
+            const { callback, config } = req.query;
 
-    this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
-        var err = null;
-        if ( req.query.callback === undefined || req.query.callback.length === 0) {
-            err = new Error('callback parameter should be present and be a function name');
-        }
+            if (callback === undefined || callback.length === 0) {
+                return next(new Error('callback parameter should be present and be a function name'));
+            }
 
-        var templateParams = {};
-        if (req.query.config) {
-            try {
-                templateParams = JSON.parse(req.query.config);
-            } catch(e) {
-                err = new Error('Invalid config parameter, should be a valid JSON');
+            if (config) {
+                try {
+                    req.body = JSON.parse(config);
+                } catch(e) {
+                    return next(new Error('Invalid config parameter, should be a valid JSON'));
+                }
             }
         }
 
-        return callback(err, templateParams);
-    });
+        req.profiler.done('checkInstantiteLayergroup');
+
+        return next();
+    };
 };
 
-MapController.prototype.create = function(req, res, prepareConfigFn) {
-    var self = this;
+MapController.prototype.checkCreateLayergroup = function () {
+    return function checkCreateLayergroupMiddleware (req, res, next) {
+        if (req.method === 'GET') {
+            const { config } = res.locals;
 
-    var mapConfig;
+            if (!config) {
+                return next(new Error('layergroup GET needs a "config" parameter'));
+            }
 
-    var context = {};
+            try {
+                req.body = JSON.parse(config);
+            } catch (err) {
+                return next(err);
+            }
+        }
 
-    step(
-        function setupParams(){
-            self.req2params(req, this);
-        },
-        prepareConfigFn,
-        function prepareAdapterMapConfig(err, requestMapConfig) {
-            assert.ifError(err);
-            context.analysisConfiguration = {
-                user: req.context.user,
+        req.profiler.done('checkCreateLayergroup');
+        return next();
+    };
+};
+
+MapController.prototype.getTemplate = function () {
+    return function getTemplateMiddleware (req, res, next) {
+        const templateParams = req.body;
+        const { user } = res.locals;
+
+        const mapconfigProvider = new NamedMapMapConfigProvider(
+            this.templateMaps,
+            this.pgConnection,
+            this.metadataBackend,
+            this.userLimitsApi,
+            this.mapConfigAdapter,
+            user,
+            req.params.template_id,
+            templateParams,
+            res.locals.auth_token,
+            res.locals
+        );
+
+        mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => {
+            req.profiler.done('named.getMapConfig');
+            if (err) {
+                return next(err);
+            }
+
+            res.locals.mapconfig = mapconfig;
+            res.locals.rendererParams = rendererParams;
+            res.locals.mapconfigProvider = mapconfigProvider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+MapController.prototype.prepareAdapterMapConfig = function () {
+    return function prepareAdapterMapConfigMiddleware(req, res, next) {
+        const requestMapConfig = req.body;
+        const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals;
+
+        const context = {
+            analysisConfiguration: {
+                user,
                 db: {
-                    host: req.params.dbhost,
-                    port: req.params.dbport,
-                    dbname: req.params.dbname,
-                    user: req.params.dbuser,
-                    pass: req.params.dbpassword
+                    host: dbhost,
+                    port: dbport,
+                    dbname: dbname,
+                    user: dbuser,
+                    pass: dbpassword
                 },
                 batch: {
-                    username: req.context.user,
-                    apiKey: req.params.api_key
+                    username: user,
+                    apiKey: api_key
                 }
-            };
-            self.mapConfigAdapter.getMapConfig(req.context.user, requestMapConfig, req.params, context, this);
-        },
-        function createLayergroup(err, requestMapConfig) {
-            assert.ifError(err);
-            var datasource = context.datasource || Datasource.EmptyDatasource();
-            mapConfig = new MapConfig(requestMapConfig, datasource);
-            self.mapBackend.createLayergroup(
-                mapConfig, req.params,
-                new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params),
-                this
-            );
-        },
-        function afterLayergroupCreate(err, layergroup) {
-            assert.ifError(err);
-            self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this);
-        },
-        function finish(err, layergroup) {
-            if (err) {
-                self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
-            } else {
-                var analysesResults = context.analysesResults || [];
-                addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
-                addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
-                res.set('X-Layergroup-Id', layergroup.layergroupid);
-                self.send(req, res, layergroup, 200);
             }
-        }
-    );
+        };
+
+        this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => {
+            req.profiler.done('anonymous.getMapConfig');
+            if (err) {
+                return next(err);
+            }
+
+            req.body = requestMapConfig;
+            res.locals.context = context;
+
+            next();
+        });
+    }.bind(this);
 };
 
-MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
-    var self = this;
+MapController.prototype.createLayergroup = function () {
+    return function createLayergroupMiddleware (req, res, next) {
+        const requestMapConfig = req.body;
+        const { context, user } = res.locals;
+        const datasource = context.datasource || Datasource.EmptyDatasource();
+        const mapconfig = new MapConfig(requestMapConfig, datasource);
+        const mapconfigProvider =
+            new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals);
 
-    var cdbuser = req.context.user;
+        res.locals.mapconfig = mapconfig;
+        res.locals.analysesResults = context.analysesResults;
 
-    var mapConfigProvider;
-    var mapConfig;
-
-    step(
-        function setupParams(){
-            self.req2params(req, this);
-        },
-        function getTemplateParams() {
-            prepareParamsFn(this);
-        },
-        function getTemplate(err, templateParams) {
-            assert.ifError(err);
-            mapConfigProvider = new NamedMapMapConfigProvider(
-                self.templateMaps,
-                self.pgConnection,
-                self.metadataBackend,
-                self.userLimitsApi,
-                self.mapConfigAdapter,
-                cdbuser,
-                req.params.template_id,
-                templateParams,
-                req.query.auth_token,
-                req.params
-            );
-            mapConfigProvider.getMapConfig(this);
-        },
-        function createLayergroup(err, mapConfig_, rendererParams) {
-            assert.ifError(err);
-            mapConfig = mapConfig_;
-            self.mapBackend.createLayergroup(
-                mapConfig, rendererParams,
-                new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams),
-                this
-            );
-        },
-        function afterLayergroupCreate(err, layergroup) {
-            assert.ifError(err);
-            self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
-        },
-        function finishTemplateInstantiation(err, layergroup) {
+        this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
+            req.profiler.done('createLayergroup');
             if (err) {
-                self.sendError(req, res, err, 'NAMED MAP LAYERGROUP');
-            } else {
-                var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
-                layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
-
-                addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj());
-                addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
-
-                res.set('X-Layergroup-Id', layergroup.layergroupid);
-                self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
-
-                self.send(req, res, layergroup, 200);
+                return next(err);
             }
-        }
-    );
+
+            res.locals.layergroup = layergroup;
+
+            next();
+        });
+    }.bind(this);
 };
 
-MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
-    var self = this;
+MapController.prototype.instantiateLayergroup = function () {
+    return function instantiateLayergroupMiddleware (req, res, next) {
+        const { user, mapconfig, rendererParams } = res.locals;
+        const mapconfigProvider =
+            new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams);
 
-    var username = req.context.user;
+        this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
+            req.profiler.done('createLayergroup');
+            if (err) {
+                return next(err);
+            }
 
-    var tasksleft = 2; // redis key and affectedTables
-    var errors = [];
+            res.locals.layergroup = layergroup;
 
-    var done = function(err) {
-        if ( err ) {
-            errors.push('' + err);
-        }
-        if ( ! --tasksleft ) {
-            err = errors.length ? new Error(errors.join('\n')) : null;
-            callback(err, layergroup);
-        }
-    };
+            const { mapconfigProvider } = res.locals;
 
-    // include in layergroup response the variables in serverMedata
-    // those variables are useful to send to the client information
-    // about how to reach this server or information about it
-    _.extend(layergroup, global.environment.serverMetadata);
+            res.locals.analysesResults = mapconfigProvider.analysesResults;
+            res.locals.template = mapconfigProvider.template;
+            res.locals.templateName = mapconfigProvider.getTemplateName();
+            res.locals.context = mapconfigProvider.context;
 
-    // Don't wait for the mapview count increment to
-    // take place before proceeding. Error will be logged
-    // asynchronously
-    this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) {
-        if (req.profiler) {
+            next();
+        });
+    }.bind(this);
+};
+
+MapController.prototype.incrementMapViewCount = function () {
+    return function incrementMapViewCountMiddleware(req, res, next) {
+        const { mapconfig, user } = res.locals;
+
+        // Error won't blow up, just be logged.
+        this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
             req.profiler.done('incMapviewCount');
-        }
-        if ( err ) {
-            global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
-        }
-        done();
-    });
 
-    var sql = mapconfig.getLayers().map(function(layer) {
-        return layer.options.sql;
-    }).join(';');
-
-    var dbName = req.params.dbname;
-    var layergroupId = layergroup.layergroupid;
-
-    step(
-        function getPgConnection() {
-            self.pgConnection.getConnection(username, this);
-        },
-        function getAffectedTablesAndLastUpdatedTime(err, connection) {
-            assert.ifError(err);
-            QueryTables.getAffectedTablesFromQuery(connection, sql, this);
-        },
-        function handleAffectedTablesAndLastUpdatedTime(err, result) {
-            if (req.profiler) {
-                req.profiler.done('queryTablesAndLastUpdated');
+            if (err) {
+                global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`);
             }
-            assert.ifError(err);
-            // feed affected tables cache so it can be reused from, for instance, layergroup controller
-            self.layergroupAffectedTables.set(dbName, layergroupId, result);
 
-            var lastUpdateTime = result.getLastUpdatedAt();
-            lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
+            next();
+        });
+    }.bind(this);
+};
 
-            // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
-            layergroup.last_updated = new Date(lastUpdateTime).toISOString();
+MapController.prototype.augmentLayergroupData = function () {
+    return function augmentLayergroupDataMiddleware (req, res, next) {
+        const { layergroup } = res.locals;
 
-            if (req.method === 'GET') {
-                var ttl = global.environment.varnish.layergroupTtl || 86400;
-                res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
-                res.set('Last-Modified', (new Date()).toUTCString());
-                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.tables && result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, result);
+        // include in layergroup response the variables in serverMedata
+        // those variables are useful to send to the client information
+        // about how to reach this server or information about it
+        _.extend(layergroup, global.environment.serverMetadata);
+
+        next();
+    };
+};
+
+MapController.prototype.getAffectedTables = function () {
+    return function getAffectedTablesMiddleware (req, res, next) {
+        const { dbname, layergroup, user, mapconfig } = res.locals;
+
+        this.pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                return next(err);
+            }
+
+            const sql = [];
+            mapconfig.getLayers().forEach(function(layer) {
+                sql.push(layer.options.sql);
+                if (layer.options.affected_tables) {
+                    layer.options.affected_tables.map(function(table) {
+                        sql.push('SELECT * FROM ' + table + ' LIMIT 0');
+                    });
                 }
-            }
+            });
 
-            return null;
-        },
-        function finish(err) {
-            done(err);
+            QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => {
+                req.profiler.done('getAffectedTablesFromQuery');
+                if (err) {
+                    return next(err);
+                }
+
+                // feed affected tables cache so it can be reused from, for instance, layergroup controller
+                this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
+
+                res.locals.affectedTables = affectedTables;
+
+                next();
+            });
+        });
+    }.bind(this);
+};
+
+MapController.prototype.setCacheChannel = function () {
+    return function setCacheChannelMiddleware (req, res, next) {
+        const { affectedTables } = res.locals;
+
+        if (req.method === 'GET') {
+            res.set('X-Cache-Channel', affectedTables.getCacheChannel());
         }
-    );
+
+        next();
+    };
+};
+
+MapController.prototype.setLastModified = function () {
+    return function setLastModifiedMiddleware (req, res, next) {
+        if (req.method === 'GET') {
+            res.set('Last-Modified', (new Date()).toUTCString());
+        }
+
+        next();
+    };
+};
+
+MapController.prototype.setLastUpdatedTimeToLayergroup = function () {
+    return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
+        const { affectedTables, layergroup, analysesResults } = res.locals;
+
+        var lastUpdateTime = affectedTables.getLastUpdatedAt();
+
+        lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
+
+        // last update for layergroup cache buster
+        layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
+        layergroup.last_updated = new Date(lastUpdateTime).toISOString();
+
+        next();
+    };
 };
 
 function getLastUpdatedTime(analysesResults, lastUpdateTime) {
@@ -338,7 +397,116 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
     }, lastUpdateTime);
 }
 
-function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) {
+MapController.prototype.setCacheControl = function () {
+    return function setCacheControlMiddleware (req, res, next) {
+        if (req.method === 'GET') {
+            var ttl = global.environment.varnish.layergroupTtl || 86400;
+            res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
+        }
+
+        next();
+    };
+};
+
+MapController.prototype.setLayerStats = function () {
+    return function setLayerStatsMiddleware(req, res, next) {
+        const { user, mapconfig, layergroup } = res.locals;
+
+        this.pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                return next(err);
+            }
+
+            this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layersStats.length > 0) {
+                    layergroup.metadata.layers.forEach(function (layer, index) {
+                        layer.meta.stats = layersStats[index];
+                    });
+                }
+
+                next();
+            });
+        });
+    }.bind(this);
+};
+
+MapController.prototype.setLayergroupIdHeader = function (useTemplateHash) {
+    return function setLayergroupIdHeaderMiddleware (req, res, next) {
+        const { layergroup, user, template } = res.locals;
+
+        if (useTemplateHash) {
+            var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8);
+            layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
+        }
+
+        res.set('X-Layergroup-Id', layergroup.layergroupid);
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () {
+    return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
+        const { layergroup, user, mapconfig } = res.locals;
+
+        this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
+
+        next();
+    }.bind(this);
+};
+
+// TODO this should take into account several URL patterns
+MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) {
+    this.addDataviewsUrls(username, layergroup, mapConfig);
+    this.addWidgetsUrl(username, layergroup, mapConfig);
+};
+
+MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) {
+    layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
+    var dataviews = mapConfig.dataviews || {};
+
+    Object.keys(dataviews).forEach(function(dataviewName) {
+        var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
+        layergroup.metadata.dataviews[dataviewName] = {
+            url: this.resourceLocator.getUrls(username, resource)
+        };
+    }.bind(this));
+};
+
+MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            var mapConfigLayer = mapConfig.layers[layerIndex];
+            if (mapConfigLayer.options && mapConfigLayer.options.widgets) {
+                layer.widgets = layer.widgets || {};
+                Object.keys(mapConfigLayer.options.widgets).forEach(function(widgetName) {
+                    var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
+                    layer.widgets[widgetName] = {
+                        type: mapConfigLayer.options.widgets[widgetName].type,
+                        url: this.resourceLocator.getUrls(username, resource)
+                    };
+                }.bind(this));
+            }
+            return layer;
+        }.bind(this));
+    }
+};
+
+MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) {
+    return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) {
+        const { layergroup, user, analysesResults = [] } = res.locals;
+
+        this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
     includeQuery = includeQuery || false;
     analysesResults = analysesResults || [];
     layergroup.metadata.analyses = [];
@@ -351,7 +519,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                     var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
                     var nodeRepr = {
                         status: node.getStatus(),
-                        url: getUrls(username, nodeResource)
+                        url: this.resourceLocator.getUrls(username, nodeResource)
                     };
                     if (includeQuery) {
                         nodeRepr.query = node.getQuery();
@@ -363,59 +531,95 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                 }
 
                 return nodesIdMap;
-            }, {})
+            }.bind(this), {})
         });
-    });
-}
+    }.bind(this));
+};
 
-// TODO this should take into account several URL patterns
-function addDataviewsAndWidgetsUrls(username, layergroup, mapConfig) {
-    addDataviewsUrls(username, layergroup, mapConfig);
-    addWidgetsUrl(username, layergroup, mapConfig);
-}
+MapController.prototype.setTurboCartoMetadataToLayergroup = function () {
+    return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
+        const { layergroup, mapconfig, context } = res.locals;
 
-function addDataviewsUrls(username, layergroup, mapConfig) {
-    layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
-    var dataviews = mapConfig.dataviews || {};
+        addContextMetadata(layergroup, mapconfig.obj(), context);
 
-    Object.keys(dataviews).forEach(function(dataviewName) {
-        var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
-        layergroup.metadata.dataviews[dataviewName] = {
-            url: getUrls(username, resource)
-        };
-    });
-}
+        next();
+    };
+};
 
-function addWidgetsUrl(username, layergroup, mapConfig) {
+function addContextMetadata(layergroup, mapConfig, context) {
     if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
         layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
-            var mapConfigLayer = mapConfig.layers[layerIndex];
-            if (mapConfigLayer.options && mapConfigLayer.options.widgets) {
-                layer.widgets = layer.widgets || {};
-                Object.keys(mapConfigLayer.options.widgets).forEach(function(widgetName) {
-                    var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
-                    layer.widgets[widgetName] = {
-                        type: mapConfigLayer.options.widgets[widgetName].type,
-                        url: getUrls(username, resource)
-                    };
-                });
+            if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
+                layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
             }
             return layer;
         });
     }
 }
 
-function getUrls(username, resource) {
-    var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
-    if (cdnUrl) {
-        return {
-            http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource,
-            https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource
-        };
-    } else {
-        var port = global.environment.port;
-        return {
-            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
-        };
+MapController.prototype.setSurrogateKeyHeader = function () {
+    return function setSurrogateKeyHeaderMiddleware(req, res, next) {
+        const { affectedTables, user, templateName } = res.locals;
+
+        if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) {
+            this.surrogateKeysCache.tag(res, affectedTables);
+        }
+
+        if (templateName) {
+            this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
+        }
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.sendResponse = function () {
+    return function sendResponseMiddleware (req, res) {
+        req.profiler.done('res');
+        const { layergroup } = res.locals;
+
+        res.status(200);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(layergroup);
+        } else {
+            res.json(layergroup);
+        }
+    };
+};
+
+MapController.prototype.augmentError = function (options) {
+    const { addContext = false, label = 'MAPS CONTROLLER' } = options;
+
+    return function augmentErrorMiddleware (err, req, res, next) {
+        req.profiler.done('error');
+        const { mapconfig } = res.locals;
+
+        if (addContext) {
+            err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err;
+        }
+
+        err.label = label;
+
+        next(err);
+    };
+};
+
+function populateError(err, mapConfig) {
+    var error = new Error(err.message);
+    error.http_status = err.http_status;
+
+    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
+        error.http_status = 400;
     }
+
+    error.type = 'layer';
+    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
+    error.layer = {
+        id: mapConfig.getLayerId(err.layerIndex),
+        index: err.layerIndex,
+        type: mapConfig.layerType(err.layerIndex)
+    };
+
+    return error;
 }

lib/cartodb/controllers/named_maps.js

@@ -3,40 +3,46 @@ var assert = require('assert');
 var _ = require('underscore');
 var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
 
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
+var vectorError = require('../middleware/vector-error');
 
-function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
+function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
-    BaseController.call(this, authApi, pgConnection);
-
     this.namedMapProviderCache = namedMapProviderCache;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
     this.surrogateKeysCache = surrogateKeysCache;
     this.tablesExtentApi = tablesExtentApi;
     this.metadataBackend = metadataBackend;
+    this.prepareContext = prepareContext;
 }
 
-util.inherits(NamedMapsController, BaseController);
-
 module.exports = NamedMapsController;
 
 NamedMapsController.prototype.register = function(app) {
-    app.get(app.base_url_templated +
-        '/:template_id/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()        
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
-        this.staticMap.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
+        this.prepareContext,
+        this.staticMap.bind(this)
+    );
 };
 
-NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
-    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
+NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) {
+    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName()));
     res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
     res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
 
@@ -68,29 +74,26 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
                     self.surrogateKeysCache.tag(res, result);
                 }
             }
-            self.send(req, res, resource, 200);
+            res.status(200);
+            res.send(body);
         }
     );
 };
 
-NamedMapsController.prototype.tile = function(req, res) {
+NamedMapsController.prototype.tile = function(req, res, next) {
     var self = this;
 
-    var cdbUser = req.context.user;
+    var cdbUser = res.locals.user;
 
     var namedMapProvider;
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
+        function getNamedMapProvider() {
             self.namedMapProviderCache.get(
                 cdbUser,
                 req.params.template_id,
                 req.query.config,
                 req.query.auth_token,
-                req.params,
+                res.locals,
                 this
             );
         },
@@ -100,11 +103,11 @@ NamedMapsController.prototype.tile = function(req, res) {
             self.tileBackend.getTile(namedMapProvider, req.params, this);
         },
         function handleImage(err, tile, headers, stats) {
-            if (req.profiler) {
-                req.profiler.add(stats);
-            }
+            req.profiler.add(stats);
+
             if (err) {
-                self.sendError(req, res, err, 'NAMED_MAP_TILE');
+                err.label = 'NAMED_MAP_TILE';
+                next(err);
             } else {
                 self.sendResponse(req, res, tile, headers, namedMapProvider);
             }
@@ -112,35 +115,37 @@ NamedMapsController.prototype.tile = function(req, res) {
     );
 };
 
-NamedMapsController.prototype.staticMap = function(req, res) {
+NamedMapsController.prototype.staticMap = function(req, res, next) {
     var self = this;
 
-    var cdbUser = req.context.user;
+    var cdbUser = res.locals.user;
 
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.format = 'png';
-    req.params.layer = 'all';
+    res.locals.format = req.params.format || 'png';
+    res.locals.layer = res.locals.layer || 'all';
 
     var namedMapProvider;
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
+        function getNamedMapProvider() {
             self.namedMapProviderCache.get(
                 cdbUser,
                 req.params.template_id,
                 req.query.config,
                 req.query.auth_token,
-                req.params,
+                res.locals,
                 this
             );
         },
-        function prepareImageOptions(err, _namedMapProvider) {
+        function prepareLayerVisibility(err, _namedMapProvider) {
             assert.ifError(err);
+
             namedMapProvider = _namedMapProvider;
-            self.getStaticImageOptions(cdbUser, req.params, namedMapProvider, this);
+
+            self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this);
+        },
+        function prepareImageOptions(err) {
+            assert.ifError(err);
+            self.getStaticImageOptions(cdbUser, res.locals, namedMapProvider, this);
         },
         function getImage(err, imageOpts) {
             assert.ifError(err);
@@ -170,13 +175,12 @@ NamedMapsController.prototype.staticMap = function(req, res) {
             });
         },
         function handleImage(err, image, headers, stats) {
-            if (req.profiler) {
-                req.profiler.done('render-' + format);
-                req.profiler.add(stats || {});
-            }
+            req.profiler.done('render-' + format);
+            req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'STATIC_VIZ_MAP');
+                err.label = 'STATIC_VIZ_MAP';
+                next(err);
             } else {
                 self.sendResponse(req, res, image, headers, namedMapProvider);
             }
@@ -184,6 +188,51 @@ NamedMapsController.prototype.staticMap = function(req, res) {
     );
 };
 
+NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (
+    user,
+    req,
+    params,
+    namedMapProvider,
+    callback
+) {
+    var self = this;
+    namedMapProvider.getTemplate(function (err, template) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!template || !template.view || !template.view.preview_layers) {
+           return callback();
+        }
+
+        var previewLayers = template.view.preview_layers;
+        var layerVisibilityFilter = [];
+
+        template.layergroup.layers.forEach(function (layer, index) {
+            if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
+                layerVisibilityFilter.push(''+index);
+            }
+        });
+
+        if (!layerVisibilityFilter.length) {
+            return callback();
+        }
+
+        // overwrites 'all' default filter
+        params.layer = layerVisibilityFilter.join(',');
+
+        // recreates the provider
+        self.namedMapProviderCache.get(
+            user,
+            req.params.template_id,
+            req.query.config,
+            req.query.auth_token,
+            params,
+            callback
+        );
+    });
+};
+
 var DEFAULT_ZOOM_CENTER = {
     zoom: 1,
     center: {

lib/cartodb/controllers/named_maps_admin.js

@@ -2,9 +2,6 @@ var step = require('step');
 var assert = require('assert');
 var templateName = require('../backends/template_maps').templateName;
 
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
@@ -15,30 +12,59 @@ var userMiddleware = require('../middleware/user');
  * @param {TemplateMaps} templateMaps
  * @constructor
  */
-function NamedMapsAdminController(authApi, pgConnection, templateMaps) {
-    BaseController.call(this, authApi, pgConnection);
-
+function NamedMapsAdminController(authApi, templateMaps) {
     this.authApi = authApi;
     this.templateMaps = templateMaps;
 }
 
-util.inherits(NamedMapsAdminController, BaseController);
-
 module.exports = NamedMapsAdminController;
 
-NamedMapsAdminController.prototype.register = function(app) {
-    app.post(app.base_url_templated, cors(), userMiddleware, this.create.bind(this));
-    app.put(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.update.bind(this));
-    app.get(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.retrieve.bind(this));
-    app.delete(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.destroy.bind(this));
-    app.get(app.base_url_templated, cors(), userMiddleware, this.list.bind(this));
-    app.options(app.base_url_templated + '/:template_id', cors('Content-Type'));
+NamedMapsAdminController.prototype.register = function (app) {
+    app.post(
+        app.base_url_templated + '/',
+        cors(),
+        userMiddleware,
+        this.create.bind(this)
+    );
+
+    app.put(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.update.bind(this)
+    );
+
+    app.get(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.retrieve.bind(this)
+    );
+
+    app.delete(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.destroy.bind(this)
+    );
+
+    app.get(
+        app.base_url_templated + '/',
+        cors(),
+        userMiddleware,
+        this.list.bind(this)
+    );
+
+    app.options(
+        app.base_url_templated + '/:template_id',
+        cors('Content-Type')
+    );
 };
 
-NamedMapsAdminController.prototype.create = function(req, res) {
+NamedMapsAdminController.prototype.create = function(req, res, next) {
     var self = this;
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
 
     step(
         function checkPerms(){
@@ -55,14 +81,14 @@ NamedMapsAdminController.prototype.create = function(req, res) {
             assert.ifError(err);
             return { template_id: tpl_id };
         },
-        finishFn(self, req, res, 'POST TEMPLATE')
+        finishFn(self, req, res, 'POST TEMPLATE', null, next)
     );
 };
 
-NamedMapsAdminController.prototype.update = function(req, res) {
+NamedMapsAdminController.prototype.update = function(req, res, next) {
     var self = this;
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
     var template;
     var tpl_id;
 
@@ -84,18 +110,16 @@ NamedMapsAdminController.prototype.update = function(req, res) {
 
             return { template_id: tpl_id };
         },
-        finishFn(self, req, res, 'PUT TEMPLATE')
+        finishFn(self, req, res, 'PUT TEMPLATE', null, next)
     );
 };
 
-NamedMapsAdminController.prototype.retrieve = function(req, res) {
+NamedMapsAdminController.prototype.retrieve = function(req, res, next) {
     var self = this;
 
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.get_template');
-    }
+    req.profiler.start('windshaft-cartodb.get_template');
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
     var tpl_id;
     step(
         function checkPerms(){
@@ -120,18 +144,16 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
             delete tpl_val.auth_id;
             return { template: tpl_val };
         },
-        finishFn(self, req, res, 'GET TEMPLATE')
+        finishFn(self, req, res, 'GET TEMPLATE', null, next)
     );
 };
 
-NamedMapsAdminController.prototype.destroy = function(req, res) {
+NamedMapsAdminController.prototype.destroy = function(req, res, next) {
     var self = this;
 
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.delete_template');
-    }
+    req.profiler.start('windshaft-cartodb.delete_template');
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
     var tpl_id;
     step(
         function checkPerms(){
@@ -148,17 +170,15 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
             assert.ifError(err);
             return '';
         },
-        finishFn(self, req, res, 'DELETE TEMPLATE', 204)
+        finishFn(self, req, res, 'DELETE TEMPLATE', 204, next)
     );
 };
 
-NamedMapsAdminController.prototype.list = function(req, res) {
+NamedMapsAdminController.prototype.list = function(req, res, next) {
     var self = this;
-    if ( req.profiler ) {
-        req.profiler.start('windshaft-cartodb.get_template_list');
-    }
+    req.profiler.start('windshaft-cartodb.get_template_list');
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
 
     step(
         function checkPerms(){
@@ -174,16 +194,23 @@ NamedMapsAdminController.prototype.list = function(req, res) {
             assert.ifError(err);
             return { template_ids: tpl_ids };
         },
-        finishFn(self, req, res, 'GET TEMPLATE LIST')
+        finishFn(self, req, res, 'GET TEMPLATE LIST', null, next)
     );
 };
 
-function finishFn(controller, req, res, description, status) {
-    return function finish(err, response){
+function finishFn(controller, req, res, description, status, next) {
+    return function finish(err, body){
         if (err) {
-            controller.sendError(req, res, err, description);
+            err.label = description;
+            next(err);
         } else {
-            controller.send(req, res, response, status || 200);
+            res.status(status || 200);
+
+            if (req.query && req.query.callback) {
+                res.jsonp(body);
+            } else {
+                res.json(body);
+            }
         }
     };
 }

lib/cartodb/middleware/allow-query-params.js

@@ -0,0 +1,9 @@
+module.exports = function allowQueryParams(params) {
+    if (!Array.isArray(params)) {
+        throw new Error('allowQueryParams must receive an Array of params');
+    }
+    return function allowQueryParamsMiddleware(req, res, next) {
+        res.locals.allowedQueryParams = params;
+        next();
+    };
+};

lib/cartodb/middleware/context/authorize.js

@@ -0,0 +1,20 @@
+module.exports = function authorizeMiddleware (authApi) {
+    return function (req, res, next) {
+        req.profiler.done('req2params.setup');
+
+        authApi.authorize(req, res, (err, authorized) => {
+            req.profiler.done('authorize');
+            if (err) {
+                return next(err);
+            }
+
+            if(!authorized) {
+                err = new Error("Sorry, you are unauthorized (permission denied)");
+                err.http_status = 403;
+                return next(err);
+            }
+
+            return next();
+        });
+    };
+};

lib/cartodb/middleware/context/clean-up-query-params.js

@@ -0,0 +1,32 @@
+const _ = require('underscore');
+
+// Whitelist query parameters and attach format
+const REQUEST_QUERY_PARAMS_WHITELIST = [
+    'config',
+    'map_key',
+    'api_key',
+    'auth_token',
+    'callback',
+    'zoom',
+    'lon',
+    'lat',
+    // analysis
+    'filters' // json
+];
+
+module.exports = function cleanUpQueryParamsMiddleware () {
+    return function cleanUpQueryParams (req, res, next) {
+        var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
+
+        if (Array.isArray(res.locals.allowedQueryParams)) {
+            allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams);
+        }
+
+        req.query = _.pick(req.query, allowedQueryParams);
+
+        // bring all query values onto res.locals object
+        _.extend(res.locals, req.query);
+        
+        next();
+    };
+};

lib/cartodb/middleware/context/db-conn-setup.js

@@ -0,0 +1,31 @@
+const _ = require('underscore');
+
+module.exports = function dbConnSetupMiddleware(pgConnection) {
+    return function dbConnSetup(req, res, next) {
+        const user = res.locals.user;
+        pgConnection.setDBConn(user, res.locals, (err) => {
+            if (err) {
+                if (err.message && -1 !== err.message.indexOf('name not found')) {
+                    err.http_status = 404;
+                }
+                req.profiler.done('req2params');
+                return next(err);
+            }
+
+            // Add default database connection parameters
+            // if none given
+            _.defaults(res.locals, {
+                dbuser: global.environment.postgres.user,
+                dbpassword: global.environment.postgres.password,
+                dbhost: global.environment.postgres.host,
+                dbport: global.environment.postgres.port
+            });
+            
+            res.set('X-Served-By-DB-Host', res.locals.dbhost);
+
+            req.profiler.done('req2params');
+            
+            next(null);
+        });
+    };
+};

lib/cartodb/middleware/context/index.js

@@ -0,0 +1,15 @@
+const locals = require('./locals');
+const cleanUpQueryParams = require('./clean-up-query-params');
+const layergroupToken = require('./layergroup-token');
+const authorize = require('./authorize');
+const dbConnSetup = require('./db-conn-setup');
+
+module.exports = function prepareContextMiddleware(authApi, pgConnection) {
+    return [
+        locals,
+        cleanUpQueryParams(),
+        layergroupToken,
+        authorize(authApi),
+        dbConnSetup(pgConnection)
+    ];
+};

lib/cartodb/middleware/context/layergroup-token.js

@@ -0,0 +1,32 @@
+var LayergroupToken = require('../../models/layergroup-token');
+
+module.exports = function layergroupTokenMiddleware(req, res, next) {
+    if (!res.locals.token) {
+        return next();
+    }
+
+    var user = res.locals.user;
+
+    var layergroupToken = LayergroupToken.parse(res.locals.token);
+    res.locals.token = layergroupToken.token;
+    res.locals.cache_buster = layergroupToken.cacheBuster;
+
+    if (layergroupToken.signer) {
+        res.locals.signer = layergroupToken.signer;
+        if (!res.locals.signer) {
+            res.locals.signer = user;
+        } else if (res.locals.signer !== user) {
+            var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`);
+            err.type = 'auth';
+            err.http_status = 403;
+            if (req.query && req.query.callback) {
+                err.http_status = 200;
+            }
+            
+            req.profiler.done('req2params');
+            return next(err);
+        }
+    }
+
+    return next();
+};

lib/cartodb/middleware/context/locals.js

@@ -0,0 +1,6 @@
+module.exports = function localsMiddleware(req, res, next) {
+    // save req.params in res.locals
+    res.locals = Object.assign(req.params || {}, res.locals);
+
+    next();
+};

lib/cartodb/middleware/error-middleware.js

@@ -0,0 +1,162 @@
+const _ = require('underscore');
+const debug = require('debug')('windshaft:cartodb:error-middleware');
+
+module.exports = function errorMiddleware (/* options */) {
+    return function error (err, req, res, next) {
+        // jshint unused:false
+        // jshint maxcomplexity:9
+        var allErrors = Array.isArray(err) ? err : [err];
+
+        allErrors = populateTimeoutErrors(allErrors);
+
+        const label = err.label || 'UNKNOWN';
+        err = allErrors[0] || new Error(label);
+        allErrors[0] = err;
+
+        var statusCode = findStatusCode(err);
+
+        if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') {
+            statusCode = 204;
+        }
+
+        debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
+
+        // If a callback was requested, force status to 200
+        if (req.query && req.query.callback) {
+            statusCode = 200;
+        }
+
+        var errorResponseBody = {
+            errors: allErrors.map(errorMessage),
+            errors_with_context: allErrors.map(errorMessageWithContext)
+        };
+
+        res.status(statusCode);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(errorResponseBody);
+        } else {
+            res.json(errorResponseBody);
+        }
+    };
+};
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}
+
+function populateTimeoutErrors (errors) {
+    return errors.map(function (error) {
+        if (isRenderTimeoutError(error)) {
+            error.subtype = 'render';
+        }
+
+        if (isDatasourceTimeoutError(error)) {
+            error.subtype = 'datasource';
+        }
+
+        if (isTimeoutError(error)) {
+            error.message = 'You are over platform\'s limits. Please contact us to know more details';
+            error.type = 'limit';
+            error.http_status = 429;
+        }
+
+        return error;
+    });
+}
+
+function findStatusCode(err) {
+    var statusCode;
+    if ( err.http_status ) {
+        statusCode = err.http_status;
+    } else {
+        statusCode = statusFromErrorMessage('' + err);
+    }
+    return statusCode;
+}
+
+module.exports.findStatusCode = findStatusCode;
+
+function statusFromErrorMessage(errMsg) {
+    // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
+    var statusCode = 400;
+    if ( -1 !== errMsg.indexOf('permission denied') ) {
+        statusCode = 403;
+    }
+    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
+        statusCode = 403;
+    }
+    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
+        statusCode = 400;
+    }
+    else if ( -1 !== errMsg.indexOf('does not exist') ) {
+        if ( -1 !== errMsg.indexOf(' role ') ) {
+            statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
+        } else {
+            statusCode = 404;
+        }
+    }
+
+
+    return statusCode;
+}
+
+function errorMessage(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    return stripConnectionInfo(message);
+}
+
+module.exports.errorMessage = errorMessage;
+
+function stripConnectionInfo(message) {
+    // Strip connection info, if any
+    return message
+        // See https://github.com/CartoDB/Windshaft/issues/173
+        .replace(/Connection string: '[^']*'\n\s/im, '')
+        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
+        .replace(/is the server.*encountered/im, 'encountered');
+}
+
+var ERROR_INFO_TO_EXPOSE = {
+    message: true,
+    layer: true,
+    type: true,
+    analysis: true,
+    subtype: true
+};
+
+function shouldBeExposed (prop) {
+    return !!ERROR_INFO_TO_EXPOSE[prop];
+}
+
+function errorMessageWithContext(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    var error = {
+        type: err.type || 'unknown',
+        message: stripConnectionInfo(message),
+    };
+
+    for (var prop in err) {
+        // type & message are properties from Error's prototype and will be skipped
+        if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
+            error[prop] = err[prop];
+        }
+    }
+
+    return error;
+}

lib/cartodb/middleware/lzma.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const LZMA = require('lzma').LZMA;
+
+const lzmaWorker = new LZMA();
+
+module.exports = function lzmaMiddleware(req, res, next) {
+    if (!req.query.hasOwnProperty('lzma')) {
+        return next();
+    }
+
+    // Decode (from base64)
+    var lzma = new Buffer(req.query.lzma, 'base64')
+        .toString('binary')
+        .split('')
+        .map(function(c) {
+            return c.charCodeAt(0) - 128;
+        });
+
+    // Decompress
+    lzmaWorker.decompress(lzma, function(result) {
+        try {
+            delete req.query.lzma;
+            Object.assign(req.query, JSON.parse(result));
+            next();
+        } catch (err) {
+            next(new Error('Error parsing lzma as JSON: ' + err));
+        }
+    });
+};

lib/cartodb/middleware/stats.js

@@ -0,0 +1,27 @@
+const Profiler = require('../stats/profiler_proxy');
+const debug = require('debug')('windshaft:cartodb:stats');
+const onHeaders = require('on-headers');
+
+module.exports = function statsMiddleware(options) {
+    const { enabled = true, statsClient } = options;
+
+    return function stats(req, res, next) {
+        req.profiler = new Profiler({
+            statsd_client: statsClient,
+            profile: enabled
+        });
+
+        onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString()));
+
+        res.on('finish', () => {
+            try {
+                // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166
+                req.profiler.sendStats();
+            } catch (err) {
+                debug("error sending profiling stats: " + err);
+            }
+        });
+
+        next();
+    };
+};

lib/cartodb/middleware/user.js

@@ -2,6 +2,7 @@ var CdbRequest = require('../models/cdb_request');
 var cdbRequest = new CdbRequest();
 
 module.exports = function userMiddleware(req, res, next) {
-    req.context.user = cdbRequest.userByReq(req);
+    res.locals.user = cdbRequest.userByReq(req);
+
     next();
 };

lib/cartodb/middleware/vector-error.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+
+const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
+
+module.exports = function vectorError() {
+    return function vectorErrorMiddleware(err, req, res, next) {
+        if(req.params.format === 'mvt') {
+
+            if (isTimeoutError(err)) {
+                res.set('Content-Type', 'application/x-protobuf');
+                return res.status(429).send(timeoutErrorVectorTile);
+            }
+        }
+
+        next(err);
+    };
+};
+
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}

lib/cartodb/models/dataview/aggregation.js

@@ -1,58 +1,178 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:aggregation');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:aggregation');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const filteredQueryTpl = ctx => `
+    filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) _cdb_filtered_source
+        ${ctx.aggregationColumn && ctx.isFloatColumn ? `
+        WHERE
+            ${ctx.aggregationColumn} != 'infinity'::float
+        AND
+            ${ctx.aggregationColumn} != '-infinity'::float
+        AND
+            ${ctx.aggregationColumn} != 'NaN'::float` :
+        ''
+        }
+    )
+`;
 
-var summaryQueryTpl = dot.template([
-    'summary AS (',
-    '  SELECT',
-    '  count(1) AS count,',
-    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
-    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
-    ')'
-].join('\n'));
+const summaryQueryTpl = ctx => `
+    summary AS (
+        SELECT
+            count(1) AS count,
+            sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count
+            ${ctx.isFloatColumn ? `,
+            sum(
+                CASE
+                    WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float
+                    THEN 1
+                    ELSE 0
+                    END
+            ) AS infinities_count,
+            sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` :
+            ''
+            }
+        FROM (${ctx.query}) _cdb_aggregation_nulls
+    )
+`;
 
-var rankedCategoriesQueryTpl = dot.template([
-    'categories AS(',
-    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
-    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
-    '  FROM ({{=it._query}}) _cdb_aggregation_all',
-    '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
-    ')'
-].join('\n'));
+const rankedCategoriesQueryTpl = ctx => `
+    categories AS(
+        SELECT
+            ${ctx.column} AS category,
+            ${ctx.aggregationFn} AS value,
+            row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
+        FROM filtered_source
+        ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
+        GROUP BY ${ctx.column}
+        ORDER BY 2 DESC
+    )
+`;
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
-    '  FROM categories',
-    ')'
-].join('\n'));
+const categoriesSummaryMinMaxQueryTpl = () => `
+    categories_summary_min_max AS(
+        SELECT
+            max(value) max_val,
+            min(value) min_val
+        FROM categories
+    )
+`;
 
-var rankedAggregationQueryTpl = dot.template([
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
-    '  WHERE rank < {{=it._limit}}',
-    'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
-    '  WHERE rank >= {{=it._limit}}',
-    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
-].join('\n'));
+const categoriesSummaryCountQueryTpl = ctx => `
+    categories_summary_count AS(
+        SELECT count(1) AS categories_count
+        FROM (
+            SELECT ${ctx.column} AS category
+            FROM (${ctx.query}) _cdb_categories
+            GROUP BY ${ctx.column}
+        ) _cdb_categories_count
+    )
+`;
 
-var aggregationQueryTpl = dot.template([
-    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
-    '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
-    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
-    'ORDER BY value DESC'
-].join('\n'));
+const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
 
-var CATEGORIES_LIMIT = 6;
+const rankedAggregationQueryTpl = ctx => `
+    SELECT
+        CAST(category AS text),
+        value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank < ${ctx.limit}
+    UNION ALL
+    SELECT
+        'Other' category,
+        ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
+        true as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank >= ${ctx.limit}
+    GROUP BY
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+`;
 
-var VALID_OPERATIONS = {
+const aggregationQueryTpl = ctx => `
+    SELECT
+        CAST(${ctx.column} AS text) AS category,
+        ${ctx.aggregationFn} AS value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
+    GROUP BY
+        category,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    ORDER BY value DESC
+`;
+
+const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`;
+
+const aggregationDataviewQueryTpl = ctx => `
+    WITH
+    ${filteredQueryTpl(ctx)},
+    ${summaryQueryTpl(ctx)},
+    ${rankedCategoriesQueryTpl(ctx)},
+    ${categoriesSummaryMinMaxQueryTpl(ctx)},
+    ${categoriesSummaryCountQueryTpl(ctx)}
+    ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`}
+`;
+
+const filterCategoriesQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.value} AS value
+    FROM (${ctx.query}) _cdb_aggregation_search
+    WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery}
+    GROUP BY ${ctx.column}
+`;
+
+const searchQueryTpl = ctx => `
+    WITH
+    search_unfiltered AS (
+        ${ctx.searchUnfiltered}
+    ),
+    search_filtered AS (
+        ${ctx.searchFiltered}
+    ),
+    search_union AS (
+        SELECT * FROM search_unfiltered
+        UNION ALL
+        SELECT * FROM search_filtered
+    )
+    SELECT category, sum(value) AS value
+    FROM search_union
+    GROUP BY category
+    ORDER BY value desc
+`;
+
+const CATEGORIES_LIMIT = 6;
+
+const VALID_OPERATIONS = {
     count: [],
     sum: ['aggregationColumn'],
     avg: ['aggregationColumn'],
@@ -60,7 +180,7 @@ var VALID_OPERATIONS = {
     max: ['aggregationColumn']
 };
 
-var TYPE = 'aggregation';
+const TYPE = 'aggregation';
 
 /**
  {
@@ -71,212 +191,154 @@ var TYPE = 'aggregation';
      }
  }
  */
-function Aggregation(query, options) {
-    if (!_.isString(options.column)) {
-        throw new Error('Aggregation expects `column` in widget options');
+module.exports = class Aggregation extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.aggregation = options.aggregation;
+        this.aggregationColumn = options.aggregationColumn;
+        this._isFloatColumn = null;
     }
 
-    if (!_.isString(options.aggregation)) {
-        throw new Error('Aggregation expects `aggregation` operation in widget options');
-    }
-
-    if (!VALID_OPERATIONS[options.aggregation]) {
-        throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
-    }
-
-    var requiredOptions = VALID_OPERATIONS[options.aggregation];
-    var missingOptions = _.difference(requiredOptions, Object.keys(options));
-    if (missingOptions.length > 0) {
-        throw new Error(
-            "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
-        );
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.column = options.column;
-    this.aggregation = options.aggregation;
-    this.aggregationColumn = options.aggregationColumn;
-}
-
-Aggregation.prototype = new BaseWidget();
-Aggregation.prototype.constructor = Aggregation;
-
-module.exports = Aggregation;
-
-Aggregation.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    var _query = this.query;
-
-    var aggregationSql;
-    if (!!override.ownFilter) {
-        aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql()
-                }),
-                categoriesSummaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
-            aggregationQueryTpl({
-                _query: _query,
-                _column: this.column,
-                _aggregation: this.getAggregationSql(),
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    } else {
-        aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql()
-                }),
-                categoriesSummaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
-            rankedAggregationQueryTpl({
-                _query: _query,
-                _column: this.column,
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    }
-
-    debug(aggregationSql);
-
-    return callback(null, aggregationSql);
-};
-
-var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
-Aggregation.prototype.getAggregationSql = function() {
-    return aggregationFnQueryTpl({
-        _aggregationFn: this.aggregation,
-        _aggregationColumn: this.aggregationColumn || 1
-    });
-};
-
-Aggregation.prototype.format = function(result) {
-    var categories = [];
-    var count = 0;
-    var nulls = 0;
-    var minValue = 0;
-    var maxValue = 0;
-    var categoriesCount = 0;
-
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        count = firstRow.count;
-        nulls = firstRow.nulls_count;
-        minValue = firstRow.min_val;
-        maxValue = firstRow.max_val;
-        categoriesCount = firstRow.categories_count;
-
-        result.rows.forEach(function(row) {
-            categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
-        });
-    }
-
-    return {
-        aggregation: this.aggregation,
-        count: count,
-        nulls: nulls,
-        min: minValue,
-        max: maxValue,
-        categoriesCount: categoriesCount,
-        categories: categories
-    };
-};
-
-var filterCategoriesQueryTpl = dot.template([
-    'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
-    'FROM ({{=it._query}}) _cdb_aggregation_search',
-    'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
-    'GROUP BY {{=it._column}}'
-].join('\n'));
-
-var searchQueryTpl = dot.template([
-    'WITH',
-    'search_unfiltered AS (',
-    '  {{=it._searchUnfiltered}}',
-    '),',
-    'search_filtered AS (',
-    '  {{=it._searchFiltered}}',
-    '),',
-    'search_union AS (',
-    '  SELECT * FROM search_unfiltered',
-    '  UNION ALL',
-    '  SELECT * FROM search_filtered',
-    ')',
-    'SELECT category, sum(value) AS value',
-    'FROM search_union',
-    'GROUP BY category',
-    'ORDER BY value desc'
-].join('\n'));
-
-
-Aggregation.prototype.search = function(psql, userQuery, callback) {
-    var self = this;
-
-    var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
-
-    // TODO unfiltered will be wrong as filters are already applied at this point
-    var query = searchQueryTpl({
-        _searchUnfiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: '0',
-            _userQuery: _userQuery
-        }),
-        _searchFiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: 'count(1)',
-            _userQuery: _userQuery
-        })
-    });
-
-    psql.query(query, function(err, result) {
-        if (err) {
-            return callback(err, result);
+    _checkOptions (options) {
+        if (typeof options.column !== 'string') {
+            throw new Error(`Aggregation expects 'column' in dataview options`);
         }
 
-        return callback(null, {type: self.getType(), categories: result.rows });
-    }, true); // use read-only transaction
-};
+        if (typeof options.aggregation !== 'string') {
+            throw new Error(`Aggregation expects 'aggregation' operation in dataview options`);
+        }
 
-Aggregation.prototype.getType = function() {
-    return TYPE;
-};
+        if (!VALID_OPERATIONS[options.aggregation]) {
+            throw new Error(`Aggregation does not support '${options.aggregation}' operation`);
+        }
 
-Aggregation.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _aggregation: this.aggregation
-    });
+        const requiredOptions = VALID_OPERATIONS[options.aggregation];
+        const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption));
+
+        if (missingOptions.length > 0) {
+            throw new Error(
+                `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`
+            );
+        }
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._shouldCheckColumnType()) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const limit = Number.isFinite(override.categories) && override.categories > 0 ?
+            override.categories :
+            CATEGORIES_LIMIT;
+
+        const aggregationSql = aggregationDataviewQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this.aggregation,
+            aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null,
+            aggregationFn: aggregationFnQueryTpl({
+                aggregation: this.aggregation,
+                aggregationColumn: this.aggregationColumn || 1
+            }),
+            isFloatColumn: this._isFloatColumn,
+            limit
+        });
+
+        debug(aggregationSql);
+
+        return callback(null, aggregationSql);
+    }
+
+    _shouldCheckColumnType () {
+        return this.aggregationColumn && this._isFloatColumn === null;
+    }
+
+    format (result) {
+        const {
+            count = 0,
+            nulls_count = 0,
+            nans_count = 0,
+            infinities_count = 0,
+            min_val = 0,
+            max_val = 0,
+            categories_count = 0
+         } = result.rows[0] || {};
+
+        return {
+            aggregation: this.aggregation,
+            count: count,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count,
+            min: min_val,
+            max: max_val,
+            categoriesCount: categories_count,
+            categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg }))
+        };
+    }
+
+    search (psql, userQuery, callback) {
+        const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`);
+        const value = this.aggregation !== 'count' && this.aggregationColumn ?
+            `${this.aggregation}(${this.aggregationColumn})` :
+            'count(1)';
+
+        // TODO unfiltered will be wrong as filters are already applied at this point
+        const query = searchQueryTpl({
+            searchUnfiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: '0',
+                userQuery: escapedUserQuery
+            }),
+            searchFiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: value,
+                userQuery: escapedUserQuery
+            })
+        });
+
+        debug(query);
+
+        psql.query(query, (err, result) => {
+            if (err) {
+                return callback(err, result);
+            }
+
+            return callback(null, {type: this.getType(), categories: result.rows });
+        }, true); // use read-only transaction
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _aggregation: this.aggregation
+        });
+    }
 };

lib/cartodb/models/dataview/base.js

@@ -1,26 +1,59 @@
-function BaseDataview() {}
+const FLOAT_OIDS = {
+    700: true,
+    701: true,
+    1700: true
+};
 
-module.exports = BaseDataview;
+const DATE_OIDS = {
+    1082: true,
+    1114: true,
+    1184: true
+};
 
-BaseDataview.prototype.getResult = function(psql, override, callback) {
-    var self = this;
-    this.sql(psql, override, function(err, query) {
-        psql.query(query, function(err, result) {
+const columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`;
 
+function getPGTypeName (pgType) {
+    return {
+        float: FLOAT_OIDS.hasOwnProperty(pgType),
+        date: DATE_OIDS.hasOwnProperty(pgType)
+    };
+}
+
+module.exports = class BaseDataview {
+    getResult (psql, override, callback) {
+        this.sql(psql, override, (err, query) => {
             if (err) {
-                return callback(err, result);
+                return callback(err);
             }
 
-            result = self.format(result, override);
-            result.type = self.getType();
+            psql.query(query, (err, result) => {
+                if (err) {
+                    return callback(err, result);
+                }
 
-            return callback(null, result);
+                result = this.format(result, override);
+                result.type = this.getType();
 
-        }, true); // use read-only transaction
-    });
+                return callback(null, result);
 
-};
-
-BaseDataview.prototype.search = function(psql, userQuery, callback) {
-    return callback(null, this.format({ rows: [] }));
+            }, true); // use read-only transaction
+        });
+    }
+
+    search (psql, userQuery, callback) {
+        return callback(null, this.format({ rows: [] }));
+    }
+
+    getColumnType (psql, column, query, callback) {
+        const readOnlyTransaction = true;
+        const columnTypeQuery = columnTypeQueryTpl({ column, query });
+
+        psql.query(columnTypeQuery, (err, result) => {
+            if (err) {
+                return callback(err);
+            }
+            const pgType = result.rows[0].pg_typeof;
+            callback(null, getPGTypeName(pgType));
+        }, readOnlyTransaction);
+    }
 };

lib/cartodb/models/dataview/factory.js

@@ -1,18 +1,20 @@
-var dataviews = require('./');
+const dataviews = require('./');
 
-var DataviewFactory = {
-    dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
-        allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
-        return allDataviews;
-    }, {}),
+module.exports = class DataviewFactory {
+    static get dataviews() {
+        return Object.keys(dataviews).reduce((allDataviews, dataviewClassName) => {
+            allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
+            return allDataviews;
+        }, {});
+    }
+
+    static getDataview (query, dataviewDefinition) {
+        const { type, options, sql } = dataviewDefinition;
 
-    getDataview: function(query, dataviewDefinition) {
-        var type = dataviewDefinition.type;
         if (!this.dataviews[type]) {
             throw new Error('Invalid dataview type: "' + type + '"');
         }
-        return new this.dataviews[type](query, dataviewDefinition.options);
+
+        return new this.dataviews[type](query, options, sql);
     }
 };
-
-module.exports = DataviewFactory;

lib/cartodb/models/dataview/formula.js

@@ -1,18 +1,36 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:formula');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:formula');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const countInfinitiesQueryTpl = ctx => `
+    SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities
+    WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float
+`;
 
-var formulaQueryTpl = dot.template([
-    'SELECT',
-    '{{=it._operation}}({{=it._column}}) AS result,',
-    '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-    'FROM ({{=it._query}}) _cdb_formula'
-].join('\n'));
+const countNansQueryTpl = ctx => `
+    SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans
+    WHERE ${ctx.column} = 'NaN'::float
+`;
 
-var VALID_OPERATIONS = {
+const filterOutSpecialNumericValuesTpl = ctx => `
+    WHERE
+        ${ctx.column} != 'infinity'::float
+    AND
+        ${ctx.column} != '-infinity'::float
+    AND
+        ${ctx.column} != 'NaN'::float
+`;
+
+const formulaQueryTpl = ctx => `
+    SELECT
+        ${ctx.operation}(${ctx.column}) AS result,
+        (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count
+        ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''}
+        ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''}
+    FROM (${ctx.query}) __cdb_formula
+    ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''}
+`;
+
+const VALID_OPERATIONS = {
     count: true,
     avg: true,
     sum: true,
@@ -20,7 +38,7 @@ var VALID_OPERATIONS = {
     max: true
 };
 
-var TYPE = 'formula';
+const TYPE = 'formula';
 
 /**
  {
@@ -31,74 +49,90 @@ var TYPE = 'formula';
      }
  }
  */
-function Formula(query, options) {
-    if (!_.isString(options.operation)) {
-        throw new Error('Formula expects `operation` in widget options');
+module.exports = class Formula extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column || '1';
+        this.operation = options.operation;
+        this._isFloatColumn = null;
     }
 
-    if (!VALID_OPERATIONS[options.operation]) {
-        throw new Error("Formula does not support '" + options.operation + "' operation");
+    _checkOptions (options) {
+        if (typeof options.operation !== 'string') {
+            throw new Error(`Formula expects 'operation' in dataview options`);
+        }
+
+        if (!VALID_OPERATIONS[options.operation]) {
+            throw new Error(`Formula does not support '${options.operation}' operation`);
+        }
+
+        if (options.operation !== 'count' && typeof options.column !== 'string') {
+            throw new Error(`Formula expects 'column' in dataview options`);
+        }
     }
 
-    if (options.operation !== 'count' && !_.isString(options.column)) {
-        throw new Error('Formula expects `column` in widget options');
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._isFloatColumn === null) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const formulaSql = formulaQueryTpl({
+            isFloatColumn: this._isFloatColumn,
+            query: this.query,
+            operation: this.operation,
+            column: this.column
+        });
+
+        debug(formulaSql);
+
+        return callback(null, formulaSql);
     }
 
-    BaseWidget.apply(this);
+    format (res) {
+        const {
+            result = 0,
+            nulls_count = 0,
+            nans_count,
+            infinities_count
+        } = res.rows[0] || {};
 
-    this.query = query;
-    this.column = options.column || '1';
-    this.operation = options.operation;
-}
-
-Formula.prototype = new BaseWidget();
-Formula.prototype.constructor = Formula;
-
-module.exports = Formula;
-
-Formula.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
+        return {
+            operation: this.operation,
+            result,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count
+        };
     }
 
-    var _query = this.query;
-    var formulaSql = formulaQueryTpl({
-        _query: _query,
-        _operation: this.operation,
-        _column: this.column
-    });
+    getType () {
+        return TYPE;
+    }
 
-    debug(formulaSql);
-
-    return callback(null, formulaSql);
-};
-
-Formula.prototype.format = function(result) {
-    var formattedResult = {
-        operation: this.operation,
-        result: 0,
-        nulls: 0
-    };
-
-    if (result.rows.length) {
-        formattedResult.operation = this.operation;
-        formattedResult.result = result.rows[0].result;
-        formattedResult.nulls = result.rows[0].nulls_count;
-    }
-
-    return formattedResult;
-};
-
-Formula.prototype.getType = function() {
-    return TYPE;
-};
-
-Formula.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _operation: this.operation
-    });
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _operation: this.operation
+        });
+    }
 };

lib/cartodb/models/dataview/histogram.js

@@ -1,307 +1,72 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:dataview:histogram');
+const debug = require('debug')('windshaft:dataview:histogram');
+const NumericHistogram = require('./histograms/numeric-histogram');
+const DateHistogram = require('./histograms/date-histogram');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const DATE_HISTOGRAM = 'DateHistogram';
+const NUMERIC_HISTOGRAM = 'NumericHistogram';
 
-var columnTypeQueryTpl = dot.template(
-    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
-);
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+module.exports = class Histogram {
+    constructor (query, options, queries) {
+        this.query = query;
+        this.options = options || {};
+        this.queries = queries;
 
-var BIN_MIN_NUMBER = 6;
-var BIN_MAX_NUMBER = 48;
-
-var basicsQueryTpl = dot.template([
-    'basics AS (',
-    '  SELECT',
-    '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
-    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
-    ')'
-].join(' \n'));
-
-var overrideBasicsQueryTpl = dot.template([
-    'basics AS (',
-    '  SELECT',
-    '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
-    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
-    ')'
-].join('\n'));
-
-var iqrQueryTpl = dot.template([
-    'iqrange AS (',
-    '  SELECT max(quartile_max) - min(quartile_max) AS iqr',
-    '  FROM (',
-    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
-    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
-    '    ) AS quartile',
-    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
-    '    WHERE quartile = 1 or quartile = 3',
-    '    GROUP BY quartile',
-    '  ) _cdb_iqr',
-    ')'
-].join('\n'));
-
-var binsQueryTpl = dot.template([
-    'bins AS (',
-    '  SELECT CASE WHEN total_rows = 0 OR iqr = 0',
-    '      THEN 1',
-    '      ELSE GREATEST(',
-    '        LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
-    '        LEAST(',
-    '          CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
-    '          {{=it._maxBins}}',
-    '        )',
-    '      )',
-    '    END AS bins_number',
-    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
-    '  LIMIT 1',
-    ')'
-].join('\n'));
-
-var overrideBinsQueryTpl = dot.template([
-    'bins AS (',
-    '  SELECT {{=it._bins}} AS bins_number',
-    ')'
-].join('\n'));
-
-var nullsQueryTpl = dot.template([
-    'nulls AS (',
-    '  SELECT',
-    '  count(*) AS nulls_count',
-    '  FROM ({{=it._query}}) _cdb_histogram_nulls',
-    '  WHERE {{=it._column}} IS NULL',
-    ')'
-].join('\n'));
-
-var histogramQueryTpl = dot.template([
-    'SELECT',
-    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
-    '    bins_number,',
-    '    nulls_count,',
-    '    avg_val,',
-    '    CASE WHEN min_val = max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
-    '    END AS bin,',
-    '    min({{=it._column}})::numeric AS min,',
-    '    max({{=it._column}})::numeric AS max,',
-    '    avg({{=it._column}})::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
-    'WHERE {{=it._column}} IS NOT NULL',
-    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
-    'ORDER BY bin'
-].join('\n'));
-
-
-var TYPE = 'histogram';
-
-/**
- {
-     type: 'histogram',
-     options: {
-         column: 'name',
-         bins: 10 // OPTIONAL
-     }
- }
- */
-function Histogram(query, options) {
-    if (!_.isString(options.column)) {
-        throw new Error('Histogram expects `column` in widget options');
+        this.histogramImplementation = this._getHistogramImplementation();
     }
 
-    this.query = query;
-    this.column = options.column;
-    this.bins = options.bins;
+    _getHistogramImplementation (override) {
+        let implementation = null;
 
-    this._columnType = null;
-}
-
-Histogram.prototype = new BaseWidget();
-Histogram.prototype.constructor = Histogram;
-
-module.exports = Histogram;
-
-var DATE_OIDS = {
-    1082: true,
-    1114: true,
-    1184: true
-};
-
-Histogram.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    var self = this;
-
-    var _column = this.column;
-
-    var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.query
-    });
-
-    if (this._columnType === null) {
-        psql.query(columnTypeQuery, function(err, result) {
-            // assume numeric, will fail later
-            self._columnType = 'numeric';
-            if (!err && !!result.rows[0]) {
-                var pgType = result.rows[0].pg_typeof;
-                if (DATE_OIDS.hasOwnProperty(pgType)) {
-                    self._columnType = 'date';
-                }
-            }
-            self.sql(psql, override, callback);
-        }, true); // use read-only transaction
-        return null;
-    }
-
-    if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
-    }
-
-    var _query = this.query;
-
-    var basicsQuery, binsQuery;
-
-    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
-        debug('overriding with %j', override);
-        basicsQuery = overrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _start: getBinStart(override),
-            _end: getBinEnd(override)
-        });
-
-        binsQuery = [
-            overrideBinsQueryTpl({
-                _bins: override.bins
-            })
-        ].join(',\n');
-    } else {
-        basicsQuery = basicsQueryTpl({
-            _query: _query,
-            _column: _column
-        });
-
-        if (override && _.has(override, 'bins')) {
-            binsQuery = [
-                overrideBinsQueryTpl({
-                    _bins: override.bins
-                })
-            ].join(',\n');
-        } else {
-            binsQuery = [
-                iqrQueryTpl({
-                    _query: _query,
-                    _column: _column
-                }),
-                binsQueryTpl({
-                    _query: _query,
-                    _minBins: BIN_MIN_NUMBER,
-                    _maxBins: BIN_MAX_NUMBER
-                })
-            ].join(',\n');
+        switch (this._getHistogramSubtype(override)) {
+            case DATE_HISTOGRAM:
+                debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new DateHistogram(this.query, this.options, this.queries);
+                break;
+            case NUMERIC_HISTOGRAM:
+                debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new NumericHistogram(this.query, this.options, this.queries);
+                break;
+            default:
+                throw new Error('Unsupported Histogram type');
         }
+
+        return implementation;
     }
 
+    _getHistogramSubtype (override) {
+        if(this._isDateHistogram(override)) {
+            return DATE_HISTOGRAM;
+        }
 
-    var histogramSql = [
-        "WITH",
-        [
-            basicsQuery,
-            binsQuery,
-            nullsQueryTpl({
-                _query: _query,
-                _column: _column
-            })
-        ].join(',\n'),
-        histogramQueryTpl({
-            _query: _query,
-            _column: _column
-        })
-    ].join('\n');
+        return NUMERIC_HISTOGRAM;
+    }
 
-    debug(histogramSql);
+    _isDateHistogram (override = {}) {
+        return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation'));
+    }
 
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype.format = function(result, override) {
-    override = override || {};
-    var buckets = [];
-
-    var binsCount = getBinsCount(override);
-    var width = getWidth(override);
-    var binsStart = getBinStart(override);
-    var nulls = 0;
-    var avg;
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        binsCount = firstRow.bins_number;
-        width = firstRow.bin_width || width;
-        avg = firstRow.avg_val;
-        nulls = firstRow.nulls_count;
-        binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
-
-        buckets = result.rows.map(function(row) {
-            return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
-        });
-    }
-
-    return {
-        bin_width: width,
-        bins_count: binsCount,
-        bins_start: binsStart,
-        nulls: nulls,
-        avg: avg,
-        bins: buckets
-    };
-};
-
-function getBinStart(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.min(override.start, override.end);
-    }
-    return override.start || 0;
-}
-
-function getBinEnd(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.max(override.start, override.end);
-    }
-    return override.end || 0;
-}
-
-function getBinsCount(override) {
-    return override.bins || 0;
-}
-
-function getWidth(override) {
-    var width = 0;
-    var binsCount = override.bins;
-
-    if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
-        width = (override.end - override.start) / binsCount;
-    }
-
-    return width;
-}
-
-Histogram.prototype.getType = function() {
-    return TYPE;
-};
-
-Histogram.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _column: this.column,
-        _query: this.query
-    });
+    getResult (psql, override, callback) {
+        this.histogramImplementation = this._getHistogramImplementation(override);
+        this.histogramImplementation.getResult(psql, override, callback);
+    }
+
+    // In order to keep previous behaviour with overviews,
+    // we have to expose the following methods to bypass
+    // the concrete overview implementation
+
+    sql (psql, override, callback) {
+        this.histogramImplementation.sql(psql, override, callback);
+    }
+
+    format (result, override) {
+        return this.histogramImplementation.format(result, override);
+    }
+
+    getType () {
+        return this.histogramImplementation.getType();
+    }
+
+    toString () {
+        return this.histogramImplementation.toString();
+    }
 };

lib/cartodb/models/dataview/histograms/base-histogram.js

@@ -0,0 +1,85 @@
+const BaseDataview = require('../base');
+
+const TYPE = 'histogram';
+
+module.exports = class BaseHistogram extends BaseDataview {
+    constructor (query, options, queries) {
+        super();
+
+        if (typeof options.column !== 'string') {
+            throw new Error('Histogram expects `column` in widget options');
+        }
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.bins = options.bins;
+
+        this._columnType = null;
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._columnType === null) {
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                // assume numeric, will fail later
+                this._columnType = 'numeric';
+                if (!err && !!type) {
+                    this._columnType = Object.keys(type).find(function (key) {
+                        return type[key];
+                    });
+                }
+                this.sql(psql, override, callback);
+            }, true); // use read-only transaction
+            return null;
+        }
+
+        return this._buildQuery(psql, override, callback);
+    }
+
+    format (result, override) {
+        const histogram = this._getSummary(result, override);
+        histogram.bins = this._getBuckets(result);
+        return histogram;
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _column: this.column,
+            _query: this.query
+        });
+    }
+
+    _hasOverridenRange (override) {
+        return override && override.hasOwnProperty('start') && override.hasOwnProperty('end');
+    }
+
+    _getBinStart (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.min(override.start, override.end);
+        }
+
+        return override.start || 0;
+    }
+
+    _getBinEnd (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.max(override.start, override.end);
+        }
+
+        return override.end || 0;
+    }
+
+    _getBinsCount (override = {}) {
+        return override.bins || 0;
+    }
+};

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -0,0 +1,302 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:date-histogram');
+
+const dateIntervalQueryTpl = ctx => `
+    WITH
+    __cdb_dates AS (
+        SELECT
+            MAX(${ctx.column}::timestamp) AS __cdb_end,
+            MIN(${ctx.column}::timestamp) AS __cdb_start
+        FROM (${ctx.query}) __cdb_source
+    ),
+    __cdb_interval_in_days AS (
+        SELECT
+            DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days
+        FROM __cdb_dates
+    ),
+    __cdb_interval_in_hours AS (
+        SELECT
+            __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours
+        FROM __cdb_interval_in_days, __cdb_dates
+    ),
+    __cdb_interval_in_minutes AS (
+        SELECT
+            __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes
+        FROM __cdb_interval_in_hours, __cdb_dates
+    ),
+    __cdb_interval_in_seconds AS (
+        SELECT
+            __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds
+        FROM __cdb_interval_in_minutes, __cdb_dates
+    )
+    SELECT
+        ROUND(__cdb_days / 365) AS year,
+        ROUND(__cdb_days / 90) AS quarter,
+        ROUND(__cdb_days / 30) AS month,
+        ROUND(__cdb_days / 7) AS week,
+        __cdb_days AS day,
+        __cdb_hours AS hour,
+        __cdb_minutes AS minute,
+        __cdb_seconds AS second
+    FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
+`;
+
+const nullsQueryTpl = ctx => `
+    __cdb_nulls AS (
+        SELECT
+            count(*) AS __cdb_nulls_count
+        FROM (${ctx.query}) __cdb_histogram_nulls
+        WHERE ${ctx.column} IS NULL
+    )
+`;
+
+const dateBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(date_part('epoch', ${ctx.column})) AS __cdb_max_val,
+            min(date_part('epoch', ${ctx.column})) AS __cdb_min_val,
+            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
+            min(
+                date_trunc(
+                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
+                )
+            ) AS __cdb_start_date,
+            max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date,
+            count(1) AS __cdb_total_rows
+        FROM (${ctx.query}) __cdb_basics_query
+    )
+`;
+
+const dateOverrideBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.end})::float AS __cdb_max_val,
+            min(${ctx.start})::float AS __cdb_min_val,
+            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
+            min(
+                date_trunc(
+                    '${ctx.aggregation}',
+                    TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}'
+                )
+            ) AS __cdb_start_date,
+            max(
+                TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}'
+            ) AS __cdb_end_date,
+            count(1) AS __cdb_total_rows
+        FROM (${ctx.query}) __cdb_basics_query
+    )
+`;
+
+const dateBinsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT
+            __cdb_bins_array,
+            ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number
+        FROM (
+            SELECT
+                ARRAY(
+                    SELECT GENERATE_SERIES(
+                        __cdb_start_date::timestamptz,
+                        __cdb_end_date::timestamptz,
+                        ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`}
+                    )
+                ) AS __cdb_bins_array
+            FROM __cdb_basics
+        ) __cdb_bins_array_query
+    )
+`;
+
+const dateHistogramQueryTpl = ctx => `
+    SELECT
+        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
+        __cdb_bins_number AS bins_number,
+        __cdb_nulls_count AS nulls_count,
+        CASE WHEN __cdb_min_val = __cdb_max_val
+        THEN 0
+        ELSE GREATEST(
+            1,
+            LEAST(
+                WIDTH_BUCKET(
+                    ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}',
+                    __cdb_bins_array
+                ),
+                __cdb_bins_number
+            )
+        ) - 1
+        END AS bin,
+        min(
+            date_part(
+                'epoch',
+                date_trunc(
+                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
+                ) AT TIME ZONE '${ctx.offset}'
+            )
+        )::numeric AS timestamp,
+        date_part('epoch', __cdb_start_date)::numeric AS timestamp_start,
+        min(date_part('epoch', ${ctx.column}))::numeric AS min,
+        max(date_part('epoch', ${ctx.column}))::numeric AS max,
+        avg(date_part('epoch', ${ctx.column}))::numeric AS avg,
+        count(*) AS freq
+    FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls
+    WHERE date_part('epoch', ${ctx.column}) IS NOT NULL
+    GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start
+    ORDER BY bin
+`;
+
+const MAX_INTERVAL_VALUE = 366;
+
+const DATE_AGGREGATIONS = {
+    'auto': true,
+    'minute': true,
+    'hour': true,
+    'day': true,
+    'week': true,
+    'month': true,
+    'quarter': true,
+    'year': true
+};
+
+/**
+    date_histogram: {
+        type: 'histogram',
+        options: {
+            column: 'date', // column data type: date
+            aggregation: 'day' // MANDATORY
+            offset: -7200 // OPTIONAL (UTC offset in seconds)
+        }
+    }
+*/
+module.exports = class DateHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+
+        this.aggregation = options.aggregation;
+        this.offset = options.offset;
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+            WITH
+            ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)},
+            ${dateBinsQueryTpl(ctx)},
+            ${nullsQueryTpl(ctx)}
+            ${dateHistogramQueryTpl(ctx)}
+        `;
+    }
+
+    _buildQuery (psql, override, callback) {
+        if (!this._isValidAggregation(override)) {
+            return callback(new Error('Invalid aggregation value. Valid ones: ' +
+                Object.keys(DATE_AGGREGATIONS).join(', ')
+            ));
+        }
+
+        if (this._getAggregation(override) === 'auto') {
+            this._getAutomaticAggregation(psql, function (err, aggregation) {
+                if (err || aggregation === 'none') {
+                    this.aggregation = 'day';
+                } else {
+                    this.aggregation = aggregation;
+                }
+                override.aggregation = this.aggregation;
+                this._buildQuery(psql, override, callback);
+            }.bind(this));
+            return null;
+        }
+
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this._getAggregation(override),
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            offset: this._parseOffset(override)
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _isValidAggregation (override) {
+        return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override));
+    }
+
+    _getAutomaticAggregation (psql, callback) {
+        const dateIntervalQuery = dateIntervalQueryTpl({
+            query: this.query,
+            column: this.column
+        });
+
+        psql.query(dateIntervalQuery, function (err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            const aggegations = result.rows[0];
+            const aggregation = Object.keys(aggegations)
+                .map(key => ({ name: key, value: aggegations[key] }))
+                .reduce((closer, current) => {
+                    if (current.value > MAX_INTERVAL_VALUE) {
+                        return closer;
+                    }
+
+                    const closerDiff = MAX_INTERVAL_VALUE - closer.value;
+                    const currentDiff = MAX_INTERVAL_VALUE - current.value;
+
+                    if (Number.isFinite(current.value) && closerDiff > currentDiff) {
+                        return current;
+                    }
+
+                    return closer;
+                }, { name: 'none', value: -1 });
+
+            callback(null, aggregation.name);
+        });
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            aggregation: this._getAggregation(override),
+            offset: this._getOffset(override),
+            timestamp_start: firstRow.timestamp_start,
+
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: firstRow.timestamp,
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
+    }
+
+    _getAggregation (override = {}) {
+        return override.aggregation ? override.aggregation : this.aggregation;
+    }
+
+    _getOffset (override = {}) {
+        return Number.isFinite(override.offset) ? override.offset : (this.offset || 0);
+    }
+
+    _parseOffset (override) {
+        if (this._shouldIgnoreOffset(override)) {
+            return '0';
+        }
+
+        const offsetInHours = Math.ceil(this._getOffset(override) / 3600);
+
+        return '' + offsetInHours;
+    }
+
+    _shouldIgnoreOffset (override) {
+        return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute');
+    }
+};

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -0,0 +1,234 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:numeric-histogram');
+
+const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`;
+
+const filterOutSpecialNumericValues = ctx => `
+        ${ctx.column} != 'infinity'::float
+    AND
+        ${ctx.column} != '-infinity'::float
+    AND
+        ${ctx.column} != 'NaN'::float
+`;
+
+const filteredQueryTpl = ctx => `
+    __cdb_filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) __cdb_filtered_source_query
+        WHERE ${ctx.column} IS NOT NULL
+        ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''}
+    )
+`;
+
+const basicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val,
+            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+const overrideBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
+            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+const iqrQueryTpl = ctx => `
+    __cdb_iqrange AS (
+        SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
+        FROM (
+            SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (
+                SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column}
+            ) AS quartile
+            FROM __cdb_filtered_source) _cdb_quartiles
+            WHERE quartile = 1 or quartile = 3
+            GROUP BY quartile
+        ) __cdb_iqr
+    )
+`;
+
+const binsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT
+            CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
+            THEN 1
+            ELSE GREATEST(
+                LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)),
+                LEAST(
+                    CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),
+                    ${ctx.maxBins}
+                )
+            )
+            END AS __cdb_bins_number
+        FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source
+        LIMIT 1
+    )
+`;
+
+const overrideBinsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT ${ctx.override.bins} AS __cdb_bins_number
+    )
+`;
+
+const nullsQueryTpl = ctx => `
+    __cdb_nulls AS (
+        SELECT
+            count(*) AS __cdb_nulls_count
+        FROM (${ctx.query}) __cdb_histogram_nulls
+        WHERE ${ctx.column} IS NULL
+    )
+`;
+
+const infinitiesQueryTpl = ctx => `
+    __cdb_infinities AS (
+        SELECT
+            count(*) AS __cdb_infinities_count
+        FROM (${ctx.query}) __cdb_infinities_query
+        WHERE
+            ${ctx.column} = 'infinity'::float
+        OR
+            ${ctx.column} = '-infinity'::float
+    )
+`;
+
+const nansQueryTpl = ctx => `
+    __cdb_nans AS (
+        SELECT
+            count(*) AS __cdb_nans_count
+        FROM (${ctx.query}) __cdb_nans_query
+        WHERE ${ctx.column} = 'NaN'::float
+    )
+`;
+
+const specialNumericValuesColumnDefinitionTpl = () => `
+    __cdb_infinities_count AS infinities_count,
+    __cdb_nans_count AS nans_count
+`;
+
+const specialNumericValuesCTETpl = () => `
+    __cdb_infinities, __cdb_nans
+`;
+
+const specialNumericValuesColumnTpl = () => `
+    infinities_count, nans_count
+`;
+
+const histogramQueryTpl = ctx => `
+    SELECT
+        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
+        __cdb_bins_number AS bins_number,
+        __cdb_nulls_count AS nulls_count,
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
+        __cdb_avg_val AS avg_val,
+        CASE WHEN __cdb_min_val = __cdb_max_val
+            THEN 0
+            ELSE GREATEST(
+                1,
+                LEAST(
+                    WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
+                    __cdb_bins_number
+                )
+            ) - 1
+        END AS bin,
+        min(${ctx.column})::numeric AS min,
+        max(${ctx.column})::numeric AS max,
+        avg(${ctx.column})::numeric AS avg,
+        count(*) AS freq
+    FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
+        ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
+    GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
+        ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
+    ORDER BY bin
+`;
+
+const BIN_MIN_NUMBER = 6;
+const BIN_MAX_NUMBER = 48;
+
+/**
+Numeric histogram:
+{
+    type: 'histogram',
+    options: {
+        column: 'name', // column data type: numeric
+        bins: 10 // OPTIONAL
+    }
+}
+*/
+module.exports = class NumericHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+    }
+
+    _buildQuery (psql, override, callback) {
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
+            isFloatColumn: this._columnType === 'float',
+            query: this.query,
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            minBins: BIN_MIN_NUMBER,
+            maxBins: BIN_MAX_NUMBER,
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+            WITH
+            ${filteredQueryTpl(ctx)},
+            ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)},
+            ${this._hasOverridenBins(ctx.override) ?
+                overrideBinsQueryTpl(ctx) :
+                `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`
+            },
+            ${nullsQueryTpl(ctx)}
+            ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''}
+            ${histogramQueryTpl(ctx)}
+        `;
+    }
+
+    _hasOverridenBins (override) {
+        return override && override.hasOwnProperty('bins');
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: this._populateBinStart(firstRow, override),
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val,
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq }));
+    }
+
+    _populateBinStart (firstRow, override = {}) {
+        let binStart;
+
+        if (override.hasOwnProperty('start')) {
+            binStart = this._getBinStart(override);
+        } else {
+            binStart = firstRow.min;
+        }
+
+        return binStart;
+    }
+
+};

lib/cartodb/models/dataview/index.js

@@ -1,6 +1,5 @@
 module.exports = {
     Aggregation: require('./aggregation'),
     Formula: require('./formula'),
-    Histogram: require('./histogram'),
-    List: require('./list')
+    Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/list.js

@@ -1,66 +0,0 @@
-var dot = require('dot');
-dot.templateSettings.strip = false;
-
-var BaseWidget = require('./base');
-
-var TYPE = 'list';
-
-var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
-
-/**
-{
-    type: 'list',
-    options: {
-        columns: ['name', 'description']
-    }
-}
-*/
-
-function List(query, options) {
-    options = options || {};
-
-    if (!Array.isArray(options.columns)) {
-        throw new Error('List expects `columns` array in widget options');
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.columns = options.columns;
-}
-
-List.prototype = new BaseWidget();
-List.prototype.constructor = List;
-
-module.exports = List;
-
-List.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-    }
-
-    var listSql = listSqlTpl({
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-
-    return callback(null, listSql);
-};
-
-List.prototype.format = function(result) {
-    return {
-        rows: result.rows
-    };
-};
-
-List.prototype.getType = function() {
-    return TYPE;
-};
-
-List.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-};

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -1,14 +1,36 @@
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../aggregation');
+var debug = require('debug')('windshaft:widget:aggregation:overview');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
+var filteredQueryTpl = dot.template([
+    'filtered_source AS (',
+    '  SELECT *',
+    '  FROM ({{=it._query}}) _cdb_filtered_source',
+    '  {{?it._aggregationColumn  && it._isFloatColumn}}WHERE',
+    '    {{=it._aggregationColumn}} != \'infinity\'::float',
+    '  AND',
+    '    {{=it._aggregationColumn}} != \'-infinity\'::float',
+    '  AND',
+    '    {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
+    ')'
+].join(' \n'));
+
 var summaryQueryTpl = dot.template([
     'summary AS (',
     '  SELECT',
     '  sum(_feature_count) AS count,',
     '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
+    '  {{?it._isFloatColumn}},sum(',
+    '    CASE',
+    '      WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
+    '      THEN 1',
+    '      ELSE 0',
+    '    END',
+    '  ) AS infinities_count,',
+    '  sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
     '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
     ')'
 ].join('\n'));
@@ -17,47 +39,65 @@ var rankedCategoriesQueryTpl = dot.template([
     'categories AS(',
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
-    '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  FROM filtered_source',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM filtered_source',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
+    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
+    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
-    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
+    'GROUP BY nulls_count, min_val, max_val, count,',
+    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
 ].join('\n'));
 
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
-    '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
-    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
+    '  nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
+    'GROUP BY category, nulls_count, min_val, max_val, count,',
+    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
     'ORDER BY value DESC'
 ].join('\n'));
 
 var CATEGORIES_LIMIT = 6;
 
- function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+ function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.aggregation = options.aggregation;
     this.aggregationColumn = options.aggregationColumn;
+    this._isFloatColumn = null;
 }
 
 Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
@@ -65,34 +105,62 @@ Aggregation.prototype.constructor = Aggregation;
 
 module.exports = Aggregation;
 
-Aggregation.prototype.sql = function(psql, filters, override, callback) {
+Aggregation.prototype.sql = function(psql, override, callback) {
+    var self = this;
+
     if (!callback) {
         callback = override;
         override = {};
     }
 
     var _query = this.rewrittenQuery(this.query);
+    var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
+
+    if (this.aggregationColumn && this._isFloatColumn === null) {
+        this._isFloatColumn = false;
+        this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
+            if (!err && !!type) {
+                self._isFloatColumn = type.float;
+            }
+            self.sql(psql, override, callback);
+        });
+        return null;
+    }
 
     var aggregationSql;
     if (!!override.ownFilter) {
         aggregationSql = [
             "WITH",
             [
-                summaryQueryTpl({
+                filteredQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
                     _query: _query,
-                    _column: this.column
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                summaryQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
                 }),
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: _aggregationColumn
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
             ].join(',\n'),
             aggregationQueryTpl({
+                _isFloatColumn: this._isFloatColumn,
                 _query: _query,
                 _column: this.column,
                 _aggregation: this.getAggregationSql(),
@@ -103,21 +171,35 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
         aggregationSql = [
             "WITH",
             [
-                summaryQueryTpl({
+                filteredQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
                     _query: _query,
-                    _column: this.column
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                summaryQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
                 }),
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: _aggregationColumn
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
             ].join(',\n'),
             rankedAggregationQueryTpl({
+                _isFloatColumn: this._isFloatColumn,
                 _query: _query,
                 _column: this.column,
                 _limit: CATEGORIES_LIMIT
@@ -125,6 +207,8 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
         ].join('\n');
     }
 
+    debug(aggregationSql);
+
     return callback(null, aggregationSql);
 };
 

lib/cartodb/models/dataview/overviews/base.js

@@ -1,14 +1,15 @@
 var _ = require('underscore');
 var BaseDataview = require('../base');
 
-function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
+function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
   this.BaseDataview = BaseDataview;
   this.query = query;
   this.queryOptions = queryOptions;
   this.queryRewriter = queryRewriter;
   this.queryRewriteData = queryRewriteData;
   this.options = options;
-  this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
+  this.queries = queries;
+  this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
 }
 
 module.exports = BaseOverviewsDataview;
@@ -55,20 +56,20 @@ BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
 };
 
 // Default behaviour
-BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
+BaseOverviewsDataview.prototype.defaultSql = function(psql, override, callback) {
     var query = this.query;
     var dataview = this.baseDataview;
     if ( SETTINGS.defaultOverviews ) {
         query = this.rewrittenQuery(query);
         dataview = new this.BaseDataview(query, this.queryOptions);
     }
-    return dataview.sql(psql, filters, override, callback);
+    return dataview.sql(psql, override, callback);
 };
 
 // default implementation that can be override in derived classes:
 
-BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
-    return this.defaultSql(psql, filters, override, callback);
+BaseOverviewsDataview.prototype.sql = function(psql, override, callback) {
+    return this.defaultSql(psql, override, callback);
 };
 
 BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {

lib/cartodb/models/dataview/overviews/factory.js

@@ -14,7 +14,8 @@ OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinit
         return parentFactory.getDataview(query, dataviewDefinition);
     }
     return new dataviews[type](
-        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
+        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options,
+        dataviewDefinition.sql
     );
 };
 

lib/cartodb/models/dataview/overviews/formula.js

@@ -1,34 +1,61 @@
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../formula');
+var debug = require('debug')('windshaft:widget:formula:overview');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
 var formulaQueryTpls = {
-  'count': dot.template([
-      'SELECT',
-      'sum(_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
-  'sum': dot.template([
-      'SELECT',
-      'sum({{=it._column}}*_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
-  'avg': dot.template([
-      'SELECT',
-      'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
+    'count': dot.template([
+        'SELECT',
+        'sum(_feature_count) AS result,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
+        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
+        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
+        'FROM ({{=it._query}}) _cdb_formula'
+    ].join('\n')),
+    'sum': dot.template([
+        'SELECT',
+        'sum({{=it._column}}*_feature_count) AS result,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
+        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
+        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
+        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
+        'FROM ({{=it._query}}) _cdb_formula',
+        '{{?it._isFloatColumn}}WHERE',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float{{?}}'
+    ].join('\n')),
+    'avg': dot.template([
+        'SELECT',
+        'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
+        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
+        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
+        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
+        'FROM ({{=it._query}}) _cdb_formula',
+        '{{?it._isFloatColumn}}WHERE',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float{{?}}'
+    ].join('\n')),
 };
 
-function Formula(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
     this.column = options.column || '1';
     this.operation = options.operation;
+    this._isFloatColumn = null;
+    this.queries = queries;
 }
 
 Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
@@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-Formula.prototype.sql = function(psql, filters, override, callback) {
+Formula.prototype.sql = function (psql, override, callback) {
+    var self = this;
     var formulaQueryTpl = formulaQueryTpls[this.operation];
 
-    if ( formulaQueryTpl ) {
+    if (formulaQueryTpl) {
         // supported formula for use with overviews
+        if (this._isFloatColumn === null) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
+                if (!err && !!type) {
+                    self._isFloatColumn = type.float;
+                }
+                self.sql(psql, override, callback);
+            });
+            return null;
+        }
+
         var formulaSql = formulaQueryTpl({
-            _query:     this.rewrittenQuery(this.query),
+            _isFloatColumn: this._isFloatColumn,
+            _query: this.rewrittenQuery(this.query),
             _operation: this.operation,
-            _column:    this.column
+            _column: this.column
         });
+
         callback = callback || override;
 
+        debug(formulaSql);
+
         return callback(null, formulaSql);
     }
 
+
     // default behaviour
-    return this.defaultSql(psql, filters, override, callback);
+    return this.defaultSql(psql, override, callback);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -1,24 +1,35 @@
 var _ = require('underscore');
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../histogram');
+var debug = require('debug')('windshaft:dataview:histogram:overview');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
-var columnTypeQueryTpl = dot.template(
-    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
-);
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
-
 var BIN_MIN_NUMBER = 6;
 var BIN_MAX_NUMBER = 48;
 
+var filteredQueryTpl = dot.template([
+    'filtered_source AS (',
+    '  SELECT *',
+    '  FROM ({{=it._query}}) _cdb_filtered_source',
+    '  WHERE',
+    '    {{=it._column}} IS NOT NULL',
+    '  {{?it._isFloatColumn}}AND',
+    '    {{=it._column}} != \'infinity\'::float',
+    '  AND',
+    '    {{=it._column}} != \'-infinity\'::float',
+    '  AND',
+    '    {{=it._column}} != \'NaN\'::float{{?}}',
+    ')'
+].join(' \n'));
+
 var basicsQueryTpl = dot.template([
     'basics AS (',
     '  SELECT',
     '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
     '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
+    '  FROM filtered_source',
     ')'
 ].join(' \n'));
 
@@ -27,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([
     '  SELECT',
     '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
     '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
+    '  FROM filtered_source',
     ')'
 ].join('\n'));
 
@@ -38,7 +49,7 @@ var iqrQueryTpl = dot.template([
     '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
     '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
     '    ) AS quartile',
-    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
+    '    FROM filtered_source) _cdb_quartiles',
     '    WHERE quartile = 1 or quartile = 3',
     '    GROUP BY quartile',
     '  ) _cdb_iqr',
@@ -57,7 +68,7 @@ var binsQueryTpl = dot.template([
     '        )',
     '      )',
     '    END AS bins_number',
-    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
+    '  FROM basics, iqrange, filtered_source',
     '  LIMIT 1',
     ')'
 ].join('\n'));
@@ -77,11 +88,34 @@ var nullsQueryTpl = dot.template([
     ')'
 ].join('\n'));
 
+var infinitiesQueryTpl = dot.template([
+    'infinities AS (',
+    '  SELECT',
+    '    count(*) AS infinities_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_infinities',
+    '  WHERE',
+    '    {{=it._column}} = \'infinity\'::float',
+    '  OR',
+    '    {{=it._column}} = \'-infinity\'::float',
+    ')'
+].join('\n'));
+
+var nansQueryTpl = dot.template([
+    'nans AS (',
+    '  SELECT',
+    '    count(*) AS nans_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_infinities',
+    '  WHERE {{=it._column}} = \'NaN\'::float',
+    ')'
+].join('\n'));
+
 var histogramQueryTpl = dot.template([
     'SELECT',
     '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
     '    bins_number,',
     '    nulls_count,',
+    '    {{?it._isFloatColumn}}infinities_count,',
+    '    nans_count,{{?}}',
     '    avg_val,',
     '    CASE WHEN min_val = max_val',
     '      THEN 0',
@@ -91,16 +125,17 @@ var histogramQueryTpl = dot.template([
     '    max({{=it._column}})::numeric AS max,',
     '    sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
     '    sum(_feature_count) AS freq',
-    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
-    'WHERE {{=it._column}} IS NOT NULL',
+    'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
     'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
+    '   {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
     'ORDER BY bin'
 ].join('\n'));
 
-function Histogram(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.bins = options.bins;
 
@@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram;
 
 module.exports = Histogram;
 
-
-var DATE_OIDS = {
-    1082: true,
-    1114: true,
-    1184: true
-};
-
 Histogram.prototype.sql = function(psql, override, callback) {
+    var self = this;
+
     if (!callback) {
         callback = override;
         override = {};
     }
 
-    var self = this;
-
-    var _column = this.column;
-
-    var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.rewrittenQuery(this.query)
-    });
 
     if (this._columnType === null) {
-        psql.query(columnTypeQuery, function(err, result) {
+        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
             // assume numeric, will fail later
             self._columnType = 'numeric';
-            if (!err && !!result.rows[0]) {
-                var pgType = result.rows[0].pg_typeof;
-                if (DATE_OIDS.hasOwnProperty(pgType)) {
-                    self._columnType = 'date';
-                }
+            if (!err && !!type) {
+                self._columnType = Object.keys(type).find(function (key) {
+                    return type[key];
+                });
             }
             self.sql(psql, override, callback);
         }, true); // use read-only transaction
@@ -149,14 +171,29 @@ Histogram.prototype.sql = function(psql, override, callback) {
     }
 
     if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
+        // overviews currently aggregate dates to NULL
+        // to avoid problem we don't use overviews for histograms of date columns
+        return this.defaultSql(psql, override, callback);
     }
 
+    var histogramSql = this._buildQuery(override);
+
+    return callback(null, histogramSql);
+};
+
+Histogram.prototype._buildQuery = function (override) {
+    var filteredQuery, basicsQuery, binsQuery;
+    var _column = this.column;
     var _query = this.rewrittenQuery(this.query);
 
-    var basicsQuery, binsQuery;
+    filteredQuery = filteredQueryTpl({
+        _isFloatColumn: this._columnType === 'float',
+        _query: _query,
+        _column: _column
+    });
 
-    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
+    if (this._shouldOverride(override)) {
+        debug('overriding with %j', override);
         basicsQuery = overrideBasicsQueryTpl({
             _query: _query,
             _column: _column,
@@ -175,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
             _column: _column
         });
 
-        if (override && _.has(override, 'bins')) {
+        if (this._shouldOverrideBins(override)) {
             binsQuery = [
                 overrideBinsQueryTpl({
                     _bins: override.bins
@@ -196,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) {
         }
     }
 
+    var cteSql = [
+        filteredQuery,
+        basicsQuery,
+        binsQuery,
+        nullsQueryTpl({
+            _query: _query,
+            _column: _column
+        })
+    ];
 
-    var histogramSql = [
-        "WITH",
-        [
-            basicsQuery,
-            binsQuery,
-            nullsQueryTpl({
+    if (this._columnType === 'float') {
+        cteSql.push(
+            infinitiesQueryTpl({
+                _query: _query,
+                _column: _column
+            }),
+            nansQueryTpl({
                 _query: _query,
                 _column: _column
             })
-        ].join(',\n'),
+        );
+    }
+
+    var histogramSql = [
+        "WITH",
+        cteSql.join(',\n'),
         histogramQueryTpl({
+            _isFloatColumn: this._columnType === 'float',
             _query: _query,
             _column: _column
         })
     ].join('\n');
 
-    return callback(null, histogramSql);
+    debug(histogramSql);
+
+    return histogramSql;
 };
+
+Histogram.prototype._shouldOverride = function (override) {
+    return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
+};
+
+Histogram.prototype._shouldOverrideBins = function (override) {
+    return override && _.has(override, 'bins');
+};
+
+

lib/cartodb/models/dataview/overviews/index.js

@@ -1,6 +1,5 @@
 module.exports = {
   Aggregation: require('./aggregation'),
   Formula: require('./formula'),
-  Histogram: require('./histogram'),
-  List: require('./list')
+  Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/overviews/list.js

@@ -1,11 +0,0 @@
-var BaseOverviewsDataview = require('./base');
-var BaseDataview = require('../list');
-
-function List(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
-}
-
-List.prototype = Object.create(BaseOverviewsDataview.prototype);
-List.prototype.constructor = List;
-
-module.exports = List;

lib/cartodb/models/filter/bbox.js

@@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([
 ].join('\n'));
 
 var bboxFilterTpl = dot.template(
-    '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'
+    'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))'
 );
 
 var LATITUDE_MAX_VALUE = 85.0511287798066;
@@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) {
         bboxes.push([west, south, east, north]);
     } else {
         bboxes.push([west, south, 180, north]);
-        bboxes.push([-180, south, east % 180, north]);
+        // here we assume west,east have been adjusted => west >= -180 => east > 180
+        bboxes.push([-180, south, east - 360, north]);
     }
 
     return bboxes;

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -115,6 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
                     }
                     layer.options.sql = analysisSql;
                     layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
+                    layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
                 } else {
                     missingNodesErrors.push(
                         new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@@ -330,4 +331,13 @@ function AnalysisError(message) {
     this.message = message;
 }
 
+function getAllAffectedTablesFromSourceNodes(node) {
+    var affectedTables = node.getAllInputNodes(function (node) {
+        return node.getType() === 'source';
+    }).reduce(function(list, node) {
+        return list.concat(node.getAffectedTables());
+    },[]);
+    return affectedTables;
+}
+
 require('util').inherits(AnalysisError, Error);

lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js

@@ -0,0 +1,25 @@
+function MapConfigBufferSizeAdapter() {
+    this.formats = ['png', 'png32', 'mvt', 'grid.json'];
+}
+
+module.exports = MapConfigBufferSizeAdapter;
+
+MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
+    if (!context.templateParams || !context.templateParams.buffersize) {
+        return callback(null, requestMapConfig);
+    }
+
+    this.formats.forEach(function (format) {
+        if (Number.isFinite(context.templateParams.buffersize[format])) {
+            if (requestMapConfig.buffersize === undefined) {
+                requestMapConfig.buffersize = {};
+            }
+
+            requestMapConfig.buffersize[format] = context.templateParams.buffersize[format];
+        }
+    });
+
+    setImmediate(function () {
+        callback(null, requestMapConfig);
+    });
+};

lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js

@@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC
 
                     if (nestedNamedLayers.length > 0) {
                         var nestedNamedMapsError = new Error('Nested named layers are not allowed');
-                        // nestedNamedMapsError.http_status = 400;
                         return done(nestedNamedMapsError);
                     }
 

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -33,7 +33,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf
                step(
                    function collectFiltersData() {
                        var filters, unfiltered_query;
-                       if ( layer.options.source && analysesResults ) {
+                       if ( layer.options.source && analysesResults && !layer.options.sql_wrap) {
                            var sourceId = layer.options.source.id;
                            var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
                            if ( node ) {

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -48,6 +48,11 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
         }
 
         requestMapConfig.layers = results.map(function(result) { return result.layer; });
+        context.turboCarto = {
+            layers: results.map(function(result) {
+                return result.meta;
+            })
+        };
 
         return callback(null, requestMapConfig);
     });
@@ -84,7 +89,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
     }
 
     var pg = new PSQL(dbParamsFromReqParams(params));
-    function processCallback(err, cartocss) {
+    function processCallback(err, cartocss, meta) {
         // Only return turbo-carto errors
         if (err && err.name === 'TurboCartoError') {
             var error = new Error(err.message);
@@ -105,7 +110,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
         if (cartocss) {
             layer.options.cartocss = cartocss;
         }
-        return callback(null, { layer: layer });
+        return callback(null, { layer: layer, meta: meta });
     }
 
     var layerSql = layer.options.sql;

lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js

@@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
     var context = {};
     step(
         function prepareContextLimits() {
-            self.userLimitsApi.getRenderLimits(self.user, this);
+            self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
         },
         function handleRenderLimits(err, renderLimits) {
             assert.ifError(err);

lib/cartodb/models/mapconfig/provider/map-store-provider.js

@@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
     var context = {};
     step(
         function prepareContextLimits() {
-            self.userLimitsApi.getRenderLimits(self.user, this);
+            self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
         },
         function handleRenderLimits(err, renderLimits) {
             assert.ifError(err);

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -90,6 +90,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
         },
         function instantiateTemplate(err, templateParams) {
             assert.ifError(err);
+            context.templateParams = templateParams;
             return self.templateMaps.instance(self.template, templateParams);
         },
         function prepareAdapterMapConfig(err, requestMapConfig) {
@@ -113,13 +114,14 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
         function prepareContextLimits(err, _mapConfig) {
             assert.ifError(err);
             mapConfig = _mapConfig;
-            self.userLimitsApi.getRenderLimits(self.owner, this);
+            self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
         },
         function cacheAndReturnMapConfig(err, renderLimits) {
             self.err = err;
             self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
             self.analysesResults = context.analysesResults || [];
             self.rendererParams = rendererParams;
+            self.context = context;
             self.context.limits = renderLimits || {};
             return callback(self.err, self.mapConfig, self.rendererParams, self.context);
         }

lib/cartodb/models/resource-locator.js

@@ -0,0 +1,119 @@
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function ResourceLocator(environment) {
+    this.environment = environment;
+
+    this.resourcesUrlTemplates = null;
+    if (this.environment.resources_url_templates) {
+        var templates = environment.resources_url_templates;
+
+        if (templates.http) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
+        }
+        if (templates.https) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
+        }
+    }
+}
+
+module.exports = ResourceLocator;
+
+ResourceLocator.prototype.getUrls = function(username, resource) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, resource);
+    }
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource);
+    if (cdnDomain) {
+        return {
+            http: 'http://' + cdnDomain.http + '/' + username + '/api/v1/map/' + resource,
+            https: 'https://' + cdnDomain.https + '/' + username + '/api/v1/map/' + resource
+        };
+    } else {
+        var port = this.environment.port;
+        return {
+            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
+        };
+    }
+};
+
+
+ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource) {
+    var urls = {};
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = this.resourcesUrlTemplates.http({
+            cdn_url: cdnDomain.http,
+            user: username,
+            port: this.environment.port,
+            resource: resource
+        });
+    }
+
+    if (this.resourcesUrlTemplates.https) {
+        urls.https = this.resourcesUrlTemplates.https({
+            cdn_url: cdnDomain.https,
+            user: username,
+            port: this.environment.port,
+            resource: resource
+        });
+    }
+
+    return urls;
+};
+
+function getCdnDomain(serverMetadata, resource) {
+    if (serverMetadata && serverMetadata.cdn_url) {
+        var cdnUrl = serverMetadata.cdn_url;
+        var http = cdnUrl.http;
+        var https = cdnUrl.https;
+        if (cdnUrl.templates) {
+            var templates = cdnUrl.templates;
+            var httpUrlTemplate = templates.http.url;
+            var httpsUrlTemplate = templates.https.url;
+            http = httpUrlTemplate
+                .replace(/^(http[s]*:\/\/)/, '')
+                .replace('{s}', subdomain(templates.http.subdomains, resource));
+            https = httpsUrlTemplate
+                .replace(/^(http[s]*:\/\/)/, '')
+                .replace('{s}', subdomain(templates.https.subdomains, resource));
+        }
+        return {
+            http: http,
+            https: https,
+        };
+    }
+    return null;
+}
+
+// ref https://jsperf.com/js-crc32
+function crcTable() {
+    var c;
+    var table = [];
+    for (var n = 0; n < 256; n++) {
+      c = n;
+      for (var k = 0; k < 8; k++) {
+        c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+      }
+      table[n] = c;
+    }
+    return table;
+}
+var CRC_TABLE = crcTable();
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+      crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return (crc ^ (-1)) >>> 0;
+}
+
+function subdomain(subdomains, resource) {
+    var index = crc32(resource) % subdomains.length;
+    return subdomains[index];
+}
+module.exports.subdomain = subdomain;

lib/cartodb/server.js

@@ -12,7 +12,8 @@ var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
 var FastlyCacheBackend = require('./cache/backend/fastly');
 
 var StatsClient = require('./stats/client');
-var Profiler = require('./stats/profiler_proxy');
+const stats = require('./middleware/stats');
+
 var RendererStatsReporter = require('./stats/reporter/renderer');
 
 var windshaft = require('windshaft');
@@ -35,12 +36,20 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin
 
 var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
 var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
+var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
 var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
 var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
 var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
 var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
 var MapConfigAdapter = require('./models/mapconfig/adapter');
 
+var StatsBackend = require('./backends/stats');
+
+const lzmaMiddleware = require('./middleware/lzma');
+const errorMiddleware = require('./middleware/error-middleware');
+
+const prepareContextMiddleware = require('./middleware/context');
+
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
     global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@@ -115,8 +124,27 @@ module.exports = function(serverOptions) {
     var onTileErrorStrategy;
     if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
         onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
-            if (err && err.message === 'Render timed out' && format === 'png') {
-                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
+
+            function isRenderTimeoutError (err) {
+                return err.message === 'Render timed out';
+            }
+
+            function isDatasourceTimeoutError (err) {
+                return err.message && err.message.match(/canceling statement due to statement timeout/i);
+            }
+
+            function isTimeoutError (err) {
+                return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+            }
+
+            function isRasterFormat (format) {
+                return format === 'png' || format === 'jpg';
+            }
+
+            if (isTimeoutError(err) && isRasterFormat(format)) {
+                return callback(null, timeoutErrorTile, {
+                    'Content-Type': 'image/png',
+                }, {});
             } else {
                 return callback(err, tile, headers, stats);
             }
@@ -130,7 +158,8 @@ module.exports = function(serverOptions) {
             grainstore: serverOptions.grainstore,
             mapnik: serverOptions.renderer.mapnik
         },
-        http: serverOptions.renderer.http
+        http: serverOptions.renderer.http,
+        mvt: serverOptions.renderer.mvt
     });
 
     // initialize render cache
@@ -147,13 +176,17 @@ module.exports = function(serverOptions) {
     var tileBackend = new windshaft.backend.Tile(rendererCache);
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
+
+    var statsBackend = new StatsBackend();
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
     var mapConfigAdapter = new MapConfigAdapter(
         new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
+        new MapConfigBufferSizeAdapter(),
         new SqlWrapMapConfigAdapter(),
         new DataviewsWidgetsAdapter(),
         new AnalysisMapConfigAdapter(analysisBackend),
@@ -180,12 +213,16 @@ module.exports = function(serverOptions) {
 
     var versions = getAndValidateVersions(serverOptions);
 
+    const prepareContext = typeof serverOptions.req2params === 'function' ?
+        serverOptions.req2params :
+        prepareContextMiddleware(authApi, pgConnection);
+
     /*******************************************************************************************************************
      * Routing
      ******************************************************************************************************************/
 
     new controller.Layergroup(
-        authApi,
+        prepareContext,
         pgConnection,
         mapStore,
         tileBackend,
@@ -198,7 +235,7 @@ module.exports = function(serverOptions) {
     ).register(app);
 
     new controller.Map(
-        authApi,
+        prepareContext,
         pgConnection,
         templateMaps,
         mapBackend,
@@ -206,12 +243,12 @@ module.exports = function(serverOptions) {
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache,
-        mapConfigAdapter
+        mapConfigAdapter,
+        statsBackend
     ).register(app);
 
     new controller.NamedMaps(
-        authApi,
-        pgConnection,
+        prepareContext,
         namedMapProviderCache,
         tileBackend,
         previewBackend,
@@ -220,9 +257,9 @@ module.exports = function(serverOptions) {
         metadataBackend
     ).register(app);
 
-    new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
+    new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
 
-    new controller.Analyses(authApi, pgConnection).register(app);
+    new controller.Analyses(prepareContext).register(app);
 
     new controller.ServerInfo(versions).register(app);
 
@@ -230,6 +267,8 @@ module.exports = function(serverOptions) {
      * END Routing
      ******************************************************************************************************************/
 
+    app.use(errorMiddleware());
+
     return app;
 };
 
@@ -302,15 +341,28 @@ function bootstrap(opts) {
     app.enable('jsonp callback');
     app.disable('x-powered-by');
     app.disable('etag');
+
+    // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
+    // See: http://expressjs.com/en/4x/api.html#app.set
+    app.set('json replacer', function (key, value) {
+        if (value !== value) {
+            return 'NaN';
+        }
+
+        if (value === Infinity) {
+            return 'Infinity';
+        }
+
+        if (value === -Infinity) {
+            return '-Infinity';
+        }
+
+        return value;
+    });
+
     app.use(bodyParser.json());
 
     app.use(function bootstrap$prepareRequestResponse(req, res, next) {
-        req.context = req.context || {};
-        req.profiler = new Profiler({
-            statsd_client: global.statsClient,
-            profile: opts.useProfiler
-        });
-
         if (global.environment && global.environment.api_hostname) {
             res.set('X-Served-By-Host', global.environment.api_hostname);
         }
@@ -318,6 +370,13 @@ function bootstrap(opts) {
         next();
     });
 
+    app.use(stats({
+        enabled: opts.useProfiler,
+        statsClient: global.statsClient
+    }));
+
+    app.use(lzmaMiddleware);
+
     // temporary measure until we upgrade to newer version expressjs so we can check err.status
     app.use(function(err, req, res, next) {
         if (err) {

lib/cartodb/server_options.js

@@ -36,7 +36,11 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
         inlineExecution: false,
         endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
         hostHeaderTemplate: '{{=it.username}}.localhost.lan'
-    }
+    },
+    logger: {
+        filename: undefined
+    },
+    limits: {}
 });
 
 module.exports = {
@@ -66,6 +70,7 @@ module.exports = {
         },
         datasource: global.environment.postgres,
         cachedir: global.environment.millstone.cache_basedir,
+        use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
         mapnik_version: global.environment.mapnik_version,
         mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
         default_layergroup_ttl: global.environment.mapConfigTTL || 7200
@@ -76,6 +81,7 @@ module.exports = {
         statsInterval: rendererConfig.statsInterval
     },
     renderer: {
+        mvt: rendererConfig.mvt,
         mapnik: _.defaults(rendererConfig.mapnik, {
             geojson: {
                 dbPoolParams: {
@@ -95,7 +101,11 @@ module.exports = {
             inlineExecution: analysisConfig.batch.inlineExecution,
             endpoint: analysisConfig.batch.endpoint,
             hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate
-        }
+        },
+        logger: {
+            filename: analysisConfig.logger.filename
+        },
+        limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

lib/cartodb/utils/query-utils.js

@@ -0,0 +1,26 @@
+function prepareQuery(sql) {
+  var affectedTableRegexCache = {
+      bbox: /!bbox!/g,
+      scale_denominator: /!scale_denominator!/g,
+      pixel_width: /!pixel_width!/g,
+      pixel_height: /!pixel_height!/g
+  };
+
+  return sql
+      .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+      .replace(affectedTableRegexCache.scale_denominator, '0')
+      .replace(affectedTableRegexCache.pixel_width, '1')
+      .replace(affectedTableRegexCache.pixel_height, '1');
+}
+
+module.exports.extractTableNames = function extractTableNames(query) {
+    return [
+        'SELECT * FROM CDB_QueryTablesText($windshaft$',
+        prepareQuery(query),
+        '$windshaft$) as tablenames'
+    ].join('');
+};
+
+module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
+    return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
+};

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.66.2",
+  "version": "4.2.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -16,46 +16,59 @@
   "contributors": [
     "Simon Tokumine <simon@vizzuality.com>",
     "Javi Santana <jsantana@vizzuality.com>",
-    "Sandro Santilli <strk@vizzuality.com>"
+    "Sandro Santilli <strk@vizzuality.com>",
+    "Carlos Matallín <matallo@carto.com>",
+    "Daniel Garcia Aubert <dgaubert@carto.com>",
+    "Mario de Frutos <mario.defrutos@carto.com>"
   ],
   "dependencies": {
-    "body-parser": "~1.14.0",
-    "camshaft": "0.37.1",
-    "cartodb-psql": "~0.6.1",
-    "cartodb-query-tables": "~0.1.0",
-    "cartodb-redis": "0.13.1",
-    "debug": "~2.2.0",
+    "body-parser": "^1.18.2",
+    "camshaft": "0.59.4",
+    "cartodb-psql": "0.10.2",
+    "cartodb-query-tables": "0.3.0",
+    "cartodb-redis": "0.14.0",
+    "debug": "^3.1.0",
     "dot": "~1.0.2",
-    "express": "~4.13.3",
+    "express": "~4.16.0",
     "fastly-purge": "~1.0.1",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "2.6.5",
     "lzma": "~2.3.2",
     "node-statsd": "~0.0.7",
+    "on-headers": "^1.0.1",
     "queue-async": "~1.0.7",
-    "redis-mpool": "~0.4.0",
-    "request": "~2.62.0",
+    "redis-mpool": "0.4.1",
+    "request": "^2.83.0",
+    "semver": "~5.3.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.15.1",
+    "turbo-carto": "0.20.2",
     "underscore": "~1.6.0",
-    "windshaft": "2.4.0"
+    "windshaft": "4.1.0",
+    "yargs": "~5.0.0"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",
-    "jshint": "~2.6.0",
-    "mocha": "~1.21.4",
+    "jshint": "~2.9.4",
+    "mocha": "~3.4.1",
+    "moment": "~2.18.1",
     "nock": "~2.11.0",
-    "redis": "~0.8.6",
-    "semver": "~1.1.4",
+    "redis": "~0.12.1",
     "strftime": "~0.8.2"
   },
   "scripts": {
+    "lint": "jshint lib test",
     "preinstall": "make pre-install",
-    "test": "make test-all"
+    "test": "make test-all",
+    "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn",
+    "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)",
+    "docker-pull": "docker pull cartoimages/windshaft-testing",
+    "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh",
+    "docker-bash": "docker run  -it -v `pwd`:/srv cartoimages/windshaft-testing bash",
+    "docker-publish": "docker push cartoimages/windshaft-carto-testing"
   },
   "engines": {
-    "node": ">=0.8 <0.11",
-    "npm": ">=2.14.16"
+    "node": ">=6.9",
+    "yarn": ">=0.27.5 <1.0.0"
   }
 }

run_tests.sh

@@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then
   ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS}
 else
   echo "Running tests"
-  mocha -u tdd -t 5000 ${TESTS}
+  ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS}
 fi
 ret=$?
 

scripts/install.sh

@@ -4,4 +4,4 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
     export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig
 fi
 
-npm install
+yarn

scripts/mvt-timeout-error.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import mapbox_vector_tile
+
+lines_list = []
+
+# main diagonal line
+lines_list.append({ "geometry":"LINESTRING (0 0, 4096 4096)"})
+
+# diagonal lines 
+for i in range(4096/32, 4096, 4096/32):
+    start = i
+    end = 4096 - i
+    
+    lines_list.append({ "geometry":"LINESTRING (0 " + str(start) + ", " + str(end) + " 4096)" })
+    lines_list.append({ "geometry":"LINESTRING (" + str(start) + " 0, 4096 " + str(end) + ")" })
+
+# box lines
+lines_list.append({ "geometry":"LINESTRING (0 0, 0 4096)"})
+lines_list.append({ "geometry":"LINESTRING (0 4096, 4096 4096)"})
+lines_list.append({ "geometry":"LINESTRING (4096 4096, 4096 0)"})
+lines_list.append({ "geometry":"LINESTRING (4096 0, 0 0)"})
+
+
+tile = mapbox_vector_tile.encode([
+    {
+        "name": "errorTileSquareLayer",
+        "features": [{ "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))" }]
+    },
+    {
+        "name": "errorTileStripesLayer",
+        "features": lines_list
+    }
+])
+
+with open('./assets/render-timeout-fallback.mvt', 'w+') as f:
+    f.write(tile)

test/acceptance/analysis/analyses-controller.js

@@ -0,0 +1,114 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+
+describe('analyses controller', function () {
+    const mapConfig = {
+        version: '1.5.0',
+        layers:
+            [{
+                type: 'cartodb',
+                options:
+                    {
+                        source: { id: 'a1' },
+                        cartocss: TestClient.CARTOCSS.POLYGONS,
+                        cartocss_version: '2.3.0'
+                    }
+            }],
+        dataviews: {},
+        analyses:
+            [{
+                id: 'a1',
+                type: 'buffer',
+                params: {
+                    source: {
+                        type: 'source',
+                        params: {
+                            query: 'select * from analysis_banks limit 1'
+                        }
+                    },
+                    radius: 250
+                }
+            }]
+    };
+
+    beforeEach(function () {
+        this.testClient = new TestClient(mapConfig, 1234);
+    });
+
+    it('should get an array of analyses from catalog', function (done) {
+        this.testClient.getAnalysesCatalog({}, (err, result) => {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(Array.isArray(result.catalog));
+            done();
+        });
+    });
+
+    it('should support jsonp responses', function (done) {
+        this.testClient.getAnalysesCatalog({ jsonp: 'jsonp_test' }, (err, result) => {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(result);
+
+            let didRunJsonCallback = false;
+            // jshint ignore:start
+            function jsonp_test(body) {
+                assert.ok(Array.isArray(body.catalog));
+                didRunJsonCallback = true;
+            }
+
+            eval(result);
+            // jshint ignore:end
+
+            assert.ok(didRunJsonCallback);
+
+            done();
+        });
+    });
+
+    it('should respond "unauthorized" when missing api_key', function (done) {
+        const apiKey = this.testClient.apiKey;
+        this.testClient.apiKey = null;
+
+        this.testClient.getAnalysesCatalog({ status: 401 }, (err, result) => {
+            if (err) {
+                return done(err);
+            }
+
+            assert.deepEqual(result.errors[0], 'Unauthorized');
+            this.testClient.apiKey = apiKey;
+            done();
+        });
+    });
+
+    it('should get an array of analyses from catalog', function (done) {
+        this.testClient.getTile(0, 0, 0, (err) => {
+            if (err) {
+                return done(err);
+            }
+
+            this.testClient.getAnalysesCatalog({}, (err, result) => {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.ok(Array.isArray(result.catalog));
+                assert.ok(result.catalog.length >=  2); // buffer & source at least
+
+                result.catalog
+                    .filter(analysis => analysis.node_id === '0a215e1f3405381cf0ea6b3b0deb6fdcfdc2fcaa')
+                    .forEach(analysis => assert.equal(analysis.type, 'buffer'));
+
+                this.testClient.drain(done);
+            });
+
+        });
+    });
+});

test/acceptance/analysis/analysis-layers-geojson.js

@@ -1,81 +0,0 @@
-require('../../support/test_helper');
-
-var assert = require('../../support/assert');
-var TestClient = require('../../support/test-client');
-
-describe('analysis-layers-dataviews-geojson', function() {
-
-    function createMapConfig(layers, dataviews, analysis) {
-        return {
-            version: '1.5.0',
-            layers: layers,
-            dataviews: dataviews || {},
-            analyses: analysis || []
-        };
-    }
-
-    var CARTOCSS = [
-        "#points {",
-        "  marker-fill-opacity: 1.0;",
-        "  marker-line-color: #FFF;",
-        "  marker-line-width: 0.5;",
-        "  marker-line-opacity: 1.0;",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  marker-width: 8;",
-        "  marker-fill: red;",
-        "  marker-allow-overlap: true;",
-        "}"
-    ].join('\n');
-
-    var mapConfig = createMapConfig(
-        [
-            {
-                "type": "cartodb",
-                "options": {
-                    "source": {
-                        "id": "2570e105-7b37-40d2-bdf4-1af889598745"
-                    },
-                    "cartocss": CARTOCSS,
-                    "cartocss_version": "2.3.0"
-                }
-            }
-        ],
-        {
-            pop_max_histogram: {
-                source: {
-                    id: '2570e105-7b37-40d2-bdf4-1af889598745'
-                },
-                type: 'histogram',
-                options: {
-                    column: 'pop_max'
-                }
-            }
-        },
-        [
-            {
-                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
-                "type": "source",
-                "params": {
-                    "query": "select * from populated_places_simple_reduced"
-                }
-            }
-        ]
-    );
-
-    it('should get pop_max column from dataview', function(done) {
-        var testClient = new TestClient(mapConfig, 1234);
-
-        testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(Array.isArray(geojson.features));
-            assert.ok(geojson.features.length > 0);
-            var feature = geojson.features[0];
-            assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
-
-            testClient.drain(done);
-        });
-    });
-
-});

test/acceptance/analysis/analysis-layers-use-cases.js

@@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client');
 var dot = require('dot');
 var debug = require('debug')('windshaft:cartodb:test');
 
-describe('analysis-layers use cases', function() {
+describe('analysis-layers use cases', function () {
 
 
-    var multitypeStyleTemplate = dot.template([
-        "#points['mapnik::geometry_type'=1] {",
-        "  marker-fill-opacity: {{=it._opacity}};",
-        "  marker-line-color: #FFF;",
-        "  marker-line-width: 0.5;",
-        "  marker-line-opacity: {{=it._opacity}};",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  marker-width: 8;",
-        "  marker-fill: {{=it._color}};",
-        "  marker-allow-overlap: true;",
-        "}",
-        "#lines['mapnik::geometry_type'=2] {",
-        "  line-color: {{=it._color}};",
-        "  line-width: 2;",
-        "  line-opacity: {{=it._opacity}};",
-        "}",
-        "#polygons['mapnik::geometry_type'=3] {",
-        "  polygon-fill: {{=it._color}};",
-        "  polygon-opacity: {{=it._opacity}};",
-        "  line-color: #FFF;",
-        "  line-width: 0.5;",
-        "  line-opacity: {{=it._opacity}};",
-        "}"
-    ].join('\n'));
+    var multitypeStyleTemplate = dot.template(
+        `#points['mapnik::geometry_type'=1] {
+          marker-fill-opacity: {{=it._opacity}};
+          marker-line-color: #FFF;
+          marker-line-width: 0.5;
+          marker-line-opacity: {{=it._opacity}};
+          marker-placement: point;
+          marker-type: ellipse;
+          marker-width: 8;
+          marker-fill: {{=it._color}};
+          marker-allow-overlap: true;
+        }
+        #lines['mapnik::geometry_type'=2] {
+          line-color: {{=it._color}};
+          line-width: 2;
+          line-opacity: {{=it._opacity}};
+        }
+        #polygons['mapnik::geometry_type'=3] {
+          polygon-fill: {{=it._color}};
+          polygon-opacity: {{=it._opacity}};
+          line-color: #FFF;
+          line-width: 0.5;
+          line-opacity: {{=it._opacity}};
+        }`
+    );
 
 
     function cartocss(color, opacity) {
@@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() {
             version: '1.5.0',
             layers: layers,
             dataviews: dataviews || {},
-            analysis: analysis || []
+            analyses: analysis || []
         };
     }
 
-    function analysisDef(analysis) {
-        return JSON.stringify(analysis);
-    }
-
     var DEFAULT_MULTITYPE_STYLE = cartocss();
 
     var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 };
 
+    var pointInPolygonDef = {
+        id: 'a1',
+        type: 'point-in-polygon',
+        params: {
+            points_source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_rent_listings'
+                }
+            },
+            polygons_source: {
+                type: 'buffer',
+                params: {
+                    source: {
+                        type: 'source',
+                        params: {
+                            query: 'select * from analysis_banks'
+                        }
+                    },
+                    radius: 250
+                }
+            }
+        }
+    };
+
+    var bufferDef = {
+        id: 'b1',
+        type: 'buffer',
+        params: {
+            source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_banks'
+                }
+            },
+            radius: 250
+        }
+    };
+
     var useCases = [
         {
             desc: '1 mapnik layer',
@@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() {
                     {
                         type: 'cartodb',
                         options: {
-                            sql: "select * from analysis_rent_listings",
+                            sql: 'select * from analysis_rent_listings',
                             cartocss: DEFAULT_MULTITYPE_STYLE,
                             cartocss_version: '2.3.0'
                         }
@@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_banks",
+                        sql: 'select * from analysis_banks',
                         cartocss: cartocss('#2167AB'),
                         cartocss_version: '2.3.0'
                     }
@@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
@@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 250
-                            }
-                        }),
-                        cartocss: cartocss('black', 0.5)
+                        source: {
+                            id: 'b1'
+                        },
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
+                        cartocss_version: '2.3.0'
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    bufferDef
+                ]
+            )
         },
 
         {
@@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
+                        source: {
+                            id: 'a1'
+                        },
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
+                        cartocss_version: '2.3.0'
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    pointInPolygonDef
+                ]
+            )
         },
 
         {
             desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                }
-            ])
-        },
-
-        {
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 300
-                            }
-                        }),
-                        cartocss: cartocss('magenta', 0.5)
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 300
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b1" },
-                        "cartocss": cartocss('green', 1.0),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b2" },
-                        "cartocss": cartocss('magenta', 0.5),
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            [
-                {
-                    id: "b2",
-                    options: {
-                        def: analysisDef({
-                            "type": "count-in-polygon",
-                            "id": "a0",
-                            "params": {
-                                "columnName": 'count_airbnb',
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        query: "select * from analysis_rent_listings"
-                                    },
-                                    dataviews: {
-                                        price_histogram: {
-                                            type: 'histogram',
-                                            options: {
-                                                column: 'price'
-                                            }
-                                        }
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "id": "b1",
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "id": "b0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    },
-                                    dataviews: {
-                                        bank_category: {
-                                            type: 'aggregation',
-                                            options: {
-                                                column: 'bank'
-                                            }
-                                        }
-                                    }
-                                }
-                            },
-                            dataviews: {
-                                count_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'count_airbnb'
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'I. Distribution centers',
             mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "b0" },
-                            "cartocss": [
-                                "#distribution_centers {",
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: blue;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            source: {
+                                id: 'a1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     },
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a0" },
-                            "cartocss": [
-                                "#shops {",
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: red;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
-                        }
-                    },
-                    {
-                        type: 'cartodb',
-                        options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#routing {",
-                                "  line-color: ramp([routing_time], colorbrewer(Reds));",
-                                "  line-width: ramp([routing_time], 2, 8);",
-                                "  line-opacity: 1.0;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            sql: 'select * from analysis_rent_listings',
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
-                {
-                    distribution_center_name_category: {
-                        source: { id: 'b0' },
-                        type: 'aggregation',
-                        options: {
-                            column: 'name'
-                        }
-                    },
-                    time_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_time'
-                        }
-                    },
-                    distance_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_distance'
-                        }
-                    }
-                },
-                // analysis
+                {},
                 [
-                    {
-                        id: 'a1',
-                        type: 'routing-n-to-n',
-                        params: {
-                            // distanceColumn: 'routing_distance',
-                            // timeColumn: 'routing_time',
-                            originSource: {
-                                id: 'b0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from distribution_centers'
-                                }
-                            },
-                            destinationSource: {
-                                id: 'a0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from shops'
-                                }
-                            }
-                        }
-                    }
+                    pointInPolygonDef
                 ]
             )
         },
 
         {
-            skip: true,
-            desc: 'II. Population analysis',
+            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
             mapConfig: mapConfig(
-            // layers
-            [
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a2" },
-                        "cartocss": [
-                            "#count_in_polygon {",
-                            "  polygon-opacity: 1.0",
-                            "  line-color: #FFF;",
-                            "  line-width: 0.5;",
-                            "  line-opacity: 0.7",
-                            "  polygon-fill: ramp([estimated_people], colorbrewer(Reds));",
-                            "}"
-                        ].join('\n'),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a0" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            // dataviews
-            {
-                total_population_formula: {
-                    "source": { id: "a3" },
-                    type: 'formula',
-                    options: {
-                        column: 'total_population',
-                        operation: 'sum'
-                    }
-                },
-                people_histogram: { // this injects a range filter at `a2` node output
-                    "source": { id: "a2" },
-                    type: 'histogram',
-                    options: {
-                        column: 'estimated_people'
-                    }
-                },
-                subway_line_category: { // this injects a category filter at `a0` node output
-                    "source": { id: "a0" },
-                    type: 'aggregation',
-                    options: {
-                        column: 'subway_line'
-                    }
-                }
-            },
-            // analysis
-            [
-                {
-                    id: 'a3',
-                    // this will union the polygons, produce just one polygon, and calculate the total population for it
-                    type: 'total-population',
-                    params: {
-                        columnName: 'total_population',
-                        source: {
-                            id: 'a2',
-                            type: 'estimated-population',
-                            params: {
-                                columnName: 'estimated_people',
-                                source: {
-                                    id: 'a1',
-                                    type: 'trade-area',
-                                    params: {
-                                        source: {
-                                            "id": "a0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from subway_stops"
-                                            }
-                                        },
-                                        kind: 'walk',
-                                        time: 300
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'III. Point in polygon',
-            mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#count_in_polygon {",
-                                "  polygon-opacity: 1.0",
-                                "  line-color: #FFF;",
-                                "  line-width: 0.5;",
-                                "  line-opacity: 0.7",
-                                "  polygon-fill: ramp([count_people], colorbrewer(Reds));",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            sql: 'select * from analysis_rent_listings',
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'a1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'b1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
-                {
-                    age_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'age'
-                        }
-                    },
-                    income_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'income'
-                        }
-                    },
-                    gender_category: {
-                        "source": { id: "a0" },
-                        type: 'aggregation',
-                        options: {
-                            column: 'gender'
-                        }
-                    }
-                },
-                // analysis
+                {},
                 [
-                    {
-                        "id": "a1",
-                        "type": "count-in-polygon",
-                        "params": {
-                            "columnName": 'count_people',
-                            "pointsSource": {
-                                "id": 'a0',
-                                "type": "source",
-                                "params": {
-                                    query: "select the_geom, age, gender, income from people"
-                                }
-                            },
-                            "polygonsSource": {
-                                "id": "b0",
-                                "type": "source",
-                                "params": {
-                                    query: "select * from postal_codes"
-                                }
-                            }
-                        }
-                    }
+                    bufferDef,
+                    pointInPolygonDef
                 ]
             )
         }
-
     ];
 
-    useCases.forEach(function(useCase, imageIdx) {
+    useCases.forEach(function (useCase) {
         if (!!useCase.skip) {
-            debug(JSON.stringify(useCase.mapConfig, null, 4));
+            return debug(JSON.stringify(useCase.mapConfig, null, 4));
         }
-        it.skip('should implement use case: "' + useCase.desc + '"', function(done) {
+        it(`should implement use case: '${useCase.desc}'`, function (done) {
 
             var testClient = new TestClient(useCase.mapConfig, 1234);
 
             var tile = useCase.tile || TILE_ANALYSIS_TABLES;
 
-            testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) {
+            testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) {
                 assert.ok(!err, err);
 
-                image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
+                //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
 
                 assert.equal(image.width(), 256);
 

test/acceptance/analysis/error-cases.js

@@ -373,5 +373,70 @@ describe('analysis-layers error cases', function() {
         });
     });
 
+    it('should return "function does not exist" indicating the node_id and context', function(done) {
+        var mapConfig = createMapConfig([{
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": '#polygons { polygon-fill: red; }',
+                "cartocss_version": "2.3.0"
+            }
+        }], {}, [{
+            "id": "HEAD",
+            "type": "buffer",
+            "params": {
+                "source": {
+                    "id": "HEAD2",
+                    "type": "buffer",
+                    "params": {
+                        "source": {
+                            "id": "HEAD3",
+                            "type": 'deprecated-sql-function',
+                            "params": {
+                                "id": "HEAD4",
+                                "function_name": 'DEP_EXT_does_not_exist_fn',
+                                "primary_source": {
+                                    "type": 'source',
+                                    "params": {
+                                        "query": "select * from populated_places_simple_reduced"
+                                    }
+                                },
+                                "function_args": ['wadus']
+                            }
+                        },
+                        "radius": 10
+                    }
+                },
+                "radius": 10
+            }
+        }]);
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(
+                layergroupResult.errors[0],
+                'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist'
+            );
+
+            assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
+            assert.equal(
+                layergroupResult.errors_with_context[0].message,
+                'function dep_ext_does_not_exist_fn(unknown, unknown, unknown, text[], unknown) does not exist'
+            );
+            assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD3');
+
+            testClient.drain(done);
+        });
+    });
+
+
 
 });

test/acceptance/analysis/named-maps.js

@@ -7,7 +7,7 @@ var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 var TestClient = require('../../support/test-client');
 
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('named-maps analysis', function() {
 

test/acceptance/buffer-size-format.js

@@ -0,0 +1,473 @@
+require('../support/test_helper');
+
+var fs = require('fs');
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+var serverOptions = require('../../lib/cartodb/server_options');
+var mapnik = require('windshaft').mapnik;
+var IMAGE_TOLERANCE_PER_MIL = 5;
+
+var CARTOCSS_LABELS = [
+    '#layer {',
+    '  polygon-fill: #374C70;',
+    '  polygon-opacity: 0.9;',
+    '  line-width: 1;',
+    '  line-color: #FFF;',
+    '  line-opacity: 0.5;',
+    '}',
+    '#layer::labels {',
+    '  text-name: [name];',
+    '  text-face-name: \'DejaVu Sans Book\';',
+    '  text-size: 20;',
+    '  text-fill: #FFFFFF;',
+    '  text-label-position-tolerance: 0;',
+    '  text-halo-radius: 1;',
+    '  text-halo-fill: #6F808D;',
+    '  text-dy: -10;',
+    '  text-allow-overlap: true;',
+    '  text-placement: point;',
+    '  text-placement-type: dummy;',
+    '}'
+].join('\n');
+
+function createMapConfig (bufferSize, cartocss) {
+    cartocss = cartocss || CARTOCSS_LABELS;
+
+    return {
+        version: '1.6.0',
+        buffersize: bufferSize,
+        layers: [{
+            type: "cartodb",
+            options: {
+                sql: [
+                    'select',
+                    '   *',
+                    'from',
+                    '   populated_places_simple_reduced',
+                ].join('\n'),
+                cartocss: cartocss,
+                cartocss_version: '2.3.0',
+                interactivity: 'cartodb_id'
+            }
+        }]
+    };
+}
+
+describe('buffer size per format', function () {
+    var testCases = [
+        {
+            desc: 'should get png tile using buffer-size 0',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 0',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt',
+            mapConfig: createMapConfig({ mvt: 0 }),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+                assert.equal(features.length, 1);
+                callback();
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 128',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt',
+            mapConfig: createMapConfig({ mvt: 128 }),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+                assert.equal(features.length, 9);
+                callback();
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
+            mapConfig: createMapConfig({ 'grid.json': 0 }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
+            mapConfig: createMapConfig({ 'grid.json': 128 }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
+            }
+        }
+    ];
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
+    testCases.forEach(function (test) {
+        var testFn = (usePostGIS) => {
+            it(test.desc, function (done) {
+                serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                this.testClient = new TestClient(test.mapConfig, 1234);
+                serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+                var coords = test.coords;
+                var options = {
+                    format: test.format,
+                    layers: test.layers
+                };
+                this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                    assert.ifError(err);
+                    // To generate images use:
+                    // tile.save(test.fixturePath);
+                    test.assert(tile, done);
+                });
+            });
+        };
+        if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
+            testFn(true);
+        }
+        testFn(false);
+    });
+});
+
+function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) {
+    cartocss = cartocss || CARTOCSS_LABELS;
+
+    return {
+        "version": "0.0.1",
+        "name": name,
+        "placeholders": placeholders || {
+            "buffersize": {
+                "type": "number",
+                "default": 0
+            }
+        },
+        "layergroup": createMapConfig(buffersize)
+    };
+}
+
+describe('buffer size per format for named maps', function () {
+    var testCases = [
+        {
+            desc: 'should get png tile using buffer-size 0 (default value in template)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128 (placehoder value)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: { buffersize: 128 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 0 (default value in template by format)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: { buffersize_png: 0 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-default-buffer-size-by-format', {
+                png: '<%= buffersize_png %>'
+            }, {
+                "buffersize_png": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: { buffersize_png: 128 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            template: createBufferSizeTemplate('named-custom-buffer-size-by-format', {
+                png: '<%= buffersize_png %>'
+            }, {
+                "buffersize_png": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: { buffersize_gridjson: 0 },
+            fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
+            template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', {
+                'grid.json': '<%= buffersize_gridjson %>'
+            }, {
+                "buffersize_gridjson": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: { buffersize_gridjson: 128 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
+            template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', {
+                'grid.json': '<%= buffersize_gridjson %>'
+            }, {
+                "buffersize_gridjson": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
+            }
+        }
+    ];
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    testCases.forEach(function (test) {
+        it(test.desc, function (done) {
+            this.testClient = new TestClient(test.template, 1234);
+            var coords = test.coords;
+            var options = {
+                format: test.format,
+                placeholders: test.placeholders,
+                layers: test.layers
+            };
+            this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                assert.ifError(err);
+                // To generate images use:
+                //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
+                test.assert(tile, done);
+            });
+        });
+    });
+});
+
+
+describe('buffer size per format for named maps w/o placeholders', function () {
+    var testCases = [
+        {
+            desc: 'should get png tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: {
+                buffersize: {
+                    png: 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: {
+                buffersize: {
+                    png: 128
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            placeholders: {
+                buffersize: {
+                    mvt: 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt',
+            template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+
+                var dataFixture = fs.readFileSync(this.fixturePath);
+                var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
+                vtile.setDataSync(dataFixture);
+                var vtileJSON = vtile.toJSON();
+                var vtileFeatures = vtileJSON[0].features;
+
+                assert.equal(features.length, vtileFeatures.length);
+                callback();
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            placeholders: {
+                buffersize: {
+                    mvt: 128
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt',
+            template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+
+                var dataFixture = fs.readFileSync(this.fixturePath);
+                var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
+                vtile.setDataSync(dataFixture);
+                var vtileJSON = vtile.toJSON();
+                var vtileFeatures = vtileJSON[0].features;
+
+                assert.equal(features.length, vtileFeatures.length);
+                callback();
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: {
+                buffersize: {
+                    'grid.json': 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
+            template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: {
+                buffersize: {
+                    'grid.json': 128
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
+            template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 0' +
+                  ' overriden by template params with no buffersize in mapconfig',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: {
+                buffersize: {
+                    png: 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+
+    ];
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
+    testCases.forEach(function (test) {
+        var testFn = (usePostGIS) => {
+                it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) {
+                    serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                    test.template.name += '_1';
+                    this.testClient = new TestClient(test.template, 1234);
+                    serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+                    var coords = test.coords;
+                    var options = {
+                        format: test.format,
+                        placeholders: test.placeholders,
+                        layers: test.layers
+                    };
+                    this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                        assert.ifError(err);
+                        // To generate images use:
+                        //tile.save(test.fixturePath);
+                        // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
+                        // require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
+                        test.assert(tile, done);
+                    });
+                });
+        };
+        if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
+            testFn(true);
+        }
+        testFn(false);
+    });
+});

test/acceptance/cache/cache_headers.js

@@ -0,0 +1,393 @@
+var testHelper = require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var qs = require('querystring');
+
+var CartodbWindshaft = require('../../../lib/cartodb/server');
+var serverOptions = require('../../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+server.setMaxListeners(0);
+
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
+
+describe('get requests with cache headers', function() {
+
+    var keysToDelete;
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        testHelper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    var statusOkResponse = {
+        status: 200
+    };
+
+    var mapConfigs = [
+        {
+            "description": "cache headers should be present",
+            "cache_headers": {
+                "x_cache_channel": {
+                    "db_name": "test_windshaft_cartodb_user_1_db",
+                    "tables": ["public.test_table"]
+                },
+                "surrogate_keys": "t:77pJnX"
+            },
+            "data":
+                {
+                    version: '1.5.0',
+                    layers: [
+                        {
+                            options: {
+                                source: {
+                                    id: "2570e105-7b37-40d2-bdf4-1af889598745"
+                                },
+                                sql: 'select * from test_table limit 2',
+                                cartocss: '#layer { marker-fill:red; }',
+                                cartocss_version: '2.3.0',
+                                attributes: {
+                                    id:'cartodb_id',
+                                    columns: [
+                                        'name',
+                                        'address'
+                                    ]
+                                }
+                            }
+                        }
+                    ],
+                    analyses: [
+                        {
+                            "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from test_table limit 2"
+                            }
+                        }
+                    ]
+                },
+        },
+        {
+            "description": "cache headers should be present and be composed with source table name",
+            "cache_headers": {
+                "x_cache_channel": {
+                    "db_name": "test_windshaft_cartodb_user_1_db",
+                    "tables": ["public.analysis_2f13a3dbd7_9eb239903a1afd8a69130d1ece0fc8b38de8592d",
+                               "public.test_table"]
+                },
+                "surrogate_keys": "t:77pJnX t:iL4eth"
+            },
+            "data":
+            {
+                version: '1.5.0',
+                layers: [
+                    {
+                        options: {
+                            source: {
+                                id: "2570e105-7b37-40d2-bdf4-1af889598745"
+                            },
+                            sql: 'select * from test_table limit 2',
+                            cartocss: '#layer { marker-fill:red; }',
+                            cartocss_version: '2.3.0',
+                            attributes: {
+                                id:'cartodb_id',
+                                columns: [
+                                    'name',
+                                    'address'
+                                ]
+                            }
+                        }
+                    }
+                ],
+                analyses: [
+                    {
+                        "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                        "type": "buffer",
+                        "params": {
+                            "source": {
+                                "type": "source",
+                                "params": {
+                                    "query": "select * from test_table limit 2"
+                                }
+                            },
+                            "radius": 50000
+                        }
+                    }
+                ]
+            }
+        }];
+
+    var layergroupRequest = function(mapConfig) {
+        return {
+            url: '/api/v1/map?api_key=1234&config=' + encodeURIComponent(JSON.stringify(mapConfig)),
+            method: 'GET',
+            headers: {
+                host: 'localhost'
+            }
+        };
+    };
+
+    function getRequest(url, addApiKey, callbackName) {
+        var params = {};
+        if (!!addApiKey) {
+            params.api_key = '1234';
+        }
+        if (!!callbackName) {
+            params.callback = callbackName;
+        }
+
+        return {
+            url: url + '?' + qs.stringify(params),
+            method: 'GET',
+            headers: {
+                host: 'localhost',
+                'Content-Type': 'application/json'
+            }
+        };
+    }
+
+    function validateCacheHeaders(done, expectedCacheHeaders) {
+        return function(res, err) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(res.headers['x-cache-channel']);
+            assert.ok(res.headers['surrogate-key']);
+            if (expectedCacheHeaders) {
+                validateXChannelHeaders(res.headers, expectedCacheHeaders);
+                assert.equal(res.headers['surrogate-key'], expectedCacheHeaders.surrogate_keys);
+            }
+
+            done();
+        };
+    }
+
+    function validateXChannelHeaders(headers, expectedCacheHeaders) {
+        var dbName = headers['x-cache-channel'].split(':')[0];
+        var tables = headers['x-cache-channel'].split(':')[1].split(',').sort();
+        assert.equal(dbName, expectedCacheHeaders.x_cache_channel.db_name);
+        assert.deepEqual(tables, expectedCacheHeaders.x_cache_channel.tables.sort());
+    }
+
+    function noCacheHeaders(done) {
+        return function(res, err) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(
+                !res.headers['x-cache-channel'],
+                'did not expect x-cache-channel header, got: `' + res.headers['x-cache-channel'] + '`'
+            );
+            assert.ok(
+                !res.headers['surrogate-key'],
+                'did not expect surrogate-key header, got: `' + res.headers['surrogate-key'] + '`'
+            );
+            done();
+        };
+    }
+
+    function withLayergroupId(mapConfig, callback) {
+        assert.response(
+            server,
+            layergroupRequest(mapConfig),
+            statusOkResponse,
+            function(res, err) {
+                if (err) {
+                    return callback(err);
+                }
+                var layergroupId = JSON.parse(res.body).layergroupid;
+                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                callback(null, layergroupId, res);
+            }
+        );
+    }
+
+    mapConfigs.forEach(function(mapConfigData) {
+        describe(mapConfigData.description, function() {
+            var mapConfig = mapConfigData.data;
+            var expectedCacheHeaders = mapConfigData.cache_headers;
+            it('/api/v1/map Map instantiation', function(done) {
+                var testFn = validateCacheHeaders(done, expectedCacheHeaders);
+                withLayergroupId(mapConfig, function(err, layergroupId, res) {
+                    testFn(res);
+                });
+            });
+
+            it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik retina tiles', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/0/0@2x.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/:token/:z/:x/:y@:scale_factor?x.:format Mapnik tiles', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/0/0.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/:token/:layer/:z/:x/:y.(:format) Per :layer rendering', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/0/0/0.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/:token/:layer/attributes/:fid endpoint for info windows', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/' + layergroupId + '/0/attributes/1', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format static maps', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/static/center/' + layergroupId + '/0/0/0/400/300.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+
+            it ('/api/v1/map/static/bbox/:token/:bbox/:width/:height.:format static maps', function(done) {
+                withLayergroupId(mapConfig, function(err, layergroupId) {
+                    assert.response(
+                        server,
+                        getRequest('/api/v1/map/static/bbox/' + layergroupId + '/-45,-45,45,45/400/300.png', true),
+                        validateCacheHeaders(done, expectedCacheHeaders)
+                    );
+                });
+            });
+        });
+    });
+
+    describe('cache headers should NOT be present', function() {
+
+        it('/', function(done) {
+            assert.response(
+                server,
+                getRequest('/'),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        it('/version', function(done) {
+            assert.response(
+                server,
+                getRequest('/version'),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        it('/health', function(done) {
+            assert.response(
+                server,
+                getRequest('/health'),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        it('/api/v1/map/named list named maps', function(done) {
+            assert.response(
+                server,
+                getRequest('/api/v1/map/named', true),
+                statusOkResponse,
+                noCacheHeaders(done)
+            );
+        });
+
+        describe('with named maps', function() {
+
+            var templateName = 'x_cache';
+
+            beforeEach(function(done) {
+                var template =  {
+                    version: '0.0.1',
+                    name: templateName,
+                    auth: {
+                        method: 'open'
+                    },
+                    layergroup:  mapConfigs[0].data
+                };
+
+                var namedMapRequest = {
+                    url: '/api/v1/map/named?api_key=1234',
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(template)
+                };
+
+                assert.response(
+                    server,
+                    namedMapRequest,
+                    statusOkResponse,
+                    function(res, err) {
+                        done(err);
+                    }
+                );
+            });
+
+            afterEach(function(done) {
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + templateName + '?api_key=1234',
+                        method: 'DELETE',
+                        headers: {
+                            host: 'localhost'
+                        }
+                    },
+                    {
+                        status: 204
+                    },
+                    function(res, err) {
+                        done(err);
+                    }
+                );
+            });
+
+
+            it('/api/v1/map/named/:template_id Named map retrieval', function(done) {
+                assert.response(
+                    server,
+                    getRequest('/api/v1/map/named/' + templateName, true),
+                    statusOkResponse,
+                    noCacheHeaders(done)
+                );
+            });
+
+            it('/api/v1/map/named/:template_id/jsonp Named map retrieval', function(done) {
+                assert.response(
+                    server,
+                    getRequest('/api/v1/map/named/' + templateName, true, 'cb'),
+                    statusOkResponse,
+                    noCacheHeaders(done)
+                );
+            });
+        });
+    });
+});

test/acceptance/dataviews/aggregation.js

@@ -1,9 +1,8 @@
 require('../../support/test_helper');
-
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
-describe('aggregations', function() {
+describe('aggregations happy cases', function() {
 
     afterEach(function(done) {
         if (this.testClient) {
@@ -13,30 +12,36 @@ describe('aggregations', function() {
         }
     });
 
-    function aggregationOperationMapConfig(operation) {
-        return {
+    function aggregationOperationMapConfig(operation, query, column, aggregationColumn) {
+        column = column || 'adm0name';
+        aggregationColumn = aggregationColumn || 'pop_max';
+        query = query || 'select * from populated_places_simple_reduced';
+
+        var mapConfig = {
             version: '1.5.0',
             layers: [
                 {
                     type: 'mapnik',
                     options: {
-                        sql: 'select * from populated_places_simple_reduced',
+                        sql: query,
                         cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
                         cartocss_version: '2.0.1',
-                        widgets: {
-                            adm0name: {
-                                type: 'aggregation',
-                                options: {
-                                    column: 'adm0name',
-                                    aggregation: operation,
-                                    aggregationColumn: 'pop_max'
-                                }
-                            }
-                        }
+                        widgets: {}
                     }
                 }
             ]
         };
+
+        mapConfig.layers[0].options.widgets[column] = {
+            type: 'aggregation',
+            options: {
+                column: column,
+                aggregation: operation,
+                aggregationColumn: aggregationColumn
+            }
+        };
+
+        return mapConfig;
     }
 
     var operations = ['count', 'sum', 'avg', 'max', 'min'];
@@ -56,4 +61,367 @@ describe('aggregations', function() {
             });
         });
     });
+
+    var query = [
+        'select 1 as val, \'a\' as cat, ST_Transform(ST_SetSRID(ST_MakePoint(0,0),4326),3857) as the_geom_webmercator',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(0,1),4326),3857)',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(1,0),4326),3857)',
+        'select null, null, ST_Transform(ST_SetSRID(ST_MakePoint(1,1),4326),3857)'
+    ].join(' UNION ALL ');
+
+    operations.forEach(function (operation) {
+        var not = operation === 'count' ? ' not ' : ' ';
+        var description = 'should' +
+            not +
+            'handle NULL values in category and aggregation columns using "' +
+            operation +
+            '" as aggregation operation';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 3);
+                assert.equal(aggregation.count, 4);
+                assert.equal(aggregation.nulls, 1);
+
+                var hasNullCategory = false;
+                aggregation.categories.forEach(function (category) {
+                    if (category.category === null) {
+                        hasNullCategory = true;
+                    }
+                });
+
+                if (operation === 'count') {
+                    assert.ok(hasNullCategory, 'aggregation has not a category NULL');
+                } else {
+                    assert.ok(!hasNullCategory, 'aggregation has category NULL');
+                }
+
+                done();
+            });
+        });
+    });
+
+    var operations_and_values = {'count': 9, 'sum': 45, 'avg': 5, 'max': 9, 'min': 1};
+
+    var query_other = [
+        'select generate_series(1,3) as val, \'other_a\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(4,6) as val, \'other_b\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(7,9) as val, \'other_c\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_1\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_2\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_3\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_4\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_5\' as cat, NULL as the_geom_webmercator'
+    ].join(' UNION ALL ');
+
+    Object.keys(operations_and_values).forEach(function (operation) {
+        var description = 'should aggregate OTHER category using "' + operation + '"';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 8);
+                assert.equal(aggregation.count, 24);
+                assert.equal(aggregation.nulls, 0);
+
+                var aggregated_categories = aggregation.categories.filter( function(category) {
+                    return category.agg === true;
+                });
+                assert.equal(aggregated_categories.length, 1);
+                assert.equal(aggregated_categories[0].value, operations_and_values[operation]);
+
+                done();
+            });
+        });
+    });
+
+    var widgetSearchExpects = {
+        'count': [ { category: 'other_a', value: 3 } ],
+        'sum': [ { category: 'other_a', value: 6 } ],
+        'avg': [ { category: 'other_a', value: 2 } ],
+        'max': [ { category: 'other_a', value: 3 } ],
+        'min': [ { category: 'other_a', value: 1 } ]
+    };
+
+    Object.keys(operations_and_values).forEach(function (operation) {
+        var description = 'should search OTHER category using "' + operation + '"';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
+            this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) {
+                assert.ifError(err);
+
+                assert.ok(searchResult);
+                assert.equal(searchResult.type, 'aggregation');
+
+                assert.equal(searchResult.categories.length, 1);
+                assert.deepEqual(
+                    searchResult.categories,
+                    widgetSearchExpects[operation]
+                );
+                done();
+            });
+        });
+    });
+});
+
+describe('aggregation-dataview: special float values', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    function createMapConfig(layers, dataviews, analysis) {
+        return {
+            version: '1.5.0',
+            layers: layers,
+            dataviews: dataviews || {},
+            analyses: analysis || []
+        };
+    }
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "a0"
+                    },
+                    "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            val_aggregation: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'avg',
+                    aggregationColumn: 'val'
+                }
+            },
+            sum_aggregation_numeric: {
+                source: {
+                    id: 'a1'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'sum',
+                    aggregationColumn: 'val'
+                }
+            }
+        },
+        [
+            {
+                "id": "a0",
+                "type": "source",
+                "params": {
+                    "query": [
+                        'SELECT',
+                        '  null::geometry the_geom_webmercator,',
+                        '  CASE',
+                        '    WHEN x % 4 = 0 THEN \'infinity\'::float',
+                        '    WHEN x % 4 = 1 THEN \'-infinity\'::float',
+                        '    WHEN x % 4 = 2 THEN \'NaN\'::float',
+                        '    ELSE x',
+                        '  END AS val,',
+                        '  CASE',
+                        '    WHEN x % 2 = 0 THEN \'category_1\'',
+                        '    ELSE \'category_2\'',
+                        '  END AS cat',
+                        'FROM generate_series(1, 1000) x'
+                    ].join('\n')
+                }
+            }, {
+                "id": "a1",
+                "type": "source",
+                "params": {
+                    "query": [
+                        'SELECT',
+                        '  null::geometry the_geom_webmercator,',
+                        '  CASE',
+                        '    WHEN x % 3 = 0 THEN \'NaN\'::numeric',
+                        '    WHEN x % 3 = 1 THEN x',
+                        '    ELSE x',
+                        '  END AS val,',
+                        '  CASE',
+                        '    WHEN x % 2 = 0 THEN \'category_1\'',
+                        '    ELSE \'category_2\'',
+                        '  END AS cat',
+                        'FROM generate_series(1, 1000) x'
+                    ].join('\n')
+                }
+            }
+        ]
+    );
+
+    // Source a0
+    // -----------------------------------------------
+    // the_geom_webmercator  |    val    |    cat
+    // ----------------------+-----------+------------
+    //                       | -Infinity | category_2
+    //                       |       NaN | category_1
+    //                       |         3 | category_2
+    //                       |  Infinity | category_1
+    //                       | -Infinity | category_2
+    //                       |       NaN | category_1
+    //                       |         7 | category_2
+    //                       |  Infinity | category_1
+    //                       | -Infinity | category_2
+    //                       |       NaN | category_1
+    //                       |        11 | category_2
+    //                       |         " |          "
+
+    var filters = [{ own_filter: 0 }, {}];
+    filters.forEach(function (filter) {
+        it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) {
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) {
+                assert.ifError(err);
+                assert.ok(dataview.infinities === (250 + 250));
+                assert.ok(dataview.nans === 250);
+                assert.ok(dataview.categories.length === 1);
+                dataview.categories.forEach(function (category) {
+                    assert.ok(category.category === 'category_2');
+                    assert.ok(category.value === 501);
+                });
+                done();
+            });
+        });
+
+        it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) {
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) {
+                assert.ifError(err);
+                assert.ok(dataview.nans === 333);
+                assert.ok(dataview.categories.length === 2);
+                dataview.categories.forEach(function (category) {
+                    assert.ok(category.value !== null);
+                });
+                done();
+            });
+        });
+    });
+});
+
+describe('aggregation dataview tuned by categories query param', function () {
+    const mapConfig = {
+        version: '1.5.0',
+        layers: [
+            {
+                type: "cartodb",
+                options: {
+                    source: {
+                        "id": "a0"
+                    },
+                    cartocss: "#points { marker-width: 10; marker-fill: red; }",
+                    cartocss_version: "2.3.0"
+                }
+            }
+        ],
+        dataviews: {
+            categories: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'sum',
+                    aggregationColumn: 'val'
+                }
+            }
+        },
+        analyses: [
+            {
+                id: "a0",
+                type: "source",
+                params: {
+                    query: `
+                        SELECT
+                            null::geometry the_geom_webmercator,
+                            CASE
+                                WHEN x % 4 = 0 THEN 1
+                                WHEN x % 4 = 1 THEN 2
+                                WHEN x % 4 = 2 THEN 3
+                                ELSE 4
+                            END AS val,
+                            CASE
+                                WHEN x % 4 = 0 THEN 'category_1'
+                                WHEN x % 4 = 1 THEN 'category_2'
+                                WHEN x % 4 = 2 THEN 'category_3'
+                                ELSE 'category_4'
+                            END AS cat
+                        FROM generate_series(1, 1000) x
+                    `
+                }
+            }
+        ]
+    };
+
+    beforeEach(function () {
+        this.testClient = new TestClient(mapConfig, 1234);
+    });
+
+    afterEach(function (done) {
+        this.testClient.drain(done);
+    });
+
+    var scenarios = [
+        {
+            params: { own_filter: 0, categories: -1 },
+            categoriesExpected: 4
+        },
+        {
+            params: { own_filter: 0, categories: 0 },
+            categoriesExpected: 4
+        },
+        {
+            params: { own_filter: 0, categories: 1 },
+            categoriesExpected: 1
+        },
+        {
+            params: { own_filter: 0, categories: 2 },
+            categoriesExpected: 2
+        },
+        {
+            params: { own_filter: 0, categories: 4 },
+            categoriesExpected: 4
+        },
+        {
+            params: { own_filter: 0, categories: 5 },
+            categoriesExpected: 4
+        }
+    ];
+
+    scenarios.forEach(function (scenario) {
+        it(`should handle cartegories to customize aggregations: ${JSON.stringify(scenario.params)}`, function (done) {
+            this.testClient.getDataview('categories', scenario.params, (err, dataview) => {
+                assert.ifError(err);
+                assert.equal(dataview.categories.length, scenario.categoriesExpected);
+                done();
+            });
+        });
+    });
 });

test/acceptance/dataviews/formula.js

@@ -0,0 +1,80 @@
+require('../../support/test_helper');
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+function createMapConfig(layers, dataviews, analysis) {
+    return {
+        version: '1.5.0',
+        layers: layers,
+        dataviews: dataviews || {},
+        analyses: analysis || []
+    };
+}
+
+describe('formula-dataview: special float values', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "a0"
+                    },
+                    "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            val_formula: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'formula',
+                options: {
+                    column: 'val',
+                    operation: 'avg'
+                }
+            }
+        },
+        [
+            {
+                "id": "a0",
+                "type": "source",
+                "params": {
+                    "query": [
+                        'SELECT',
+                        '  null::geometry the_geom_webmercator,',
+                        '  CASE',
+                        '    WHEN x % 4 = 0 THEN \'infinity\'::float',
+                        '    WHEN x % 4 = 1 THEN \'-infinity\'::float',
+                        '    WHEN x % 4 = 2 THEN \'NaN\'::float',
+                        '    ELSE x',
+                        '  END AS val',
+                        'FROM generate_series(1, 1000) x'
+                    ].join('\n')
+                }
+            }
+        ]
+    );
+
+    it('should filter infinities out and count them in the summary', function(done) {
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('val_formula', {}, function(err, dataview) {
+            assert.ok(!err, err);
+            assert.equal(dataview.result, 501);
+            assert.ok(dataview.infinities === (250 + 250));
+            assert.ok(dataview.nans === 250);
+            done();
+        });
+    });
+});

test/acceptance/dataviews/overviews.js

@@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() {
                 params: {
                   query: 'select * from test_table_overviews'
                 }
+            },
+            {
+                id: 'data-source-special-float-values',
+                type: 'source',
+                params: {
+                  query: 'select * from test_special_float_values_table_overviews'
+                }
             }
         ],
         dataviews:  {
@@ -144,6 +151,43 @@ describe('dataviews using tables with overviews', function() {
                     aggregationColumn: 'name',
                 }
             },
+            test_categories_special_values: {
+                type: 'aggregation',
+                source: {
+                    id: 'data-source-special-float-values'
+                },
+                options: {
+                    column: 'name',
+                    aggregation: 'sum',
+                    aggregationColumn: 'value',
+                }
+            },
+            test_histogram: {
+                type: 'histogram',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    bins: 2
+                }
+            },
+            test_histogram_date: {
+                type: 'histogram',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'updated_at',
+                    bins: 2
+                }
+            },
+            test_histogram_special_values: {
+                type: 'histogram',
+                source: {
+                    id: 'data-source-special-float-values'
+                },
+                options: {
+                    column: 'value',
+                    bins: 2
+                }
+            },
             test_avg: {
                 type: 'formula',
                 source: {id: 'data-source'},
@@ -152,6 +196,16 @@ describe('dataviews using tables with overviews', function() {
                     operation: 'avg'
                 }
             },
+            test_formula_sum_special_values: {
+                type: 'formula',
+                source: {
+                    id: 'data-source-special-float-values'
+                },
+                options: {
+                    column: 'value',
+                    operation: 'sum'
+                }
+            },
             test_count: {
                 type: 'formula',
                 source: {id: 'data-source'},
@@ -186,6 +240,17 @@ describe('dataviews using tables with overviews', function() {
                     cartocss_version: '2.3.0',
                     source: { id: 'data-source' }
                 }
+            },
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from test_special_float_values_table_overviews',
+                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                    cartocss_version: '2.3.0',
+                    source: {
+                        id: 'data-source-special-float-values'
+                    }
+                }
             }
         ]
     };
@@ -196,7 +261,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"sum",
+                "result":15,
+                "infinities": 0,
+                "nans": 0,
+                "nulls":0,
+                "type":"formula"
+            });
 
             testClient.drain(done);
         });
@@ -208,7 +280,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"avg",
+                "result":3,
+                "nulls":0,
+                "type":"formula",
+                "infinities": 0,
+                "nans": 0
+            });
 
             testClient.drain(done);
         });
@@ -220,7 +299,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"count",
+                "result":5,
+                "nulls":0,
+                "type":"formula",
+                "infinities": 0,
+                "nans": 0
+            });
 
             testClient.drain(done);
         });
@@ -232,7 +318,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation": "max",
+                "result": 5,
+                "nulls": 0,
+                "infinities": 0,
+                "nans": 0,
+                "type": "formula"
+            });
 
             testClient.drain(done);
         });
@@ -244,7 +337,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation": "min",
+                "result": 1,
+                "nulls": 0,
+                "infinities": 0,
+                "nans": 0,
+                "type": "formula"
+            });
 
             testClient.drain(done);
         });
@@ -259,14 +359,96 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"sum",
+                "result":15,
+                "nulls":0,
+                "infinities": 0,
+                "nans": 0,
+                "type":"formula"
+            });
 
             testClient.drain(done);
         });
     });
 
+    it("should expose a histogram", function (done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_histogram', function (err, histogram) {
+            if (err) {
+                return done(err);
+            }
+            assert.ok(histogram);
+            assert.equal(histogram.type, 'histogram');
+            assert.ok(Array.isArray(histogram.bins));
+            testClient.drain(done);
+        });
+    });
+
     describe('filters', function() {
 
+        describe('histogram', function () {
+
+            it("should expose a filtered histogram", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram: { min: 2 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 4);
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered histogram with no results", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram: { max: -1 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 0);
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered date histogram with no results", function (done) {
+                // This most likely works because the overviews will pass
+                // the responsibility to the normal dataviews.
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram_date: { max: -1 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram_date', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 0);
+                    testClient.drain(done);
+                });
+            });
+        });
+
         describe('category', function () {
 
             var params = {
@@ -281,7 +463,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"sum",
+                        "result":1,
+                        "nulls":0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type":"formula"
+                    });
                     testClient.drain(done);
                 });
             });
@@ -292,7 +481,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"avg",
+                        "result":1,
+                        "nulls":0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type":"formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -304,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"count",
+                        "result":1,
+                        "infinities": 0,
+                        "nans": 0,
+                        "nulls":0,
+                        "type":"formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -316,7 +519,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation": "max",
+                        "result": 1,
+                        "nulls": 0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type": "formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -328,7 +538,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation": "min",
+                        "result": 1,
+                        "nulls": 0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type": "formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -346,7 +563,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"sum",
+                        "result":1,
+                        "nulls":0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type":"formula"
+                    });
                     testClient.drain(done);
                 });
             });
@@ -354,5 +578,69 @@ describe('dataviews using tables with overviews', function() {
 
         });
 
+        describe('aggregation special float values', function () {
+            var params = {};
+
+            it("should expose an aggregation dataview filtering special float values out", function (done) {
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_categories_special_values', params, function (err, dataview) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(dataview, {
+                        aggregation: 'sum',
+                        count: 5,
+                        nulls: 0,
+                        nans: 1,
+                        infinities: 1,
+                        min: 6,
+                        max: 6,
+                        categoriesCount: 1,
+                        categories: [ { category: 'Hawai', value: 6, agg: false } ],
+                        type: 'aggregation'
+                    });
+                    testClient.drain(done);
+                });
+            });
+
+            it('should expose a histogram dataview filtering special float values out', function (done) {
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(dataview, {
+                        bin_width: 0,
+                        bins_count: 1,
+                        bins_start: 3,
+                        nulls: 0,
+                        infinities: 1,
+                        nans: 1,
+                        avg: 3,
+                        bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ],
+                        type: 'histogram'
+                    });
+                    testClient.drain(done);
+                });
+            });
+
+            it('should expose a formula (sum) dataview filtering special float values out', function (done) {
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(dataview, {
+                        operation: 'sum',
+                        result: 6,
+                        nulls: 0,
+                        nans: 1,
+                        infinities: 1,
+                        type: 'formula'
+                    });
+                    testClient.drain(done);
+                });
+            });
+        });
     });
 });