Release 6.5.0

Update cartodb-psql to 0.13.1 and related reverse dependenciess (WIP)

.travis.yml

@@ -4,8 +4,14 @@ jobs:
       services:
         - docker
       language: generic
-      before_install: docker pull carto/nodejs6-xenial-pg101
-      script: npm run docker-test
+      before_install: docker pull carto/nodejs6-xenial-pg101:postgis-2.4.4.5
+      script: npm run docker-test -- nodejs6
+    - sudo: required
+      services:
+        - docker
+      language: generic
+      before_install: docker pull carto/nodejs10-xenial-pg101:postgis-2.4.4.5
+      script: npm run docker-test -- nodejs10
     - dist: precise
       addons:
         postgresql: "9.5"
@@ -28,7 +34,7 @@ jobs:
         - sudo add-apt-repository -y ppa:cartodb/gis-testing
 
         - sudo apt-get update
-      
+
         # Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis)
         - sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2
 
@@ -59,9 +65,12 @@ jobs:
         - createdb template_postgis
         - createuser publicuser
         - psql -c "CREATE EXTENSION postgis" template_postgis
-        
+
+        - psql -c "select version();" template_postgis
+        - psql -c "select postgis_version();" template_postgis
+
         # install yarn 0.27.5
-        - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 0.27.5	
+        - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 0.27.5
         - export PATH="$HOME/.yarn/bin:$PATH"
 
         # instal redis 4

NEWS.md

@@ -1,5 +1,53 @@
 # Changelog
 
+## 6.5.0
+Released 2018-12-26
+
+New features
+- Suport Node.js 10
+- Configure travis to run docker tests against Node.js 6 & 10 versions
+- Aggregation time dimensions
+- Update sample configurations to use PostGIS to generate MVT's by default (as in production)
+- Upgrades Windshaft to [4.12.1](https://github.com/CartoDB/Windshaft/blob/4.12.1/NEWS.md#version-4121)
+  - `pg-mvt`: Use `query-rewriter` to compose the query to render a MVT tile. If not defined, it will use a Default Query Rewriter.
+  - `pg-mvt`: Fix bug while building query and there is no columns defined for the layer.
+  - `pg-mvt`: Accept trailing semicolon in input queries.
+  - `Renderer Cache Entry`: Do not throw errors for integrity checks.
+  - Fix bug when releasing the renderer cache entry in some scenarios.
+  - Upgrade grainstore to [1.10.0](https://github.com/CartoDB/grainstore/releases/tag/1.10.0)
+- Upgrade cartodb-redis to [2.1.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/2.1.0)
+- Upgrade cartodb-query-tables to [0.4.0](https://github.com/CartoDB/node-cartodb-query-tables/releases/tag/0.4.0)
+- Upgrade cartodb-psql to [0.13.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.13.1)
+- Upgrade turbo-carto to [0.21.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.21.0)
+- Upgrade camshaft to [0.63.1](https://github.com/CartoDB/camshaft/releases/tag/0.63.1)
+- Upgrade redis-mpool to [0.7.0](https://github.com/CartoDB/node-redis-mpool/releases/tag/0.7.0)
+
+Bug Fixes:
+- Prevent from uncaught exception: Range filter Error from camshaft when getting analysis query.
+- Make all modules to use strict mode semantics.
+
+## 6.4.0
+Released 2018-09-24
+
+- Upgrades Camshaft to [0.62.3](https://github.com/CartoDB/camshaft/releases/tag/0.61.11):
+  - Build query from node's cache to compute output columns when building analysis
+  - Adds metadata columns for street level geocoding
+- Remove use of `step` module to handle asynchronous code, now it's defined as development dependency.
+- Bug Fixes: (#1020)
+  - Fix bug in date-wrapper regarding columns with spaces
+  - Fix bug in aggregation-query regarding columns with spaces
+- Upgrades Windshaft to [4.10.0](https://github.com/CartoDB/Windshaft/blob/4.10.0/NEWS.md#version-4100)
+  - `pg-mvt`:
+    - Now matches the behaviour of the `mapnik` renderer for MVTs.
+    - Removed undocummented filtering by `layer.options.columns`.
+    - Implement timeout in getTile.
+    - Several bugfixes.
+  - Dependency updates: Fixed a bug in Mapnik MVT renderer and cleanup in `tilelive-mapnik`.
+  - [MapConfig 1.8.0 released](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.8.0.md) with new options for MVTs:
+    - Add **`vector_extent`** option in MapConfig to setup the layer extent.
+    - Add **`vector_simplify_extent`** option in MapConfig to configure the simplification process.
+  - Remove use of `step` module to handle asynchronous code, now it's defined as development dependency.
+
 ## 6.3.0
 Released 2018-07-26
 

app.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var http = require('http');
 var https = require('https');
 var path = require('path');
@@ -126,6 +128,40 @@ listener.on('listening', function() {
     );
 });
 
+function getCPUUsage (oldUsage) {
+    let usage;
+
+    if (oldUsage && oldUsage._start) {
+        usage = Object.assign({}, process.cpuUsage(oldUsage._start.cpuUsage));
+        usage.time = Date.now() - oldUsage._start.time;
+    } else {
+        usage = Object.assign({}, process.cpuUsage());
+        usage.time = process.uptime() * 1000; // s to ms
+    }
+
+    usage.percent = (usage.system + usage.user) / (usage.time * 10);
+
+    Object.defineProperty(usage, '_start', {
+        value: {
+            cpuUsage: process.cpuUsage(),
+            time: Date.now()
+        }
+    });
+
+    return usage;
+}
+
+let previousCPUUsage = getCPUUsage();
+setInterval(function cpuUsageMetrics () {
+    const CPUUsage = getCPUUsage(previousCPUUsage);
+
+    Object.keys(CPUUsage).forEach(property => {
+        global.statsClient.gauge(`windshaft.cpu.${property}`, CPUUsage[property]);
+    });
+
+    previousCPUUsage = CPUUsage;
+}, 5000);
+
 setInterval(function() {
     var memoryUsage = process.memoryUsage();
     Object.keys(memoryUsage).forEach(function(k) {

carto-package.json

@@ -0,0 +1,16 @@
+{
+    "name": "carto_windshaft",
+    "current_version": {
+        "requires": {
+            "node": ">=6.9.2 <10.0.0",
+            "yarn": ">=0.27.5 <1.0.0",
+            "mapnik": ">=3.0.15"
+        },
+        "works_with": {
+            "redis": ">=3.0.0",
+            "postgresql": ">=9.5.0",
+            "postgis": ">=2.2.0.0",
+            "carto_postgresql_ext": ">=0.19.0"
+        }
+    }
+}

config/environments/development.js.example

@@ -130,7 +130,7 @@ var config = {
         //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
         //PostGIS 2.4 is required for this to work
         //If disabled it will use Mapnik MVT generation
-        usePostGIS: false
+        usePostGIS: true
       },
       mapnik: {
           // The size of the pool of internal mapnik backend

config/environments/production.js.example

@@ -130,7 +130,7 @@ var config = {
         //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
         //PostGIS 2.4 is required for this to work
         //If disabled it will use Mapnik MVT generation
-        usePostGIS: false
+        usePostGIS: true
       },
       mapnik: {
           // The size of the pool of internal mapnik backend

config/environments/staging.js.example

@@ -130,7 +130,7 @@ var config = {
         //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
         //PostGIS 2.4 is required for this to work
         //If disabled it will use Mapnik MVT generation
-        usePostGIS: false
+        usePostGIS: true
       },
       mapnik: {
           // The size of the pool of internal mapnik backend

config/environments/test.js.example

@@ -130,7 +130,7 @@ var config = {
         //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
         //PostGIS 2.4 is required for this to work
         //If disabled it will use Mapnik MVT generation
-        usePostGIS: false
+        usePostGIS: true
       },
       mapnik: {
           // The size of the pool of internal mapnik backend

docker-bash.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run  -it -v `pwd`:/srv carto/${1:-nodejs10-xenial-pg101:postgis-2.4.4.5} bash

docker-test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 [nodejs6|nodejs10]"
+    exit 1
+}
+
+echo "$0 $1"
+
+NODEJS_VERSION=${1-nodejs10}
+
+if [ "$NODEJS_VERSION" = "nodejs10" ];
+then
+    DOCKER='nodejs10-xenial-pg101:postgis-2.4.4.5'
+elif [ "$NODEJS_VERSION" = "nodejs6" ];
+then
+    DOCKER='nodejs6-xenial-pg101:postgis-2.4.4.5'
+else
+    usage
+fi
+
+docker run -v `pwd`:/srv carto/${DOCKER} bash run_tests_docker.sh ${NODEJS_VERSION} && \
+    docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v

docker/Dockerfile-nodejs10-xenial-pg101:postgis-2.4.5.5

@@ -0,0 +1,88 @@
+FROM ubuntu:xenial
+
+# Use UTF8 to avoid encoding problems with pgsql
+ENV LANG C.UTF-8
+ENV NPROCS 1
+ENV JOBS 1
+ENV CXX g++-4.9
+ENV PGUSER postgres
+
+# Add external repos
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        curl \
+        software-properties-common \
+        locales \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && add-apt-repository -y ppa:cartodb/postgresql-10 \
+    && add-apt-repository -y ppa:cartodb/gis \
+    && curl -sL https://deb.nodesource.com/setup_10.x | bash \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        g++-4.9 \
+        gcc-4.9 \
+        git \
+        libcairo2-dev \
+        libgdal-dev \
+        libgdal1i \
+        libgdal20 \
+        libgeos-dev \
+        libgif-dev \
+        libjpeg8-dev \
+        libjson-c-dev \
+        libpango1.0-dev \
+        libpixman-1-dev \
+        libproj-dev \
+        libprotobuf-c-dev \
+        libxml2-dev \
+        gdal-bin \
+        make \
+        nodejs \
+        protobuf-c-compiler \
+        pkg-config \
+        wget \
+        zip \
+        postgresql-10 \
+        postgresql-10-plproxy \
+        postgis=2.4.4.5+carto-1 \
+        postgresql-10-postgis-2.4=2.4.4.5+carto-1 \
+        postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \
+        postgresql-10-postgis-scripts=2.4.4.5+carto-1 \
+        postgresql-client-10 \
+        postgresql-client-common \
+        postgresql-common \
+        postgresql-contrib \
+        postgresql-plpython-10 \
+        postgresql-server-dev-10 \
+    && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
+    && tar xvzf redis-4.0.8.tar.gz \
+    && cd redis-4.0.8 \
+    && make \
+    && make install \
+    && cd .. \
+    && rm redis-4.0.8.tar.gz \
+    && rm -R redis-4.0.8 \
+    && apt-get purge -y wget protobuf-c-compiler \
+    && apt-get autoremove -y
+
+# Configure PostgreSQL
+RUN set -ex \
+    && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
+    && echo "local     all       all                     trust" >  /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && /etc/init.d/postgresql start \
+    && createdb template_postgis \
+    && createuser publicuser \
+    && psql -c "CREATE EXTENSION postgis" template_postgis \
+    && /etc/init.d/postgresql stop
+
+WORKDIR /srv
+EXPOSE 5858
+
+CMD /etc/init.d/postgresql start

docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4

@@ -0,0 +1,89 @@
+FROM ubuntu:xenial
+
+# Use UTF8 to avoid encoding problems with pgsql
+ENV LANG C.UTF-8
+ENV NPROCS 1
+ENV JOBS 1
+ENV CXX g++-4.9
+ENV PGUSER postgres
+
+# Add external repos
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        curl \
+        software-properties-common \
+        locales \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && add-apt-repository -y ppa:cartodb/postgresql-10 \
+    && add-apt-repository -y ppa:cartodb/gis \
+    && curl -sL https://deb.nodesource.com/setup_6.x | bash \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8
+
+# Install dependencies and PostGIS 2.4 from sources
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        g++-4.9 \
+        gcc-4.9 \
+        git \
+        libcairo2-dev \
+        libgdal-dev \
+        libgdal1i \
+        libgdal20 \
+        libgeos-dev \
+        libgif-dev \
+        libjpeg8-dev \
+        libjson-c-dev \
+        libpango1.0-dev \
+        libpixman-1-dev \
+        libproj-dev \
+        libprotobuf-c-dev \
+        libxml2-dev \
+        gdal-bin \
+        make \
+        nodejs \
+        protobuf-c-compiler \
+        pkg-config \
+        wget \
+        zip \
+        postgresql-10 \
+        postgresql-10-plproxy \
+        postgresql-10-postgis-2.4 \
+        postgresql-10-postgis-2.4-scripts \
+        postgresql-10-postgis-scripts \
+        postgresql-client-10 \
+        postgresql-client-common \
+        postgresql-common \
+        postgresql-contrib \
+        postgresql-plpython-10 \
+        postgresql-server-dev-10 \
+        postgis \
+    && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
+    && tar xvzf redis-4.0.8.tar.gz \
+    && cd redis-4.0.8 \
+    && make \
+    && make install \
+    && cd .. \
+    && rm redis-4.0.8.tar.gz \
+    && rm -R redis-4.0.8 \
+    && apt-get purge -y wget protobuf-c-compiler \
+    && apt-get autoremove -y 
+    
+# Configure PostgreSQL
+RUN set -ex \
+    && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
+    && echo "local     all       all                     trust" >  /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && /etc/init.d/postgresql start \
+    && createdb template_postgis \
+    && createuser publicuser \
+    && psql -c "CREATE EXTENSION postgis" template_postgis \
+    && /etc/init.d/postgresql stop
+
+WORKDIR /srv
+EXPOSE 5858
+
+CMD /etc/init.d/postgresql start

docker/Dockerfile-nodejs6-xenial-pg101:postgis-2.4.4.5

@@ -0,0 +1,88 @@
+FROM ubuntu:xenial
+
+# Use UTF8 to avoid encoding problems with pgsql
+ENV LANG C.UTF-8
+ENV NPROCS 1
+ENV JOBS 1
+ENV CXX g++-4.9
+ENV PGUSER postgres
+
+# Add external repos
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        curl \
+        software-properties-common \
+        locales \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && add-apt-repository -y ppa:cartodb/postgresql-10 \
+    && add-apt-repository -y ppa:cartodb/gis \
+    && curl -sL https://deb.nodesource.com/setup_6.x | bash \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8
+
+RUN set -ex \
+    && apt-get update \
+    && apt-get install -y \
+        g++-4.9 \
+        gcc-4.9 \
+        git \
+        libcairo2-dev \
+        libgdal-dev \
+        libgdal1i \
+        libgdal20 \
+        libgeos-dev \
+        libgif-dev \
+        libjpeg8-dev \
+        libjson-c-dev \
+        libpango1.0-dev \
+        libpixman-1-dev \
+        libproj-dev \
+        libprotobuf-c-dev \
+        libxml2-dev \
+        gdal-bin \
+        make \
+        nodejs \
+        protobuf-c-compiler \
+        pkg-config \
+        wget \
+        zip \
+        postgresql-10 \
+        postgresql-10-plproxy \
+        postgis=2.4.4.5+carto-1 \
+        postgresql-10-postgis-2.4=2.4.4.5+carto-1 \
+        postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \
+        postgresql-10-postgis-scripts=2.4.4.5+carto-1 \
+        postgresql-client-10 \
+        postgresql-client-common \
+        postgresql-common \
+        postgresql-contrib \
+        postgresql-plpython-10 \
+        postgresql-server-dev-10 \
+    && wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
+    && tar xvzf redis-4.0.8.tar.gz \
+    && cd redis-4.0.8 \
+    && make \
+    && make install \
+    && cd .. \
+    && rm redis-4.0.8.tar.gz \
+    && rm -R redis-4.0.8 \
+    && apt-get purge -y wget protobuf-c-compiler \
+    && apt-get autoremove -y
+
+# Configure PostgreSQL
+RUN set -ex \
+    && echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
+    && echo "local     all       all                     trust" >  /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       0.0.0.0/0     trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && echo "host      all       all       ::1/128       trust" >> /etc/postgresql/10/main/pg_hba.conf \
+    && /etc/init.d/postgresql start \
+    && createdb template_postgis \
+    && createuser publicuser \
+    && psql -c "CREATE EXTENSION postgis" template_postgis \
+    && /etc/init.d/postgresql stop
+
+WORKDIR /srv
+EXPOSE 5858
+
+CMD /etc/init.d/postgresql start

docker/reference.md

@@ -11,13 +11,13 @@ https://hub.docker.com/r/carto/
 
 ## Update image
 - Edit the docker image file with your desired changes
-- Build image: 
+- Build image:
   - `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/`
 
 - Upload to docker hub:
-  - Login into docker hub: 
+  - Login into docker hub:
     - `docker login`
-  - Create tag: 
+  - Create tag:
     - `docker tag carto/IMAGE carto/IMAGE`
-  - Upload: 
+  - Upload:
     - `docker push carto/IMAGE`

docs/aggregation.md

@@ -134,6 +134,10 @@ of the original dataset applying three different aggregate functions.
 
 > Note that you can use the original column names as names of the result, but all the result column names must be unique.  In particular, the names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used for aggregated columns, as they correspond to columns always present in the result.
 
+#### Limitations:
+* The iso text format does not admit `starting` or `count` parameters
+* Cyclic units (day of the week, etc.) don't admit `count` or `starting` either.
+
 ### `resolution`
 
 Defines the cell-size of the spatial aggregation grid. This is equivalent to the [CartoCSS `-torque-resolution`](https://carto.com/docs/carto-engine/cartocss/properties-for-torque/#-torque-resolution-float) property of Torque maps.

lib/cartodb/api/api-router.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const { Router: router } = require('express');
 
 const RedisPool = require('redis-mpool');

lib/cartodb/api/map/analyses-catalog-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const PSQL = require('cartodb-psql');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');

lib/cartodb/api/map/analysis-layergroup-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');

lib/cartodb/api/map/anonymous-map-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const windshaft = require('windshaft');
 const MapConfig = windshaft.model.MapConfig;
 const Datasource = windshaft.model.Datasource;
@@ -152,8 +154,17 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
             }
         };
 
-        mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => {
+        mapConfigAdapter.getMapConfig(user,
+                                      requestMapConfig,
+                                      params,
+                                      context,
+                                      (err, requestMapConfig, stats = { overviewsAddedToMapconfig : false }) => {
             req.profiler.done('anonymous.getMapConfig');
+
+            stats.mapType = 'anonymous';
+            req.profiler.add(stats);
+
+
             if (err) {
                 return next(err);
             }

lib/cartodb/api/map/attributes-layergroup-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');

lib/cartodb/api/map/dataview-layergroup-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const layergroupToken = require('../middlewares/layergroup-token');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');

lib/cartodb/api/map/map-router.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const { Router: router } = require('express');
 
 const AnalysisLayergroupController = require('./analysis-layergroup-controller');

lib/cartodb/api/map/preview-layergroup-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const layergroupToken = require('../middlewares/layergroup-token');
 const coordinates = require('../middlewares/coordinates');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');

lib/cartodb/api/map/preview-template-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
 const dbConnSetup = require('../middlewares/db-conn-setup');

lib/cartodb/api/map/tile-layergroup-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const layergroupToken = require('../middlewares/layergroup-token');
 const coordinates = require('../middlewares/coordinates');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@@ -147,10 +149,6 @@ function incrementErrorMetrics (statsClient) {
 
 function tileError () {
     return function tileErrorMiddleware (err, req, res, next) {
-        if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
-            res.statusCode = 204;
-            return next();
-        }
 
         // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
         let errMsg = err.message ? ( '' + err.message ) : ( '' + err );

lib/cartodb/api/middlewares/augment-layergroup-data.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const _ = require('underscore');
 
 module.exports = function augmentLayergroupData () {

lib/cartodb/api/middlewares/authorize.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function authorize (authBackend) {
     return function authorizeMiddleware (req, res, next) {
         authBackend.authorize(req, res, (err, authorized) => {

lib/cartodb/api/middlewares/cache-channel-header.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function setCacheChannelHeader () {
     return function setCacheChannelHeaderMiddleware (req, res, next) {
         if (req.method !== 'GET') {

lib/cartodb/api/middlewares/cache-control-header.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
 
 module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) {

lib/cartodb/api/middlewares/check-json-content-type.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function checkJsonContentType () {
     return function checkJsonContentTypeMiddleware(req, res, next) {
         if (req.method === 'POST' && !req.is('application/json')) {

lib/cartodb/api/middlewares/check-static-image-format.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const VALID_IMAGE_FORMATS = ['png', 'jpg'];
 
 module.exports = function checkStaticImageFormat () {

lib/cartodb/api/middlewares/clean-up-query-params.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const _ = require('underscore');
 
 // Whitelist query parameters and attach format

lib/cartodb/api/middlewares/coordinates.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const positiveIntegerNumberRegExp = /^\d+$/;
 const integerNumberRegExp = /^-?\d+$/;
 const invalidZoomMessage = function (zoom) {

lib/cartodb/api/middlewares/cors.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function cors () {
     return function corsMiddleware (req, res, next) {
         const headers = [

lib/cartodb/api/middlewares/credentials.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const basicAuth = require('basic-auth');
 
 module.exports = function credentials () {

lib/cartodb/api/middlewares/db-conn-setup.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const _ = require('underscore');
 
 module.exports = function dbConnSetup (pgConnection) {

lib/cartodb/api/middlewares/error-middleware.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const _ = require('underscore');
 const debug = require('debug')('windshaft:cartodb:error-middleware');
 

lib/cartodb/api/middlewares/increment-map-view-count.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function incrementMapViewCount (metadataBackend) {
     return function incrementMapViewCountMiddleware(req, res, next) {
         const { mapConfig, user } = res.locals;

lib/cartodb/api/middlewares/init-profiler.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function initProfiler (isTemplateInstantiation) {
     const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
 

lib/cartodb/api/middlewares/initialize-status-code.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function initializeStatusCode () {
     return function initializeStatusCodeMiddleware (req, res, next) {
         if (req.method !== 'OPTIONS') {

lib/cartodb/api/middlewares/last-modified-header.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function setLastModifiedHeader () {
     return function setLastModifiedHeaderMiddleware(req, res, next) {
         if (req.method !== 'GET') {

lib/cartodb/api/middlewares/last-updated-time-layergroup.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function setLastUpdatedTimeToLayergroup () {
     return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
         const { mapConfigProvider, analysesResults } = res.locals;

lib/cartodb/api/middlewares/layer-stats.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function setLayerStats (pgConnection, statsBackend) {
     return function setLayerStatsMiddleware(req, res, next) {
         const { user, mapConfig } = res.locals;

lib/cartodb/api/middlewares/layergroup-id-header.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash) {
     return function setLayergroupIdHeaderMiddleware (req, res, next) {
         const { user, template } = res.locals;

lib/cartodb/api/middlewares/layergroup-metadata.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function setMetadataToLayergroup (layergroupMetadata, includeQuery) {
     return function setMetadataToLayergroupMiddleware (req, res, next) {
         const { user, mapConfig, analysesResults = [], context, api_key: userApiKey } = res.locals;

lib/cartodb/api/middlewares/layergroup-token.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const LayergroupToken = require('../../models/layergroup-token');
 const authErrorMessageTemplate = function (signer, user) {
     return `Cannot use map signature of user "${signer}" on db of user "${user}"`;

lib/cartodb/api/middlewares/logger.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function logger (options) {
     if (!global.log4js || !options.log_format) {
         return function dummyLoggerMiddleware (req, res, next) {

lib/cartodb/api/middlewares/lzma.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const LZMA = require('lzma').LZMA;
 
 module.exports = function lzma () {

lib/cartodb/api/middlewares/map-error.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function mapError (options) {
     const { addContext = false, label = 'MAPS CONTROLLER' } = options;
 

lib/cartodb/api/middlewares/map-store-map-config-provider.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const MapStoreMapConfigProvider = require('../../models/mapconfig/provider/map-store-provider');
 
 module.exports = function createMapStoreMapConfigProvider (

lib/cartodb/api/middlewares/named-map-provider.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) {
     return function getNamedMapProviderMiddleware (req, res, next) {
         const { user, token, cache_buster, api_key } = res.locals;

lib/cartodb/api/middlewares/noop.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function noop () {
     return function noopMiddleware (req, res, next) {
         next();

lib/cartodb/api/middlewares/send-response.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function sendResponse () {
     return function sendResponseMiddleware (req, res) {
         req.profiler.done('res');

lib/cartodb/api/middlewares/served-by-host-header.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const os = require('os');
 
 module.exports = function servedByHostHeader () {

lib/cartodb/api/middlewares/stats.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const Profiler = require('../../stats/profiler_proxy');
 const debug = require('debug')('windshaft:cartodb:stats');
 const onHeaders = require('on-headers');

lib/cartodb/api/middlewares/surrogate-key-header.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const NamedMapsCacheEntry = require('../../cache/model/named_maps_entry');
 const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider');
 

lib/cartodb/api/middlewares/syntax-error.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function syntaxError () {
     return function syntaxErrorMiddleware (err, req, res, next) {
         if (err.name === 'SyntaxError') {

lib/cartodb/api/middlewares/user.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const CdbRequest = require('../../models/cdb_request');
 
 module.exports = function user () {

lib/cartodb/api/middlewares/vector-error.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const fs = require('fs');
 const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../../assets/render-timeout-fallback.mvt');
 

lib/cartodb/api/template/admin-template-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const { templateName } = require('../../backends/template_maps');
 const credentials = require('../middlewares/credentials');
 const rateLimit = require('../middlewares/rate-limit');

lib/cartodb/api/template/named-template-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');
 const dbConnSetup = require('../middlewares/db-conn-setup');
@@ -165,8 +167,12 @@ function getTemplate (
             params
         );
 
-        mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => {
+        mapConfigProvider.getMapConfig((err, mapConfig, rendererParams, context, stats = {}) => {
             req.profiler.done('named.getMapConfig');
+
+            stats.mapType = 'named';
+            req.profiler.add(stats);
+
             if (err) {
                 return next(err);
             }

lib/cartodb/api/template/template-router.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const { Router: router } = require('express');
 
 const NamedMapController = require('./named-template-controller');

lib/cartodb/api/template/tile-template-controller.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const coordinates = require('../middlewares/coordinates');
 const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
 const credentials = require('../middlewares/credentials');

lib/cartodb/backends/analysis-status.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var PSQL = require('cartodb-psql');
 
 function AnalysisStatusBackend() {

lib/cartodb/backends/auth.js

@@ -1,3 +1,5 @@
+'use strict';
+
 /**
  *
  * @param {PgConnection} pgConnection

lib/cartodb/backends/dataview.js

@@ -1,7 +1,7 @@
-var assert = require('assert');
+'use strict';
+
 var _ = require('underscore');
 var PSQL = require('cartodb-psql');
-var step = require('step');
 var BBoxFilter = require('../models/filter/bbox');
 var DataviewFactory = require('../models/dataview/factory');
 var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
@@ -21,53 +21,76 @@ function DataviewBackend(analysisBackend) {
 module.exports = DataviewBackend;
 
 DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
+    const dataviewName = params.dataviewName;
 
-    var dataviewName = params.dataviewName;
-    step(
-        function getMapConfig() {
-            mapConfigProvider.getMapConfig(this);
-        },
-        function runDataviewQuery(err, mapConfig) {
-            assert.ifError(err);
+    mapConfigProvider.getMapConfig(function (err, mapConfig) {
+        if (err) {
+            return callback(err);
+        }
 
-            var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
-            if (!dataviewDefinition) {
-                throw new Error("Dataview '" + dataviewName + "' does not exists");
-            }
+        var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
+        if (!dataviewDefinition) {
+            const error = new Error(`Dataview '${dataviewName}' does not exist`);
+            error.type = 'dataview';
+            error.http_status = 400;
+            return callback(error);
+        }
 
+        if (!validFilterParams(params)) {
+            const error = new Error('Both own_filter and no_filters cannot be sent in the same request');
+            error.type = 'dataview';
+            error.http_status = 400;
+            return callback(error);
+        }
+
+        var pg;
+        var overrideParams;
+        var dataview;
+
+        try {
+            pg = new PSQL(dbParamsFromReqParams(params));
+            var query = getQueryWithFilters(dataviewDefinition, params);
+            var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
+            var dataviewFactory = DataviewFactoryWithOverviews.getFactory(overviewsQueryRewriter, queryRewriteData, {
+                bbox: params.bbox
+            });
+            dataview = dataviewFactory.getDataview(query, dataviewDefinition);
             var ownFilter = +params.own_filter;
-            var noFilters = +params.no_filters;
-            if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) {
-                err = new Error();
-                err.message = 'Both own_filter and no_filters cannot be sent in the same request';
-                err.type = 'dataview';
-                err.http_status = 400;
+            overrideParams = getOverrideParams(params, !!ownFilter);
+        } catch (error) {
+            return callback(error);
+        }
+
+        dataview.getResult(pg, overrideParams, function (err, dataviewResult, stats = {}) {
+            if (err) {
                 return callback(err);
             }
 
-            var pg = new PSQL(dbParamsFromReqParams(params));
-
-            var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters);
-            if (params.bbox) {
-                var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
-                query = bboxFilter.sql(query);
-            }
-
-            var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
-
-            var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
-                overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
-            );
-
-            var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
-            dataview.getResult(pg, getOverrideParams(params, !!ownFilter), this);
-        },
-        function returnCallback(err, result) {
-            return callback(err, result);
-        }
-    );
+            return callback(null, dataviewResult, stats);
+        });
+    });
 };
 
+function validFilterParams (params) {
+    var ownFilter = +params.own_filter;
+    var noFilters = +params.no_filters;
+
+    return !(Number.isFinite(ownFilter) && Number.isFinite(noFilters));
+}
+
+function getQueryWithFilters (dataviewDefinition, params) {
+    var ownFilter = +params.own_filter;
+    var noFilters = +params.no_filters;
+    var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters);
+
+    if (params.bbox) {
+        var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
+        query = bboxFilter.sql(query);
+    }
+
+    return query;
+}
+
 function getDataviewQuery(dataviewDefinition, ownFilter, noFilters) {
     if (noFilters) {
         return dataviewDefinition.sql.no_filters;
@@ -129,41 +152,56 @@ function getOverrideParams(params, ownFilter) {
 }
 
 DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) {
-    step(
-        function getMapConfig() {
-            mapConfigProvider.getMapConfig(this);
-        },
-        function runDataviewSearchQuery(err, mapConfig) {
-            assert.ifError(err);
-
-            var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
-            if (!dataviewDefinition) {
-                throw new Error("Dataview '" + dataviewName + "' does not exists");
-            }
-
-            var pg = new PSQL(dbParamsFromReqParams(params));
-
-            var ownFilter = +params.own_filter;
-            ownFilter = !!ownFilter;
-
-            var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
-
-            if (params.bbox) {
-                var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
-                query = bboxFilter.sql(query);
-            }
-
-            var userQuery = params.q;
-
-            var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
-            dataview.search(pg, userQuery, this);
-        },
-        function returnCallback(err, result) {
-            return callback(err, result);
+    mapConfigProvider.getMapConfig(function (err, mapConfig) {
+        if (err) {
+            return callback(err);
         }
-    );
+
+        var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
+        if (!dataviewDefinition) {
+            const error = new Error(`Dataview '${dataviewName}' does not exist`);
+            error.type = 'dataview';
+            error.http_status = 400;
+            return callback(error);
+        }
+
+        var pg;
+        var query;
+        var dataview;
+        var userQuery = params.q;
+
+        try {
+            pg = new PSQL(dbParamsFromReqParams(params));
+            query = getQueryWithOwnFilters(dataviewDefinition, params);
+            dataview = DataviewFactory.getDataview(query, dataviewDefinition);
+        } catch (error) {
+            return callback(error);
+        }
+
+        dataview.search(pg, userQuery, function (err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            return callback(null, result);
+        });
+    });
 };
 
+function getQueryWithOwnFilters (dataviewDefinition, params) {
+    var ownFilter = +params.own_filter;
+    ownFilter = !!ownFilter;
+
+    var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
+
+    if (params.bbox) {
+        var bboxFilter = new BBoxFilter({ column: 'the_geom', srid: 4326 }, { bbox: params.bbox });
+        query = bboxFilter.sql(query);
+    }
+
+    return query;
+}
+
 function getDataviewDefinition(mapConfig, dataviewName) {
     var dataviews = mapConfig.dataviews || {};
     return dataviews[dataviewName];

lib/cartodb/backends/filter-stats.js

@@ -1,5 +1,6 @@
+'use strict';
+
 var _ = require('underscore');
-var step = require('step');
 var AnalysisFilter = require('../models/filter/analysis');
 
 function FilterStatsBackends(pgQueryRunner) {
@@ -24,36 +25,29 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) {
 }
 
 FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
-  var stats = {};
-  var self = this;
-  step(
-      function getUnfilteredRows() {
-          getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this);
-      },
-      function receiveUnfilteredRows(err, rows) {
-          if (err){
-              callback(err);
-              return;
-          }
-          stats.unfiltered_rows = rows;
-          this(null, rows);
-      },
-      function getFilteredRows() {
-          if ( filters && !_.isEmpty(filters)) {
-              var analysisFilter = new AnalysisFilter(filters);
-              var query = analysisFilter.sql(unfiltered_query);
-              getEstimatedRows(self.pgQueryRunner, username, query, this);
-          } else {
-              this(null, null);
-          }
-      },
-      function receiveFilteredRows(err, rows) {
-          if (err){
-              callback(err);
-              return;
-          }
-          stats.filtered_rows = rows;
-          callback(null, stats);
-      }
-  );
+    var stats = {};
+
+    getEstimatedRows(this.pgQueryRunner, username, unfiltered_query, (err, rows) => {
+        if (err){
+            return callback(err);
+        }
+
+        stats.unfiltered_rows = rows;
+
+        if (!filters || _.isEmpty(filters)) {
+            return callback(null, stats);
+        }
+
+        var analysisFilter = new AnalysisFilter(filters);
+        var query = analysisFilter.sql(unfiltered_query);
+
+        getEstimatedRows(this.pgQueryRunner, username, query, (err, rows) => {
+            if (err){
+                return callback(err);
+            }
+
+            stats.filtered_rows = rows;
+            return callback(null, stats);
+        });
+    });
 };

lib/cartodb/backends/layer-stats/empty-layer-stats.js

@@ -1,3 +1,5 @@
+'use strict';
+
 function EmptyLayerStats(types) {
     this._types = types || {};
 }

lib/cartodb/backends/layer-stats/factory.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var LayerStats = require('./layer-stats');
 var EmptyLayerStats = require('./empty-layer-stats');
 var MapnikLayerStats = require('./mapnik-layer-stats');

lib/cartodb/backends/layer-stats/layer-stats.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var queue = require('queue-async');
 
 function LayerStats(layerStatsIterator) {

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -1,5 +1,8 @@
+'use strict';
+
 const queryUtils = require('../../utils/query-utils');
 const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig');
+const aggregationQuery = require('../../models/aggregation/aggregation-query');
 
 function MapnikLayerStats () {
     this._types = {
@@ -19,6 +22,9 @@ function columnAggregations(field) {
     if (field.type === 'date') { // TODO other types too?
         return ['min', 'max'];
     }
+    if (field.type === 'timeDimension') {
+        return ['min', 'max'];
+    }
     return [];
 }
 
@@ -67,13 +73,13 @@ function _geometryType(ctx) {
         const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn();
         const sqlQuery = _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn));
         return queryUtils.queryPromise(ctx.dbConnection, sqlQuery)
-            .then(res => ({ geometryType: res.rows[0].geom_type }));
+            .then(res => ({ geometryType: (res.rows[0] || {}).geom_type }));
     }
     return Promise.resolve();
 }
 
 function _columns(ctx) {
-    if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) {
+    if (ctx.metaOptions.columns || ctx.metaOptions.columnStats || ctx.metaOptions.dimensions) {
         // note: post-aggregation columns are in layer.options.columns when aggregation is present
         return queryUtils.queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0)))
             .then(res => formatResultFields(ctx.dbConnection, res.fields));
@@ -137,51 +143,89 @@ function _sample(ctx, numRows) {
     return Promise.resolve();
 }
 
-function _columnStats(ctx, columns) {
+function _columnsMetadataRequired(options) {
+    // We need determine the columns of a query
+    // if either column stats or dimension stats are required,
+    // since we'll ultimately use the same query to fetch both
+    return options.columnStats || options.dimensions;
+}
+
+function _columnStats(ctx, columns, dimensions) {
     if (!columns) {
         return Promise.resolve();
     }
-    if (ctx.metaOptions.columnStats) {
+    if (_columnsMetadataRequired(ctx.metaOptions)) {
         let queries = [];
         let aggr = [];
-        queries.push(new Promise(resolve => resolve(columns))); // add columns as first result
-        Object.keys(columns).forEach(name => {
-            aggr = aggr.concat(
-                columnAggregations(columns[name])
-                .map(fn => `${fn}(${name}) AS ${name}_${fn}`)
-            );
-            if (columns[name].type === 'string') {
-                const topN = ctx.metaOptions.columnStats.topCategories || 1024;
-                const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ?
-                    ctx.metaOptions.columnStats.includeNulls :
-                    true;
-
-                // TODO:  ctx.metaOptions.columnStats.maxCategories
-                //        => use PG stats to dismiss columns with more distinct values
-                queries.push(
-                    queryUtils.queryPromise(
-                        ctx.dbConnection,
-                        _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls))
-                    ).then(res => ({ [name]: { categories: res.rows } }))
+        if (ctx.metaOptions.columnStats) {
+            queries.push(new Promise(resolve => resolve({ columns }))); // add columns as first result
+            Object.keys(columns).forEach(name => {
+                aggr = aggr.concat(
+                    columnAggregations(columns[name])
+                    .map(fn => `${fn}("${name}") AS "${name}_${fn}"`)
                 );
-            }
-        });
+                if (columns[name].type === 'string') {
+                    const topN = ctx.metaOptions.columnStats.topCategories || 1024;
+                    const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ?
+                        ctx.metaOptions.columnStats.includeNulls :
+                        true;
+
+                    // TODO:  ctx.metaOptions.columnStats.maxCategories
+                    //        => use PG stats to dismiss columns with more distinct values
+                    queries.push(
+                        queryUtils.queryPromise(
+                            ctx.dbConnection,
+                            _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls))
+                        ).then(res => ({ columns: { [name]: { categories: res.rows } } }))
+                    );
+                }
+            });
+        }
+        const dimensionsStats = {};
+        let dimensionsInfo = {};
+        if (ctx.metaOptions.dimensions && dimensions) {
+            dimensionsInfo = aggregationQuery.infoForOptions({ dimensions });
+            Object.keys(dimensionsInfo).forEach(dimName => {
+                const info = dimensionsInfo[dimName];
+                if (info.type === 'timeDimension') {
+                    dimensionsStats[dimName] = {
+                        params: info.params
+                    };
+                    aggr = aggr.concat(
+                        columnAggregations(info).map(fn => `${fn}(${info.sql}) AS "${dimName}_${fn}"`)
+                    );
+                }
+            });
+        }
         queries.push(
             queryUtils.queryPromise(
                 ctx.dbConnection,
                 _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`)
             ).then(res => {
-                let stats = {};
+                let stats = { columns: {}, dimensions: {} };
                 Object.keys(columns).forEach(name => {
-                    stats[name] = {};
+                    stats.columns[name] = {};
                     columnAggregations(columns[name]).forEach(fn => {
-                        stats[name][fn] = res.rows[0][`${name}_${fn}`];
+                        stats.columns[name][fn] = res.rows[0][`${name}_${fn}`];
                     });
                 });
+                Object.keys(dimensionsInfo).forEach(name => {
+                    stats.dimensions[name] = stats.dimensions[name] || Object.assign({}, dimensionsStats[name]);
+                    let type = null;
+                    columnAggregations(dimensionsInfo[name]).forEach(fn => {
+                        type = type ||
+                            fieldTypeSafe(ctx.dbConnection, res.fields.find(f => f.name === `${name}_${fn}`));
+                        stats.dimensions[name][fn] = res.rows[0][`${name}_${fn}`];
+                    });
+                    stats.dimensions[name].type = type;
+                });
                 return stats;
             })
         );
-        return Promise.all(queries).then(results => ({ columns: mergeColumns(results) }));
+        return Promise.all(queries).then(results => ({
+            columns: mergeColumns(results.map(r => r.columns)),
+            dimensions: mergeColumns(results.map( r => r.dimensions))
+        }));
     }
     return Promise.resolve({ columns });
 }
@@ -211,19 +255,17 @@ function fieldType(cname) {
     return tname;
 }
 
+function fieldTypeSafe(dbConnection, field) {
+    const cname = dbConnection.typeName(field.dataTypeID);
+    return cname ? fieldType(cname) : `unknown(${field.dataTypeID})`;
+}
+
 // columns are returned as an object { columnName1: { type1: ...}, ..}
 // for consistency with SQL API
 function formatResultFields(dbConnection, fields = []) {
     let nfields = {};
     for (let field of fields) {
-      const cname = dbConnection.typeName(field.dataTypeID);
-      let tname;
-      if ( ! cname ) {
-        tname = 'unknown(' + field.dataTypeID + ')';
-      } else {
-        tname = fieldType(cname);
-      }
-      nfields[field.name] = { type: tname };
+      nfields[field.name] = { type: fieldTypeSafe(dbConnection, field) };
     }
     return nfields;
 }
@@ -237,7 +279,7 @@ function (layer, dbConnection, callback) {
         dbConnection,
         preQuery,
         aggrQuery,
-        metaOptions: layer.options.metadata || {}
+        metaOptions: layer.options.metadata || {},
     };
 
     // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere
@@ -248,6 +290,8 @@ function (layer, dbConnection, callback) {
     // TODO: add support for sample.exclude option by, in that case, forcing the columns query and
     // passing the results to the sample query function.
 
+    const dimensions = (layer.options.aggregation || {}).dimensions;
+
     Promise.all([
         _estimatedFeatureCount(ctx).then(
             ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount)
@@ -256,9 +300,10 @@ function (layer, dbConnection, callback) {
         _featureCount(ctx),
         _aggrFeatureCount(ctx),
         _geometryType(ctx),
-        _columns(ctx).then(columns => _columnStats(ctx, columns))
+        _columns(ctx).then(columns => _columnStats(ctx, columns, dimensions))
     ]).then(results => {
-        callback(null, mergeResults(results));
+        results = mergeResults(results);
+        callback(null, results);
     }).catch(error => {
         callback(error);
     });

lib/cartodb/backends/layer-stats/torque-layer-stats.js

@@ -1,3 +1,5 @@
+'use strict';
+
 function TorqueLayerStats() {
     this._types = {
         torque: true

lib/cartodb/backends/overviews-metadata.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const queryUtils = require('../utils/query-utils');
 
 function OverviewsMetadataBackend(pgQueryRunner) {

lib/cartodb/backends/pg_connection.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var PSQL = require('cartodb-psql');
 var _ = require('underscore');
 const debug = require('debug')('cachechan');

lib/cartodb/backends/pg_query_runner.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var PSQL = require('cartodb-psql');
 
 function PgQueryRunner(pgConnection) {

lib/cartodb/backends/stats.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var layerStats = require('./layer-stats/factory');
 
 function StatsBackend() {

lib/cartodb/backends/tables-extent.js

@@ -1,3 +1,5 @@
+'use strict';
+
 function TablesExtentBackend(pgQueryRunner) {
     this.pgQueryRunner = pgQueryRunner;
 }

lib/cartodb/backends/template_maps.js

@@ -1,7 +1,7 @@
-var assert = require('assert');
+'use strict';
+
 var crypto = require('crypto');
 var debug = require('debug')('windshaft:templates');
-var step = require('step');
 var _ = require('underscore');
 var dot = require('dot');
 
@@ -69,27 +69,19 @@ TemplateMaps.prototype._userTemplateLimit = function() {
  * @param callback - function to pass results too.
  */
 TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
-  var redisClient;
-  var that = this;
-  var db = that.db_signatures;
+    this.redis_pool.acquire(this.db_signatures, (err, redisClient) => {
+        if (err) {
+            return callback(err);
+        }
 
-  step(
-    function getRedisClient() {
-      that.redis_pool.acquire(db, this);
-    },
-    function executeQuery(err, data) {
-      assert.ifError(err);
-      redisClient = data;
-      redisArgs.push(this);
-      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
-    },
-    function releaseRedisClient(err, data) {
-      if ( ! _.isUndefined(redisClient) ) {
-          that.redis_pool.release(db, redisClient);
-      }
-      callback(err, data);
-    }
-  );
+        redisClient[redisFunc.toUpperCase()](...redisArgs, (err, data) => {
+            this.redis_pool.release(this.db_signatures, redisClient);
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, data);
+        });
+    });
 };
 
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
@@ -184,6 +176,37 @@ function templateDefaults(template) {
     });
 }
 
+/**
+ * Checks if the if the user reaches the templetes limit
+ *
+ * @param userTemplatesKey user templat key in Redis
+ * @param owner cartodb username of the template owner
+ * @param callback returns error if the user reaches the limit
+ */
+TemplateMaps.prototype._checkUserTemplatesLimit = function(userTemplatesKey, owner, callback) {
+    const limit = this._userTemplateLimit();
+
+    if(!limit) {
+        return callback();
+    }
+
+    this._redisCmd('HLEN', [userTemplatesKey], (err, numberOfTemplates) => {
+        if (err) {
+            return callback(err);
+        }
+
+        if (numberOfTemplates >= limit) {
+            const limitReachedError = new Error(
+                `User '${owner}' reached limit on number of templates (${numberOfTemplates}/${limit})`
+            );
+            limitReachedError.http_status = 409;
+            return callback(limitReachedError);
+        }
+
+        return callback();
+    });
+};
+
 //--------------- PUBLIC API -------------------------------------
 
 // Add a template
@@ -199,52 +222,41 @@ function templateDefaults(template) {
 //        Return template identifier (only valid for given user)
 //
 TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
-    var self = this;
-
     template = templateDefaults(template);
 
     var invalidError = this._checkInvalidTemplate(template);
-    if ( invalidError ) {
+    if (invalidError) {
         return callback(invalidError);
     }
 
-    var templateName = template.name;
-    var userTemplatesKey = this.key_usr_tpl({ owner:owner });
-    var limit = this._userTemplateLimit();
+    var userTemplatesKey = this.key_usr_tpl({ owner });
 
-    step(
-        function checkLimit() {
-            if ( ! limit ) {
-                return 0;
-            }
-            self._redisCmd('HLEN', [ userTemplatesKey ], this);
-        },
-        function installTemplateIfDoesNotExist(err, numberOfTemplates) {
-            assert.ifError(err);
-            if ( limit && numberOfTemplates >= limit ) {
-                var limitReachedError = new Error("User '" + owner + "' reached limit on number of templates (" +
-                    numberOfTemplates + "/" + limit + ")");
-                limitReachedError.http_status = 409;
-                throw limitReachedError;
-            }
-            self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
-        },
-        function validateInstallation(err, wasSet) {
-            assert.ifError(err);
-            if ( ! wasSet ) {
-                throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists");
-            }
-
-            return true;
-        },
-        function finish(err) {
-            if (!err) {
-                self.emit('add', owner, templateName, template);
-            }
-
-            callback(err, templateName, template);
+    this._checkUserTemplatesLimit(userTemplatesKey, owner, err => {
+        if (err) {
+            return callback(err);
         }
-  );
+
+        let templateString;
+        try {
+            templateString = JSON.stringify(template);
+        } catch (error) {
+            return callback(error);
+        }
+
+        this._redisCmd('HSETNX', [userTemplatesKey, template.name, templateString], (err, wasSet) => {
+            if (err) {
+                return callback(err);
+            }
+
+            if (!wasSet) {
+                var templateExistsError = new Error(`Template '${template.name}' of user '${owner}' already exists`);
+                return callback(templateExistsError);
+            }
+
+            this.emit('add', owner, template.name, template);
+            return callback(null, template.name, template);
+        });
+    });
 };
 
 // Delete a template
@@ -257,26 +269,18 @@ TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
 // @param callback function(err)
 //
 TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
-    var self = this;
-    step(
-        function deleteTemplate() {
-            self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this);
-        },
-        function handleDeletion(err, deleted) {
-            assert.ifError(err);
-            if (!deleted) {
-                throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
-            }
-            return true;
-        },
-        function finish(err) {
-            if (!err) {
-                self.emit('delete', owner, tpl_id);
-            }
-
-            callback(err);
+    this._redisCmd('HDEL', [ this.key_usr_tpl({ owner:owner }), tpl_id ], (err, deleted) => {
+        if (err) {
+            return callback(err);
         }
-    );
+
+        if (!deleted) {
+            return callback(new Error(`Template '${tpl_id}' of user '${owner}' does not exist`));
+        }
+
+        this.emit('delete', owner, tpl_id);
+        return callback();
+    });
 };
 
 // Update a template
@@ -296,56 +300,58 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
 // @param callback function(err)
 //
 TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
-
-    var self = this;
-
     template = templateDefaults(template);
 
     var invalidError = this._checkInvalidTemplate(template);
-
-    if ( invalidError ) {
+    if (invalidError) {
         return callback(invalidError);
     }
 
-    var templateName = template.name;
-
-    if ( tpl_id !== templateName ) {
-        return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')"));
+    if (tpl_id !== template.name) {
+        return callback(new Error(`Cannot update name of a map template ('${tpl_id}' != '${template.name}')`));
     }
 
-    var userTemplatesKey = this.key_usr_tpl({ owner:owner });
+    var userTemplatesKey = this.key_usr_tpl({ owner });
 
-    var previousTemplate = null;
+    this._redisCmd('HGET', [userTemplatesKey, tpl_id], (err, beforeUpdateTemplate) => {
+        if (err) {
+            return callback(err);
+        }
 
-    step(
-        function getExistingTemplate() {
-            self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this);
-        },
-        function updateTemplate(err, _currentTemplate) {
-            assert.ifError(err);
-            if (!_currentTemplate) {
-                throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
+        if (!beforeUpdateTemplate) {
+            return callback(new Error(`Template '${tpl_id}' of user '${owner}' does not exist`));
+        }
+
+        let templateString;
+        try {
+            templateString = JSON.stringify(template);
+        } catch (error) {
+            return callback(error);
+        }
+
+        this._redisCmd('HSET', [userTemplatesKey, template.name, templateString], (err, didSetNewField) => {
+            if (err) {
+                return callback(err);
             }
-            previousTemplate = _currentTemplate;
-            self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
-        },
-        function handleTemplateUpdate(err, didSetNewField) {
-            assert.ifError(err);
+
             if (didSetNewField) {
                 debug('New template created on update operation');
             }
-            return true;
-        },
-        function finish(err) {
-            if (!err) {
-                if (self.fingerPrint(JSON.parse(previousTemplate)) !== self.fingerPrint(template)) {
-                    self.emit('update', owner, templateName, template);
-                }
+
+            let beforeUpdateTemplateObject;
+            try {
+                beforeUpdateTemplateObject = JSON.parse(beforeUpdateTemplate);
+            } catch (error) {
+                return callback(error);
             }
 
-            callback(err, template);
-        }
-    );
+            if (this.fingerPrint(beforeUpdateTemplateObject) !== this.fingerPrint(template)) {
+                this.emit('update', owner, template.name, template);
+            }
+
+            return callback(null, template);
+        });
+    });
 };
 
 // List user templates
@@ -370,19 +376,20 @@ TemplateMaps.prototype.listTemplates = function(owner, callback) {
 //        Return full template definition
 //
 TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) {
-    var self = this;
-    step(
-        function getTemplate() {
-            self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this);
-        },
-        function parseTemplate(err, tpl_val) {
-            assert.ifError(err);
-            return JSON.parse(tpl_val);
-        },
-        function finish(err, tpl) {
-            callback(err, tpl);
+    this._redisCmd('HGET', [this.key_usr_tpl({owner:owner}), tpl_id], (err, template) => {
+        if (err) {
+            return callback(err);
         }
-    );
+
+        let templateObject;
+        try {
+            templateObject = JSON.parse(template);
+        } catch (error) {
+            return callback(error);
+        }
+
+        return callback(null, templateObject);
+    });
 };
 
 TemplateMaps.prototype.isAuthorized = function(template, authTokens) {

lib/cartodb/backends/user-limits.js

@@ -1,4 +1,4 @@
-var step = require('step');
+'use strict';
 
 /**
  *
@@ -41,45 +41,38 @@ UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callba
 };
 
 UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
-    var self = this;
-
-    step(
-        function isAuthorized() {
-            var next = this;
-
-            if (!apiKey) {
-                return next(null, false);
-            }
-
-            self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
-                if (err) {
-                    return next(err);
-                }
-
-                return next(null, userApiKey === apiKey);
-            });
-        },
-        function getUserTimeoutRenderLimits(err, authorized) {
-            var next = this;
+    isAuthorized(this.metadataBackend, username, apiKey, (err, authorized) => {
+        if (err) {
+            return callback(err);
+        }
 
+        this.metadataBackend.getUserTimeoutRenderLimits(username, (err, timeoutRenderLimit) => {
             if (err) {
-                return next(err);
+                return callback(err);
             }
 
-            self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
-                if (err) {
-                    return next(err);
-                }
-
-                next(null, {
-                    render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
-                });
-            });
-        },
-        callback
-    );
+            return callback(
+                null,
+                { render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic }
+            );
+        });
+    });
 };
 
+function isAuthorized(metadataBackend, username, apiKey, callback) {
+    if (!apiKey) {
+        return callback(null, false);
+    }
+
+    metadataBackend.getUserMapKey(username, function (err, userApiKey) {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, userApiKey === apiKey);
+    });
+}
+
 UserLimitsBackend.prototype.preprareRateLimit = function () {
     if (this.options.limits.rateLimitsEnabled) {
         this.metadataBackend.loadRateLimitsScript();

lib/cartodb/cache/backend/fastly.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var FastlyPurge = require('fastly-purge');
 
 function FastlyCacheBackend(apiKey, serviceId) {

lib/cartodb/cache/backend/varnish_http.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var request = require('request');
 
 function VarnishHttpCacheBackend(host, port) {

lib/cartodb/cache/layergroup_affected_tables.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var LruCache = require('lru-cache');
 
 function LayergroupAffectedTables() {

lib/cartodb/cache/model/named_maps_entry.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var crypto = require('crypto');
 
 function NamedMaps(owner, name) {

lib/cartodb/cache/named_map_provider_cache.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var _ = require('underscore');
 var dot = require('dot');
 var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');

lib/cartodb/cache/surrogate_keys_cache.js

@@ -1,3 +1,5 @@
+'use strict';
+
 var queue = require('queue-async');
 
 /**

lib/cartodb/models/aggregation/aggregation-mapconfig.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const MapConfig = require('windshaft').model.MapConfig;
 const aggregationQuery = require('./aggregation-query');
 const aggregationValidator = require('./aggregation-validator');

lib/cartodb/models/aggregation/aggregation-query.js

@@ -1,5 +1,10 @@
+'use strict';
+
+const timeDimension = require('./time-dimension');
+
 const DEFAULT_PLACEMENT = 'point-sample';
 
+
 /**
  * Returns a template function (function that accepts template parameters and returns a string)
  * to generate an aggregation query.
@@ -24,6 +29,16 @@ const templateForOptions = (options) => {
     return templateFn;
 };
 
+function optionsToParams (options) {
+    return {
+        sourceQuery: options.query,
+        res: 256/options.resolution,
+        columns: options.columns,
+        dimensions: options.dimensions,
+        filters: options.filters
+    };
+}
+
 /**
  * Generates an aggregation query given the aggregation options:
  * - query
@@ -38,16 +53,23 @@ const templateForOptions = (options) => {
  * When placement, columns or dimensions are specified, columns are aggregated as requested
  * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
  */
-const queryForOptions = (options) => templateForOptions(options)({
-    sourceQuery: options.query,
-    res: 256/options.resolution,
-    columns: options.columns,
-    dimensions: options.dimensions,
-    filters: options.filters
-});
+const queryForOptions = (options) => templateForOptions(options)(optionsToParams(options));
 
 module.exports = queryForOptions;
 
+module.exports.infoForOptions = (options) => {
+    const params = optionsToParams(options);
+    const dimensions = {};
+    dimensionNamesAndExpressions(params).forEach(([dimensionName, info]) => {
+        dimensions[dimensionName] = {
+            sql: info.sql,
+            params: info.effectiveParams,
+            type: info.type
+        };
+    });
+    return dimensions;
+};
+
 const SUPPORTED_AGGREGATE_FUNCTIONS = {
     'count': {
         sql: (column_name, params) => `count(${params.aggregated_column || '*'})`
@@ -113,22 +135,56 @@ const aggregateColumnDefs = ctx => {
 
 const aggregateDimensions = ctx => ctx.dimensions || {};
 
-const dimensionNames = (ctx, table) => {
-    let dimensions = aggregateDimensions(ctx);
-    if (table) {
-        return sep(Object.keys(dimensions).map(
-            dimension_name => `${table}.${dimension_name}`
-        ));
+const timeDimensionParameters = definition => {
+    // definition.column should correspond to a wrapped date column
+    const group = definition.group || {};
+    return {
+        time: `to_timestamp("${definition.column}")`,
+        timezone: group.timezone || 'utc',
+        units: group.units,
+        count: group.count || 1,
+        starting: group.starting,
+        format: definition.format
+    };
+};
+
+// Adapt old-style dimension definitions for backwards compatibility
+const adaptDimensionDefinition = definition => {
+    if (typeof(definition) === 'string') {
+        return { column: definition };
     }
-    return sep(Object.keys(dimensions));
+    return definition;
+};
+
+const dimensionExpression = definition => {
+    if (definition.group) {
+        // Currently only time dimensions are supported with parameters
+        return Object.assign({ type: 'timeDimension' }, timeDimension(timeDimensionParameters(definition)));
+    } else {
+        return { sql: `"${definition.column}"` };
+    }
+};
+
+const dimensionNamesAndExpressions = (ctx) => {
+    let dimensions = aggregateDimensions(ctx);
+    return Object.keys(dimensions).map(dimensionName => {
+        const dimension = adaptDimensionDefinition(dimensions[dimensionName]);
+        const expression = dimensionExpression(dimension);
+        return [dimensionName, expression];
+    });
+};
+
+const dimensionNames = (ctx, table) => {
+    return sep(dimensionNamesAndExpressions(ctx).map(([dimensionName]) => {
+        return table ? `${table}."${dimensionName}"` : `"${dimensionName}"`;
+    }));
 };
 
 const dimensionDefs = ctx => {
-    let dimensions = aggregateDimensions(ctx);
-    return sep(Object.keys(dimensions).map(dimension_name => {
-        const expression = dimensions[dimension_name];
-        return `${expression} AS ${dimension_name}`;
-    }));
+    return sep(
+        dimensionNamesAndExpressions(ctx)
+        .map(([dimensionName, expression]) => `${expression.sql} AS "${dimensionName}"`)
+    );
 };
 
 const aggregateFilters = ctx => ctx.filters || {};
@@ -274,23 +330,24 @@ const spatialFilter = `
 // * This queries are used for rendering and the_geom is omitted in the results for better performance
 // * If the MVT extent or tile buffer was 0 or a multiple of the resolution we could use directly
 //   the bbox for them, but in general we need to find the nearest cell limits inside the bbox.
-// * bbox coordinates can have an error in the last digites; we apply a small correction before
-//   applying CEIL or FLOOR to compensate for this.
+// * bbox coordinates can have an error in the last digits; we apply a small correction before
+//   applying CEIL or FLOOR to compensate for this, so that coordinates closer than a small (`eps`)
+//   fraction of the cell size to a cell limit are moved to the exact limit.
 const sqlParams = (ctx) => `
 _cdb_res AS (
     SELECT
     ${gridResolution(ctx)} AS res,
     !bbox! AS bbox,
-    (2*2.220446049250313e-16::double precision) AS eps
+    (1E-6::double precision) AS eps
 ),
 _cdb_params AS (
     SELECT
       res,
       bbox,
-      CEIL((ST_XMIN(bbox) - eps*ABS(ST_XMIN(bbox)))/res)*res AS xmin,
-      FLOOR((ST_XMAX(bbox) + eps*ABS(ST_XMAX(bbox)))/res)*res AS xmax,
-      CEIL((ST_YMIN(bbox) - eps*ABS(ST_YMIN(bbox)))/res)*res AS ymin,
-      FLOOR((ST_YMAX(bbox) + eps*ABS(ST_YMAX(bbox)))/res)*res AS ymax
+      CEIL((ST_XMIN(bbox) - eps*res)/res)*res AS xmin,
+      FLOOR((ST_XMAX(bbox) + eps*res)/res)*res AS xmax,
+      CEIL((ST_YMIN(bbox) - eps*res)/res)*res AS ymin,
+      FLOOR((ST_YMAX(bbox) + eps*res)/res)*res AS ymax
       FROM _cdb_res
 )
 `;
@@ -379,7 +436,7 @@ const aggregationQueryTemplates = {
         )
         SELECT
             _cdb_clusters.cartodb_id,
-            the_geom, the_geom_webmercator
+            the_geom_webmercator
             ${dimensionNames(ctx, '_cdb_clusters')}
             ${aggregateColumnNames(ctx, '_cdb_clusters')}
         FROM

lib/cartodb/models/aggregation/aggregation-validator.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = function aggregationValidator (mapconfig) {
     return function validateProperty (key, validator) {
         for (let index = 0; index < mapconfig.getLayers().length; index++) {

lib/cartodb/models/aggregation/time-dimension.js

@@ -0,0 +1,267 @@
+'use strict';
+
+// timezones can be defined either by an numeric offset in seconds or by
+// a valid (case-insensitive) tz/PG name;
+// they include abbreviations defined by PG (which have precedence and
+// are fixed offsets, not handling DST) or general names that can handle DST.
+function timezone(tz) {
+    if (isFinite(tz)) {
+        return `INTERVAL '${tz} seconds'`;
+    }
+    return `'${tz}'`;
+}
+
+// We assume t is a TIMESTAMP WITH TIME ZONE.
+// If this was to be used with a t which is a TIMESTAMP or TIME (no time zone)
+// it should be converted with `timezone('utc',t)` to a type with time zone.
+// Note that by default CARTO uses timestamp with time zone columns for dates
+// and VectorMapConfigAdapter converts them to epoch numbers.
+// So, for using this with aggregations, relying on dates & times
+// converted to UTC UNIX epoch numbers, apply `to_timestamp` to the
+// (converted) column.
+function timeExpression(t, tz) {
+   if (tz !== undefined) {
+        return `timezone(${timezone(tz)}, ${t})`;
+   }
+   return t;
+}
+
+function epochWithDefaults(epoch) {
+    /* jshint maxcomplexity:9 */ // goddammit linter, I like this as is!!
+    const format = /^(\d\d\d\d)(?:\-?(\d\d)(?:\-?(\d\d)(?:[T\s]?(\d\d)(?:(\d\d)(?:\:(\d\d))?)?)?)?)?$/;
+    const match  = (epoch || '').match(format) || [];
+    const year   = match[1] || '0001';
+    const month  = match[2] || '01';
+    const day    = match[3] || '01';
+    const hour   = match[4] || '00';
+    const minute = match[5] || '00';
+    const second = match[6] || '00';
+    return `${year}-${month}-${day}T${hour}:${minute}:${second}`;
+}
+
+// Epoch should be an ISO timestamp literal without time zone
+// (it is interpreted as in the defined timzezone for the input time)
+// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc.
+// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00
+// It returns a timestamp without time zone
+function epochExpression(epoch) {
+    return `TIMESTAMP '${epoch}'`;
+ }
+
+const YEARSPAN = "(date_part('year', $t)-date_part('year', $epoch))";
+// Note that SECONDSPAN is not a UTC epoch, but an epoch in the specified TZ,
+// so we can use it to compute any multiple of seconds with it  without using date_part or date_trunc
+const SECONDSPAN = "(date_part('epoch', $t) - date_part('epoch', $epoch))";
+
+const serialParts = {
+    second: {
+        sql: `FLOOR(${SECONDSPAN})`,
+        zeroBased: true
+    },
+    minute: {
+        sql: `FLOOR(${SECONDSPAN}/60)`,
+        zeroBased: true
+    },
+    hour: {
+        sql: `FLOOR(${SECONDSPAN}/3600)`,
+        zeroBased: true
+    },
+    day: {
+        sql: `1 + FLOOR(${SECONDSPAN}/86400)`,
+        zeroBased: false
+    },
+    week: {
+        sql: `1 + FLOOR(${SECONDSPAN}/(7*86400))`,
+        zeroBased: false
+    },
+    month: {
+        sql: `1 + date_part('month', $t) - date_part('month', $epoch) + 12*${YEARSPAN}`,
+        zeroBased: false
+    },
+    quarter: {
+        sql: `1 + date_part('quarter', $t) - date_part('quarter', $epoch) + 4*${YEARSPAN}`,
+        zeroBased: false
+    },
+    semester: {
+        sql: `1 + FLOOR((date_part('month', $t) - date_part('month', $epoch))/6) + 2*${YEARSPAN}`,
+        zeroBased: false
+    },
+    trimester: {
+        sql: `1 + FLOOR((date_part('month', $t) - date_part('month', $epoch))/4) + 3*${YEARSPAN}`,
+        zeroBased: false
+    },
+    year: {
+        // for the default epoch this coincides with date_part('year', $t)
+        sql: `1 + ${YEARSPAN}`,
+        zeroBased: false
+    },
+    decade: {
+        // for the default epoch this coincides with date_part('decade', $t)
+        sql: `FLOOR((${YEARSPAN} + 1)/10)`,
+        zeroBased: true
+    },
+    century: {
+        // for the default epoch this coincides with date_part('century', $t)
+        sql: `1 + FLOOR(${YEARSPAN}/100)`,
+        zeroBased: false
+    },
+    millennium: {
+        // for the default epoch this coincides with date_part('millennium', $t)
+        sql: `1 + FLOOR(${YEARSPAN}/1000)`,
+        zeroBased: false
+    }
+};
+
+function serialSqlExpr(params) {
+    const { sql, zeroBased } = serialParts[params.units];
+    const column = timeExpression(params.time, params.timezone);
+    const epoch  = epochExpression(params.starting);
+    const serial = sql.replace(/\$t/g, column).replace(/\$epoch/g, epoch);
+    let expr = serial;
+    if (params.count !== 1) {
+        if (zeroBased) {
+            expr = `FLOOR((${expr})/(${params.count}::double precision))::int`;
+        } else {
+            expr = `CEIL((${expr})/(${params.count}::double precision))::int`;
+        }
+    } else {
+        expr = `(${expr})::int`;
+    }
+    return expr;
+}
+
+const isoParts = {
+    second: `to_char($t, 'YYYY-MM-DD"T"HH24:MI:SS')`,
+    minute: `to_char($t, 'YYYY-MM-DD"T"HH24:MI')`,
+    hour: `to_char($t, 'YYYY-MM-DD"T"HH24')`,
+    day: `to_char($t, 'YYYY-MM-DD')`,
+    month: `to_char($t, 'YYYY-MM')`,
+    year: `to_char($t, 'YYYY')`,
+    week: `to_char($t, 'IYYY-"W"IW')`,
+    quarter: `to_char($t, 'YYYY-"Q"Q')`,
+    semester: `to_char($t, 'YYYY"S"') || to_char(CEIL(date_part('month', $t)/6), '9')`,
+    trimester: `to_char($t, 'YYYY"t"') || to_char(CEIL(date_part('month', $t)/4), '9')`,
+    decade: `to_char(date_part('decade', $t), '"D"999')`,
+    century: `to_char($t, '"C"CC')`,
+    millennium: `to_char(date_part('millennium', $t), '"M"999')`
+};
+
+function isoSqlExpr(params) {
+    const column = timeExpression(params.time, params.timezone);
+    if (params.count > 1) {
+        // TODO: it would be sensible to return the ISO of the first unit in the period
+        throw new Error('Multiple time units not supported for ISO format');
+    }
+    return isoParts[params.units].replace(/\$t/g, column);
+}
+
+const cyclicParts = {
+    dayOfWeek: `date_part('isodow', $t)`,                         // 1 = monday to 7 = sunday;
+    dayOfMonth: `date_part('day', $t)`,                           // 1 to 31
+    dayOfYear: `date_part('doy', $t)`,                            // 1 to 366
+    hourOfDay: `date_part('hour', $t)`,                           // 0 to 23
+    monthOfYear: `date_part('month', $t)`,                        // 1 to 12
+    quarterOfYear: `date_part('quarter', $t)`,                    // 1 to 4
+    semesterOfYear: `FLOOR((date_part('month', $t)-1)/6.0) + 1`,  // 1 to 2
+    trimesterOfYear: `FLOOR((date_part('month', $t)-1)/4.0) + 1`, // 1 to 3
+    weekOfYear: `date_part('week', $t)`,                          // 1 to 53
+    minuteOfHour: `date_part('minute', $t)`                       // 0 to 59
+};
+
+function cyclicSqlExpr(params) {
+    const column = timeExpression(params.time, params.timezone);
+    return cyclicParts[params.units].replace(/\$t/g, column);
+}
+
+const ACCEPTED_PARAMETERS = ['time', 'units', 'timezone', 'count', 'starting', 'format'];
+const REQUIRED_PARAMETERS = ['time', 'units'];
+
+function validateParameters(params, checker) {
+    const errors = [];
+    const presentParams = Object.keys(params);
+    const invalidParams = presentParams.filter(param => !ACCEPTED_PARAMETERS.includes(param));
+    if (invalidParams.length) {
+        errors.push(`Invalid parameters: ${invalidParams.join(', ')}`);
+    }
+    const missingParams = REQUIRED_PARAMETERS.filter(param => !presentParams.includes(param));
+    if (missingParams.length) {
+        errors.push(`Missing parameters: ${missingParams.join(', ')}`);
+    }
+    const params_errors = checker(params);
+    errors.push(...params_errors.errors);
+    if (errors.length) {
+        throw new Error(`Invalid time dimension:\n${errors.join("\n")}`);
+    }
+    return params_errors.params;
+}
+
+const VALID_CYCLIC_UNITS = Object.keys(cyclicParts);
+const VALID_SERIAL_UNITS = Object.keys(serialParts);
+const VALID_ISO_UNITS = Object.keys(isoParts);
+
+function cyclicCheckParams(params) {
+    const errors = [];
+    if (!VALID_CYCLIC_UNITS.includes(params.units)) {
+        errors.push(`Invalid units "${params.units}"`);
+    }
+    if (params.count && params.count > 1) {
+        errors.push(`Count ${params.count} not supported for cyclic ${params.units}`);
+    }
+    return { errors: errors, params: params };
+}
+
+function serialCheckParams(params) {
+    const errors = [];
+    if (!VALID_SERIAL_UNITS.includes(params.units)) {
+        errors.push(`Invalid grouping units "${params.units}"`);
+    }
+    return { errors: errors, params: Object.assign({}, params, { starting: epochWithDefaults(params.starting) }) };
+}
+
+function isoCheckParams(params) {
+    const errors = [];
+    if (!VALID_ISO_UNITS.includes(params.units)) {
+        errors.push(`Invalid units "${params.units}"`);
+    }
+    if (params.starting) {
+        errors.push("Parameter 'starting' not supported for ISO format");
+    }
+    return { errors: errors, params: params };
+}
+
+const CLASSIFIERS = {
+    cyclic: {
+        sqlExpr: cyclicSqlExpr,
+        checkParams: cyclicCheckParams
+    },
+    iso: {
+        sqlExpr: isoSqlExpr,
+        checkParams: isoCheckParams
+    },
+    serial: {
+        sqlExpr: serialSqlExpr,
+        checkParams: serialCheckParams
+    }
+};
+
+function isCyclic(units) {
+    return VALID_CYCLIC_UNITS.includes(units);
+}
+
+function classifierFor(params) {
+    let classifier = 'serial';
+    if (params.units && isCyclic(params.units)) {
+        classifier = 'cyclic';
+    } else if (params.format === 'iso') {
+        classifier = 'iso';
+    }
+    return CLASSIFIERS[classifier];
+}
+
+function classificationSql(params) {
+    const classifier = classifierFor(params);
+    params = validateParameters(params, classifier.checkParams);
+    return { sql: classifier.sqlExpr(params), effectiveParams: params };
+}
+
+module.exports = classificationSql;

lib/cartodb/models/cdb_request.js

@@ -1,3 +1,5 @@
+'use strict';
+
 function CdbRequest() {
     this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host ||
             '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"

lib/cartodb/models/dataview/aggregation.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const BaseDataview = require('./base');
 const debug = require('debug')('windshaft:dataview:aggregation');
 

lib/cartodb/models/dataview/base.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const FLOAT_OIDS = {
     700: true,
     701: true,
@@ -21,7 +23,7 @@ function getPGTypeName (pgType) {
 
 module.exports = class BaseDataview {
     getResult (psql, override, callback) {
-        this.sql(psql, override, (err, query) => {
+        this.sql(psql, override, (err, query, flags = null) => {
             if (err) {
                 return callback(err);
             }
@@ -34,8 +36,20 @@ module.exports = class BaseDataview {
                 result = this.format(result, override);
                 result.type = this.getType();
 
-                return callback(null, result);
+                //Overviews logging
+                const stats = {};
 
+                if (flags && flags.usesOverviews !== undefined) {
+                    stats.usesOverviews = flags.usesOverviews;
+                } else {
+                    stats.usesOverviews = false;
+                }
+
+                if (this.getType) {
+                    stats.dataviewType = this.getType();
+                }
+
+                return callback(null, result, stats);
             }, true); // use read-only transaction
         });
     }

lib/cartodb/models/dataview/factory.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const dataviews = require('./');
 
 module.exports = class DataviewFactory {

lib/cartodb/models/dataview/formula.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const BaseDataview = require('./base');
 const debug = require('debug')('windshaft:dataview:formula');
 const utils = require('../../utils/query-utils');

lib/cartodb/models/dataview/histogram.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const debug = require('debug')('windshaft:dataview:histogram');
 const NumericHistogram = require('./histograms/numeric-histogram');
 const DateHistogram = require('./histograms/date-histogram');

lib/cartodb/models/dataview/histograms/base-histogram.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const BaseDataview = require('../base');
 
 const TYPE = 'histogram';

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const BaseHistogram = require('./base-histogram');
 const debug = require('debug')('windshaft:dataview:date-histogram');
 const utils = require('../../../utils/query-utils');

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const BaseHistogram = require('./base-histogram');
 const debug = require('debug')('windshaft:dataview:numeric-histogram');
 const utils = require('../../../utils/query-utils');

lib/cartodb/models/dataview/index.js

@@ -1,3 +1,5 @@
+'use strict';
+
 module.exports = {
     Aggregation: require('./aggregation'),
     Formula: require('./formula'),