Release 5.2.0

Add Mapnik 'cache-features' option

.travis.yml

@@ -5,10 +5,10 @@ services:
   - docker
 
 before_install:
-  - docker pull cartoimages/windshaft-carto-testing
+  - docker pull cartoimages/windshaft-testing
 
 script:
-  - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing
+  - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh
 
 language: generic
 

NEWS.md

@@ -1,5 +1,60 @@
 # Changelog
 
+## 5.2.0
+Released 2018-02-01
+
+Announcements:
+ - Upgrade windshaft to [4.3.3](https://github.com/CartoDB/windshaft/releases/tag/4.3.2) adding support for cache-features' in Mapnik/CartoDB layers.
+
+## 5.1.0
+Released 2018-01-30
+New features:
+ - Now mapnik has support for fine-grained metrics.
+ - Variables can be passed for later substitution in postgis datasource.
+
+Announcements:
+ - Upgrade windshaft to [4.3.1](https://github.com/CartoDB/windshaft/releases/tag/4.3.1). Underneath it upgrades mapnik and all the related dependencies.
+
+## 5.0.1
+Released 2018-01-29
+
+Bug Fixes:
+- Allow aggregation for queries with no the_geom (only the_geom_webmercator) #856
+
+## 5.0.0
+Released 2018-01-29
+
+Backward incompatible changes:
+- Aggregation dataview returns categories with the same type as the database type. For example, if we are aggretating by a numeric field, the resulting JSON will contain a number instead of a stringified number.
+
+## 4.8.0
+Released 2018-01-04
+
+New features:
+ - Return url template in metadata #838.
+
+Bux fixes:
+ - Tests: Order torque objects before comparison
+
+## 4.7.0
+Released 2018-01-03
+
+New features:
+ - Return tilejson in metadata #837.
+
+Bug fixes:
+ - Allow to create vector map-config for layers that doesn't have points. Layers with lines or polygons won't be aggregated by default.
+
+
+## 4.6.0
+Released 2018-01-02
+
+Announcements:
+ - Upgrades windshaft to [4.2.0](https://github.com/CartoDB/windshaft/releases/tag/4.2.0).
+ - Validate aggregation input params.
+ - Fix column names collisions in histograms [#828](https://github.com/CartoDB/Windshaft-cartodb/pull/828).
+ - Add full-sample aggregation support for vector map-config.
+
 ## 4.5.0
 Released 2017-12-19
 

app.js

@@ -4,6 +4,7 @@ var path = require('path');
 var fs = require('fs');
 var _ = require('underscore');
 var semver = require('semver');
+const setICUEnvVariable = require('./lib/cartodb/utils/icu_data_env_setter');
 
 // jshint undef:false
 var log = console.log.bind(console);
@@ -16,6 +17,9 @@ if (!semver.satisfies(nodejsVersion, '>=6.9.0')) {
     process.exit(1);
 }
 
+// This function should be called before the require('yargs').
+setICUEnvVariable();
+
 var argv = require('yargs')
     .usage('Usage: $0 <environment> [options]')
     .help('h')

config/environments/development.js.example

@@ -204,7 +204,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {

config/environments/production.js.example

@@ -198,7 +198,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {

config/environments/staging.js.example

@@ -198,7 +198,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {

config/environments/test.js.example

@@ -197,7 +197,12 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
+
       },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request

docs/MapConfig-Aggregation-extension.md

@@ -0,0 +1,62 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.7.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.7.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension introduces a new layer options for aggregated data tile generation.
+
+## 2.1 Aggregation options
+
+The layer options attribute is extended with a new optional `aggregation` attribute.
+The value of this attribute can be `false` to explicitly disable aggregation for the layer.
+
+```javascript
+{
+    aggregation: {
+
+        // OPTIONAL
+        // string, defines the placement of aggregated geometries. Can be one of:
+        // * "point-sample", the default places geometries at a sample point (one of the aggregated geometries)
+        // * "point-grid" places geometries at the center of the aggregation grid cells
+        // * "centroid" places geometriea at the average position of the aggregated points
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#placement for more details
+        placement: "point-sample",
+
+        // OPTIONAL
+        // object, defines the columns of the aggregated datasets. Each property corresponds to a columns name and
+        // should contain an object with two properties: "aggregate_function" (one of "sum", "max", "min", "avg", "mode" or "count"),
+        // and "aggregated_column" (the name of a column of the original layer query or "*")
+        // A column defined as `"_cdb_features_count": {"aggregate_function": "count", aggregated_column: "*"}`
+        // is always generated in addition to the defined columns.
+        // The column names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used
+        // for aggregated columns, as they correspond to columns always present in the result.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#columns for more details
+        columns: {
+            "aggregated_column_1": {
+                "aggregate_function": "sum",
+                "aggregated_column": "original_column_1"
+            }
+        },
+
+        // OPTIONAL
+        // Number, defines the cell-size of the spatial aggregation grid as a pixel resolution power of two (1/4, 1/2,... 2, 4, 16)
+        // to scale from 256x256 pixels; the default is 1 corresponding to 256x256 cells per tile.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#resolution for more details
+        resolution: 1,
+
+        // OPTIONAL
+        // Number, the minimum number of (estimated) rows in the dataset (query results) for aggregation to be applied.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#threshold for more details
+        threshold: 500000
+    }
+}
+```
+
+# History
+
+## 1.0.0
+
+ - Initial version

docs/aggregation.md

@@ -0,0 +1,187 @@
+# Tile Aggregation
+
+To be able to represent a large amount of data (say, hundred of thousands to millions of points) in a tile. This can be useful both for raster tiles (where the aggregation reduces the number of features to be rendered) and vector tiles (the tile contais less features).
+
+Aggregation is available only for point geometries. During aggregation the points are grouped using a grid; all the points laying in the same cell of the grid are summarized in a single aggregated result point.
+ - The position of the aggregated point is controlled by the `placement` parameter.
+ - The aggregated rows always contain at least a column, named `_cdb_feature_count`, which contains the number of the original points that the aggregated point represents.
+
+### Special default aggregation
+
+When no placement or columns are specified a special default aggregation is performed.
+
+This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_features_count` with the number of features in the group.
+
+Regarding the randomness of the sample: currently we use the row with the minimum `cartodb_id` value in each group.
+
+The rationale behind having this special aggregation with all the original columns is to provide a mostly transparent way to handle large datasets without having to provide special map configurations for those cases (i.e. preserving the logic used to produce the maps with smaller datasets). [Overviews have been used so far with this intent](https://carto.com/docs/tips-and-tricks/back-end-data-performance/), but they are inflexible.
+
+### User defined aggregations
+
+When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_features_count`, which is always present.
+
+We might decide in the future to allow sampling column values for any of the different placement modes.
+
+### Behaviour for raster and vector tiles
+
+The vector tiles from a vector-only map will be aggregated by default.
+However, Raster tiles (or vector tiles from a map which defines CartoCSS styles) will be aggregated only upon request.
+
+Aggregation that would otherwise occur can be disabled by passing an `aggregation=false` parameter to the map instantiation HTTP call.
+
+To control how aggregation is performed, an aggregation option can be added to the layer:
+
+```json
+{
+    "layers": [
+        {
+            "options": {
+                "sql": "SELECT * FROM data",
+                "aggregation": {
+                    "placement": "centroid",
+                    "columns": {
+                        "value": {
+                            "aggregate_function": "sum",
+                            "aggregated_column": "value"
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
+```
+
+Even if aggregation is explicitly requested it may not be activated, e.g., if the geometries are not points
+or the whole dataset is too small. The map instantiation response contains metadata that informs if any particular
+layer will be aggregated when tiles are requested, both for vector (mvt) and raster (png) tiles.
+
+```json
+{
+  "layergroupid": "7b97b6e76590fef889b63edd2efb1c79:1513608333045",
+  "metadata": {
+    "layers": [
+      {
+        "type": "mapnik",
+        "id": "layer0",
+        "meta": {
+          "stats": {
+            "estimatedFeatureCount": 6232136
+          },
+          "aggregation": {
+            "png": true,
+            "mvt": true
+          }
+        }
+      }
+    ]
+  }
+}
+```
+
+## Aggregation parameters
+
+The aggregation parameters for a layer are defined inside an `aggregation` option of the layer:
+
+```json
+{
+    "layers": [
+        {
+            "options": {
+                "sql": "SELECT * FROM data",
+                "aggregation": {"...": "..."}
+            }
+        }
+    ]
+}
+```
+
+### `placement`
+
+Determines the kind of aggregated geometry generated:
+
+#### `point-sample`
+
+This is the default placement. It will place the aggregated point at a random sample of the grouped points,
+like the default aggregation does. No other attribute is sampled, though, the point will contain the aggregated attributes determined by the `columns` parameter.
+
+#### `point-grid`
+
+Generates points at the center of the aggregation grid cells (squares).
+
+#### `centroid`
+
+Generates points with the averaged coordinated of the grouped points (i.e. the points inside each grid cell).
+
+### `columns`
+
+The aggregated attributes defined by `columns` are computed by a applying an _aggregate function_ to all the points in each group.
+Valid aggregate functions are `sum`, `avg` (average), `min` (minimum), `max` (maximum) and `mode` (the most frequent value in the group).
+The values to be aggregated are defined by the _aggregated column_ of the source data. The column keys define the name of the resulting column in the aggregated dataset.
+
+For example here we define three aggregate attributes named `total`, `max_price` and `price` which are all computed with the same column, `price`,
+of the original dataset applying three different aggregate functions.
+
+```json
+{
+    "columns": {
+        "total": { "aggregate_function": "sum", "aggregated_column": "price" },
+        "max_price": { "aggregate_function": "max", "aggregated_column": "price" },
+        "price": { "aggregate_function": "avg", "aggregated_column": "price" }
+    }
+}
+```
+
+> Note that you can use the original column names as names of the result, but all the result column names must be unique.  In particular, the names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used for aggregated columns, as they correspond to columns always present in the result.
+
+### `resolution`
+
+Defines the cell-size of the spatial aggregation grid. This is equivalent to the [CartoCSS `-torque-resolution`](https://carto.com/docs/carto-engine/cartocss/properties-for-torque/#-torque-resolution-float) property of Torque maps.
+
+The aggregation cells are `resolution`×`resolution` pixels in size, where pixels here are defined to be 1/256 of the (linear) size of a tile.
+The default value is 1, so that aggregation coincides with raster pixels. A value of 2 would make each cell to be 4 (2×2) pixels, and a value of
+0.5 would yield 4 cells per pixel. In teneral values less than 1 produce sub-pixel precision.
+
+> Note that is independent of the number of pixels for raster tile or the coordinate resolution (mvt_extent) of vector tiles.
+
+
+### `threshold`
+
+This is the minimum number of (estimated) rows in the dataset (query results) for aggregation to be applied. If the number of rows estimate is less than the threshold aggregation will be disabled for the layer; the instantiation response will reflect that and tiles will be generated without aggregation.
+
+### Example
+
+```json
+{
+    "version": "1.7.0",
+    "extent": [-20037508.5, -20037508.5, 20037508.5, 20037508.5],
+    "srid": 3857,
+    "maxzoom": 18,
+    "minzoom": 3,
+    "layers": [
+        {
+            "type": "mapnik",
+            "options": {
+                "sql": "select * from table",
+                "cartocss": "#table { marker-width: [total]; marker-fill: ramp(value, (red, green, blue), jenks); }",
+                "cartocss_version": "2.3.0",
+                "aggregation": {
+                    "placement": "centroid",
+                    "columns": {
+                        "value": {
+                            "aggregate_function": "avg",
+                            "aggregated_column": "value"
+                        },
+                        "total": {
+                            "aggregate_function": "sum",
+                            "aggregated_column": "value"
+                        }
+                    },
+                    "resolution": 2,
+                    "threshold": 500000
+                }
+            }
+        }
+    ]
+}
+```

docs/anonymous_maps.md

@@ -1,6 +1,7 @@
 # Anonymous Maps
 
-Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec).
+Alternatively, you can get the data for the map (geometry and attributes for each layer) using vector tiles (in which case CartoCSS is not required).
 
 
 ## Instantiate
@@ -41,6 +42,13 @@ updated_at | The ISO date of the last time the data involved in the query was up
 metadata | Includes information about the layers.
 cdn_url | URLs to fetch the data using the best CDN for your zone.
 
+**Improved response metadata**
+
+Originally, you needed to concantenate the `layergroupid` with the correct domain and the path for the tiles.
+Now, for convenience, the layergroup includes the final URLs in two formats:
+1. Leaflet's urlTemplate alike: useful when working with raster tiles or with libraries with an API similar to Leaflet's one.
+1. [TileJSON spec](https://github.com/mapbox/tilejson-spec): useful when working with Mapbox GL or any other library that supports TileJSON.
+
 ### Example
 
 #### Call
@@ -61,30 +69,49 @@ curl 'https://{username}.carto.com/api/v1/map' -H 'Content-Type: application/jso
         "type": "mapnik",
         "meta": {}
       }
-    ]
+    ],
+    "tilejson": {
+        "raster": {
+            "tilejson": "2.2.0",
+            "tiles": [
+                "http://a.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png",
+                "http://b.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png"
+            ]
+        }
+    },
+    "url": {
+        "raster": {
+            "urlTemplate": "http://{s}.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png",
+            "subdomains": ["a", "b"]
+        }
+    }
   },
   "cdn_url": {
     "http": "http://cdb.com",
-    "https": "https://cdb.com"
+    "https": "https://cdb.com",
+    "templates": {
+         "http": { "subdomains": ["a","b"], "url": "http://{s}.cdb.com" },
+         "https": { "subdomains": ["a","b"], "url": "https://{s}.example.com" },
+    }
   }
 }
 ```
 
 ## Map Tile Rendering
 
-Map tiles create the graphical representation of your map in a web browser. The performance rendering of map tiles is dependent on the type of geospatial data model (raster or vector) that you are using.
+Map tiles are used to create the graphic representation of your map in a web browser. Tiles can be requested either as pre-rendered *raster* tiles (images) or as *vector* map data to be rendered by the client (browser).
 
-- **Raster**: Generates map tiles based on a grid of pixels to represent your data. Each cell is a fixed size and contains values for particular map features. On the server-side, each request queries a dataset to retrieve data for each map tile. The grid size of map tiles can often lead to graphic quality issues.
+- **Raster**: If a tile is requested as a raster image format, like PNG, the map will be rendered on the server, using the CartoCSS styles defined in the layers of the map. It is necessary that all the layers of a map define CartoCSS styles in order to obtain raster tiles. Raster tiles are made up of 256x256 pixels; to avoid graphic quality issues tiles should be used unscaled to represent the zoom level (Z) for which they are requested. In order to render tiles, data will be retrieved from the database (in vector format) on the server-side.
 
-- **Vector**: Generates map tiles based on pre-defined coordinates to represent your data, similar to how basemap image tiles are rendered. On the client-side, map tiles represent real-world geometries of a map. Depending on the coordinates, vertices are used to connect the data and display points, lines, or polygons for the map tiles.
+- **Vector**: Tiles can also be requested as MVT (Mapbox Vector Tiles). In this case, only the geospatial vector data, without any styling, is returned. These tiles should be processed in the client-side to render the map.  In this case layers do not need to define CartoCSS, as any rendering and styling will be performed on the client side. The vector data of a tile represents real-world geometries by defining the vertices of points, lines or polygons in a tile-specific coordinate system.
 
 ## Retrieve resources from the layergroup
 
 When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection.
 
-### Mapnik tiles
+### Raster tiles
 
-These raster tiles retrieve just the Mapnik layers. See [individual layers](#individual-layers) for details about how to retrieve other layers.
+These raster tiles are PNG images that represent only the Mapnik layers of a map. See [individual layers](#individual-layers) for details about how to retrieve other layers.
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
@@ -92,9 +119,9 @@ https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 
 ### Mapbox Vector Tiles (MVT)
 
-[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server.
+[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that transfer geographic vector data to the client-side. Browser performance is fast since you can pan and zoom without having to query the server.
 
-CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files.
+CARTO uses Web Graphics Library (WebGL) to process MVT files on the browser. This is useful since WebGL is compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files.
 
 The following examples describe how to fetch MVT tiles with a cURL request.
 
@@ -245,7 +272,7 @@ center: [30, 0]
 map.setStyle({
     "version": 7,
     "glyphs": "...",
-    "constants": {...}, 
+    "constants": {...},
     "sources": {
       "cartodb": {
         "type": "vector",

lib/cartodb/controllers/map.js

@@ -90,6 +90,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal
         this.setAnalysesMetadataToLayergroup(includeQuery),
         this.setTurboCartoMetadataToLayergroup(),
         this.setAggregationMetadataToLayergroup(),
+        this.setTilejsonMetadataToLayergroup(),
         this.setSurrogateKeyHeader(),
         this.sendResponse(),
         this.augmentError({ label, addContext })
@@ -313,6 +314,84 @@ MapController.prototype.augmentLayergroupData = function () {
     };
 };
 
+function getTemplateUrl(url) {
+    return url.https || url.http;
+}
+
+function getTilejson(tiles, grids) {
+    const tilejson = {
+        tilejson: '2.2.0',
+        tiles: tiles.https || tiles.http
+    };
+
+    if (grids) {
+        tilejson.grids = grids.https || grids.http;
+    }
+
+    return tilejson;
+}
+
+MapController.prototype.setTilejsonMetadataToLayergroup = function () {
+    return function augmentLayergroupTilejsonMiddleware (req, res, next) {
+        const { layergroup, user, mapconfig } = res.locals;
+
+        const isVectorOnlyMapConfig = mapconfig.isVectorOnlyMapConfig();
+        let hasMapnikLayers = false;
+        layergroup.metadata.layers.forEach((layerMetadata, index) => {
+            const layerId = mapconfig.getLayerId(index);
+            const rasterResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.png`;
+            if (mapconfig.layerType(index) === 'mapnik') {
+                hasMapnikLayers = true;
+                const vectorResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.mvt`;
+                const layerTilejson = {
+                    vector: getTilejson(this.resourceLocator.getTileUrls(user, vectorResource))
+                };
+                if (!isVectorOnlyMapConfig) {
+                    let grids = null;
+                    const layer = mapconfig.getLayer(index);
+                    if (layer.options.interactivity) {
+                        const gridResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.grid.json`;
+                        grids = this.resourceLocator.getTileUrls(user, gridResource);
+                    }
+                    layerTilejson.raster = getTilejson(
+                        this.resourceLocator.getTileUrls(user, rasterResource),
+                        grids
+                    );
+                }
+                layerMetadata.tilejson = layerTilejson;
+            } else {
+                layerMetadata.tilejson = {
+                    raster: getTilejson(this.resourceLocator.getTileUrls(user, rasterResource))
+                };
+            }
+        });
+
+        const tilejson = {};
+        const url = {};
+
+        if (hasMapnikLayers) {
+            const vectorResource = `${layergroup.layergroupid}/{z}/{x}/{y}.mvt`;
+            tilejson.vector = getTilejson(
+                this.resourceLocator.getTileUrls(user, vectorResource)
+            );
+            url.vector = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, vectorResource));
+
+            if (!isVectorOnlyMapConfig) {
+                const rasterResource = `${layergroup.layergroupid}/{z}/{x}/{y}.png`;
+                tilejson.raster = getTilejson(
+                    this.resourceLocator.getTileUrls(user, rasterResource)
+                );
+                url.raster = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, rasterResource));
+            }
+        }
+
+        layergroup.metadata.tilejson = tilejson;
+        layergroup.metadata.url = url;
+
+        next();
+    }.bind(this);
+};
+
 MapController.prototype.getAffectedTables = function () {
     return function getAffectedTablesMiddleware (req, res, next) {
         const { dbname, layergroup, user, mapconfig } = res.locals;

lib/cartodb/controllers/named_maps.js

@@ -1,12 +1,31 @@
-var step = require('step');
-var assert = require('assert');
-var _ = require('underscore');
-var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const cors = require('../middleware/cors');
+const userMiddleware = require('../middleware/user');
+const allowQueryParams = require('../middleware/allow-query-params');
+const vectorError = require('../middleware/vector-error');
 
-var cors = require('../middleware/cors');
-var userMiddleware = require('../middleware/user');
-var allowQueryParams = require('../middleware/allow-query-params');
-var vectorError = require('../middleware/vector-error');
+const DEFAULT_ZOOM_CENTER = {
+    zoom: 1,
+    center: {
+        lng: 0,
+        lat: 0
+    }
+};
+
+function numMapper(n) {
+    return +n;
+}
+
+function getRequestParams(locals) {
+    const params = Object.assign({}, locals);
+
+    delete params.template;
+    delete params.affectedTablesAndLastUpdate;
+    delete params.namedMapProvider;
+    delete params.allowedQueryParams;
+
+    return params;
+}
 
 function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
@@ -27,7 +46,15 @@ NamedMapsController.prototype.register = function(app) {
         cors(),
         userMiddleware,
         this.prepareContext,
-        this.tile.bind(this),
+        this.getNamedMapProvider('NAMED_MAP_TILE'),
+        this.getAffectedTables(),
+        this.getTile('NAMED_MAP_TILE'),
+        this.setSurrogateKey(),
+        this.setCacheChannelHeader(),
+        this.setLastModifiedHeader(),
+        this.setCacheControlHeader(),
+        this.setContentTypeHeader(),
+        this.respond(),
         vectorError()
     );
 
@@ -37,298 +64,320 @@ NamedMapsController.prototype.register = function(app) {
         userMiddleware,
         allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
         this.prepareContext,
-        this.staticMap.bind(this)
+        this.getNamedMapProvider('STATIC_VIZ_MAP'),
+        this.getAffectedTables(),
+        this.getTemplate('STATIC_VIZ_MAP'),
+        this.prepareLayerFilterFromPreviewLayers('STATIC_VIZ_MAP'),
+        this.getStaticImageOptions(),
+        this.getImage('STATIC_VIZ_MAP'),
+        this.incrementMapViews(),
+        this.setSurrogateKey(),
+        this.setCacheChannelHeader(),
+        this.setLastModifiedHeader(),
+        this.setCacheControlHeader(),
+        this.setContentTypeHeader(),
+        this.respond()
     );
 };
 
-NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) {
-    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName()));
-    res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
-    res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
+NamedMapsController.prototype.getNamedMapProvider = function (label) {
+    return function getNamedMapProviderMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const { config, auth_token } = req.query;
+        const { template_id } = req.params;
 
-    var self = this;
+        // We force always the tile to be generated using PNG because
+        // is the only format we support by now
+        res.locals.format = 'png';
+        res.locals.layer = res.locals.layer || 'all';
 
-    step(
-        function getAffectedTablesAndLastUpdatedTime() {
-            namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
-        },
-        function sendResponse(err, result) {
+        const params = getRequestParams(res.locals);
+
+        this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.namedMapProvider = namedMapProvider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getAffectedTables = function () {
+    return function getAffectedTables (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
             req.profiler.done('affectedTables');
-            if (err) {
-                global.logger.log('ERROR generating cache channel: ' + err);
-            }
-            if (!result || !!result.tables) {
-                // we increase cache control as we can invalidate it
-                res.set('Cache-Control', 'public,max-age=31536000');
-
-                var lastModifiedDate;
-                if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.getLastUpdatedAt());
-                } else {
-                    lastModifiedDate = new Date();
-                }
-                res.set('Last-Modified', lastModifiedDate.toUTCString());
-
-                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, result);
-                }
-            }
-            res.status(200);
-            res.send(body);
-        }
-    );
-};
-
-NamedMapsController.prototype.tile = function(req, res, next) {
-    var self = this;
-
-    var cdbUser = res.locals.user;
-
-    var namedMapProvider;
-    step(
-        function getNamedMapProvider() {
-            self.namedMapProviderCache.get(
-                cdbUser,
-                req.params.template_id,
-                req.query.config,
-                req.query.auth_token,
-                res.locals,
-                this
-            );
-        },
-        function getTile(err, _namedMapProvider) {
-            assert.ifError(err);
-            namedMapProvider = _namedMapProvider;
-            self.tileBackend.getTile(namedMapProvider, req.params, this);
-        },
-        function handleImage(err, tile, headers, stats) {
-            req.profiler.add(stats);
 
             if (err) {
-                err.label = 'NAMED_MAP_TILE';
-                next(err);
-            } else {
-                self.sendResponse(req, res, tile, headers, namedMapProvider);
+                return next(err);
             }
-        }
-    );
+
+            res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
+
+            next();
+        });
+    }.bind(this);
 };
 
-NamedMapsController.prototype.staticMap = function(req, res, next) {
-    var self = this;
-
-    var cdbUser = res.locals.user;
-
-    var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    // We force always the tile to be generated using PNG because
-    // is the only format we support by now
-    res.locals.format = 'png';
-    res.locals.layer = res.locals.layer || 'all';
-
-    var namedMapProvider;
-    step(
-        function getNamedMapProvider() {
-            self.namedMapProviderCache.get(
-                cdbUser,
-                req.params.template_id,
-                req.query.config,
-                req.query.auth_token,
-                res.locals,
-                this
-            );
-        },
-        function prepareLayerVisibility(err, _namedMapProvider) {
-            assert.ifError(err);
-
-            namedMapProvider = _namedMapProvider;
-
-            self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this);
-        },
-        function prepareImageOptions(err) {
-            assert.ifError(err);
-            self.getStaticImageOptions(cdbUser, res.locals, namedMapProvider, this);
-        },
-        function getImage(err, imageOpts) {
-            assert.ifError(err);
-
-            var width = +req.params.width;
-            var height = +req.params.height;
-
-            if (!_.isUndefined(imageOpts.zoom) && imageOpts.center) {
-                self.previewBackend.getImage(
-                    namedMapProvider, format, width, height, imageOpts.zoom, imageOpts.center, this);
-            } else {
-                self.previewBackend.getImage(
-                    namedMapProvider, format, width, height, imageOpts.bounds, this);
-            }
-        },
-        function incrementMapViews(err, image, headers, stats) {
-            assert.ifError(err);
-
-            var next = this;
-            namedMapProvider.getMapConfig(function(mapConfigErr, mapConfig) {
-                self.metadataBackend.incMapviewCount(cdbUser, mapConfig.obj().stat_tag, function(sErr) {
-                    if (err) {
-                        global.logger.log("ERROR: failed to increment mapview count for user '%s': %s", cdbUser, sErr);
-                    }
-                    next(err, image, headers, stats);
-                });
-            });
-        },
-        function handleImage(err, image, headers, stats) {
-            req.profiler.done('render-' + format);
-            req.profiler.add(stats || {});
+NamedMapsController.prototype.getTemplate = function (label) {
+    return function getTemplateMiddleware (req, res, next) {
+        const { namedMapProvider } = res.locals;
 
+        namedMapProvider.getTemplate((err, template) => {
             if (err) {
-                err.label = 'STATIC_VIZ_MAP';
-                next(err);
-            } else {
-                self.sendResponse(req, res, image, headers, namedMapProvider);
+                err.label = label;
+                return next(err);
             }
-        }
-    );
+
+            res.locals.template = template;
+
+            next();
+        });
+    };
 };
 
-NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (
-    user,
-    req,
-    params,
-    namedMapProvider,
-    callback
-) {
-    var self = this;
-    namedMapProvider.getTemplate(function (err, template) {
-        if (err) {
-            return callback(err);
-        }
+NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (label) {
+    return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) {
+        const { user, template } = res.locals;
+        const { template_id } = req.params;
+        const { config, auth_token } = req.query;
 
         if (!template || !template.view || !template.view.preview_layers) {
-           return callback();
+            return next();
         }
 
         var previewLayers = template.view.preview_layers;
         var layerVisibilityFilter = [];
 
-        template.layergroup.layers.forEach(function (layer, index) {
+        template.layergroup.layers.forEach((layer, index) => {
             if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
                 layerVisibilityFilter.push(''+index);
             }
         });
 
         if (!layerVisibilityFilter.length) {
-            return callback();
+            return next();
         }
 
+        const params = getRequestParams(res.locals);
+
         // overwrites 'all' default filter
         params.layer = layerVisibilityFilter.join(',');
 
         // recreates the provider
-        self.namedMapProviderCache.get(
-            user,
-            req.params.template_id,
-            req.query.config,
-            req.query.auth_token,
-            params,
-            callback
-        );
-    });
+        this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, provider) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.namedMapProvider = provider;
+
+            next();
+        });
+    }.bind(this);
 };
 
-var DEFAULT_ZOOM_CENTER = {
-    zoom: 1,
-    center: {
-        lng: 0,
-        lat: 0
+NamedMapsController.prototype.getTile = function (label) {
+    return function getTileMiddleware (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        this.tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
+            req.profiler.add(stats);
+
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.body = tile;
+            res.locals.headers = headers;
+            res.locals.stats = stats;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getStaticImageOptions = function () {
+    return function getStaticImageOptionsMiddleware(req, res, next) {
+        const { user, namedMapProvider, template } = res.locals;
+
+        const imageOpts = getImageOptions(res.locals, template);
+
+        if (imageOpts) {
+            res.locals.imageOpts = imageOpts;
+            return next();
+        }
+
+        res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
+
+        namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
+            if (err) {
+                return next();
+            }
+
+            var affectedTables = affectedTablesAndLastUpdate.tables || [];
+
+            if (affectedTables.length === 0) {
+                return next();
+            }
+
+            this.tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
+                if (err) {
+                    return next();
+                }
+
+                res.locals.imageOpts = bounds;
+
+                return next();
+            });
+        });
+    }.bind(this);
+};
+
+function getImageOptions (params, template) {
+    const { zoom, lon, lat, bbox } = params;
+
+    let imageOpts = getImageOptionsFromCoordinates(zoom, lon, lat);
+    if (imageOpts) {
+        return imageOpts;
     }
-};
 
-function numMapper(n) {
-    return +n;
+    imageOpts = getImageOptionsFromBoundingBox(bbox);
+    if (imageOpts) {
+        return imageOpts;
+    }
+
+    imageOpts = getImageOptionsFromTemplate(template, zoom);
+    if (imageOpts) {
+        return imageOpts;
+    }
 }
 
-NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, params, namedMapProvider, callback) {
-    var self = this;
-
-    if ([params.zoom, params.lon, params.lat].map(numMapper).every(Number.isFinite)) {
-        return callback(null, {
-            zoom: params.zoom,
+function getImageOptionsFromCoordinates (zoom, lon, lat) {
+    if ([zoom, lon, lat].map(numMapper).every(Number.isFinite)) {
+        return {
+            zoom: zoom,
             center: {
-                lng: params.lon,
-                lat: params.lat
+                lng: lon,
+                lat: lat
             }
-        });
+        };
     }
+}
 
-    if (params.bbox) {
-        var bbox = params.bbox.split(',').map(numMapper);
-        if (bbox.length === 4 && bbox.every(Number.isFinite)) {
-            return callback(null, {
-                bounds: {
-                    west: bbox[0],
-                    south: bbox[1],
-                    east: bbox[2],
-                    north: bbox[3]
-                }
-            });
+
+function getImageOptionsFromTemplate (template, zoom) {
+    if (template.view) {
+        var zoomCenter = templateZoomCenter(template.view);
+        if (zoomCenter) {
+            if (Number.isFinite(+zoom)) {
+                zoomCenter.zoom = +zoom;
+            }
+
+            return zoomCenter;
+        }
+
+        var bounds = templateBounds(template.view);
+        if (bounds) {
+            return bounds;
         }
     }
+}
 
-    step(
-        function getTemplate() {
-            namedMapProvider.getTemplate(this);
-        },
-        function handleTemplateView(err, template) {
-            assert.ifError(err);
+function getImageOptionsFromBoundingBox (bbox = '') {
+    var _bbox = bbox.split(',').map(numMapper);
 
-            if (template.view) {
-                var zoomCenter = templateZoomCenter(template.view);
-                if (zoomCenter) {
-                    if (Number.isFinite(+params.zoom)) {
-                        zoomCenter.zoom = +params.zoom;
-                    }
-                    return zoomCenter;
-                }
-
-                var bounds = templateBounds(template.view);
-                if (bounds) {
-                    return bounds;
-                }
+    if (_bbox.length === 4 && _bbox.every(Number.isFinite)) {
+        return {
+            bounds: {
+                west: _bbox[0],
+                south: _bbox[1],
+                east: _bbox[2],
+                north: _bbox[3]
             }
+        };
+    }
+}
 
-            return false;
-        },
-        function estimateBoundsIfNoImageOpts(err, imageOpts) {
-            if (imageOpts) {
-                return imageOpts;
-            }
+NamedMapsController.prototype.getImage = function (label) {
+    return function getImageMiddleware (req, res, next) {
+        const { imageOpts, namedMapProvider } = res.locals;
+        const { zoom, center, bounds } = imageOpts;
 
-            var next = this;
-            namedMapProvider.getAffectedTablesAndLastUpdatedTime(function(err, affectedTablesAndLastUpdate) {
+        let { width, height } = req.params;
+
+        width = +width;
+        height = +height;
+
+        const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
+
+        if (zoom !== undefined && center) {
+            return this.previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
+                (err, image, headers, stats) => {
                 if (err) {
-                    return next(null);
+                    err.label = label;
+                    return next(err);
                 }
 
-                var affectedTables = affectedTablesAndLastUpdate.tables || [];
+                res.locals.body = image;
+                res.locals.headers = headers;
+                res.locals.stats = stats;
 
-                if (affectedTables.length === 0) {
-                    return next(null);
-                }
-
-                self.tablesExtentApi.getBounds(cdbUser, affectedTables, function(err, result) {
-                    return next(null, result);
-                });
+                next();
             });
-
-        },
-        function returnCallback(err, imageOpts) {
-            return callback(err, imageOpts || DEFAULT_ZOOM_CENTER);
         }
-    );
+
+        this.previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.body = image;
+            res.locals.headers = headers;
+            res.locals.stats = stats;
+
+            next();
+        });
+    }.bind(this);
+};
+
+function incrementMapViewsError (ctx) {
+    return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
+}
+
+NamedMapsController.prototype.incrementMapViews = function () {
+    return function incrementMapViewsMiddleware(req, res, next) {
+        const { user, namedMapProvider } = res.locals;
+
+        namedMapProvider.getMapConfig((err, mapConfig) => {
+            if (err) {
+                global.logger.log(incrementMapViewsError({ user, err }));
+                return next();
+            }
+
+            const statTag = mapConfig.obj().stat_tag;
+
+            this.metadataBackend.incMapviewCount(user, statTag, (err) => {
+                if (err) {
+                    global.logger.log(incrementMapViewsError({ user, err }));
+                }
+
+                next();
+            });
+        });
+    }.bind(this);
 };
 
 function templateZoomCenter(view) {
-    if (!_.isUndefined(view.zoom) && view.center) {
+    if (view.zoom !== undefined && view.center) {
         return {
             zoom: view.zoom,
             center: view.center
@@ -339,9 +388,8 @@ function templateZoomCenter(view) {
 
 function templateBounds(view) {
     if (view.bounds) {
-        var hasAllBounds = _.every(['west', 'south', 'east', 'north'], function(prop) {
-            return Number.isFinite(view.bounds[prop]);
-        });
+        var hasAllBounds = ['west', 'south', 'east', 'north'].every(prop => Number.isFinite(view.bounds[prop]));
+
         if (hasAllBounds) {
             return {
                 bounds: {
@@ -357,3 +405,86 @@ function templateBounds(view) {
     }
     return false;
 }
+
+NamedMapsController.prototype.setCacheChannelHeader = function () {
+    return function setCacheChannelHeaderMiddleware (req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
+        }
+
+        next();
+    };
+};
+
+NamedMapsController.prototype.setSurrogateKey = function () {
+    return function setSurrogateKeyMiddleware(req, res, next) {
+        const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
+
+        this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            if (affectedTablesAndLastUpdate.tables.length > 0) {
+                this.surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
+            }
+        }
+
+        next();
+    }.bind(this);
+};
+
+NamedMapsController.prototype.setLastModifiedHeader = function () {
+    return function setLastModifiedHeaderMiddleware(req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            var lastModifiedDate;
+            if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
+                lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
+            } else {
+                lastModifiedDate = new Date();
+            }
+
+            res.set('Last-Modified', lastModifiedDate.toUTCString());
+        }
+
+        next();
+    };
+ };
+
+NamedMapsController.prototype.setCacheControlHeader = function () {
+    return function setCacheControlHeaderMiddleware(req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            // we increase cache control as we can invalidate it
+            res.set('Cache-Control', 'public,max-age=31536000');
+        }
+
+        next();
+    };
+ };
+
+NamedMapsController.prototype.setContentTypeHeader = function () {
+    return function setContentTypeHeaderMiddleware(req, res, next) {
+        const { headers = {} } = res.locals;
+
+        res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
+
+        next();
+    };
+};
+
+NamedMapsController.prototype.respond = function () {
+    return function respondMiddleware (req, res) {
+        const { body, stats = {}, format } = res.locals;
+
+        req.profiler.done('render-' + format);
+        req.profiler.add(stats);
+
+        res.status(200);
+        res.send(body);
+    };
+};

lib/cartodb/controllers/named_maps_admin.js

@@ -1,10 +1,6 @@
-var step = require('step');
-var assert = require('assert');
-var templateName = require('../backends/template_maps').templateName;
-
-var cors = require('../middleware/cors');
-var userMiddleware = require('../middleware/user');
-
+const { templateName } = require('../backends/template_maps');
+const cors = require('../middleware/cors');
+const userMiddleware = require('../middleware/user');
 
 /**
  * @param {AuthApi} authApi
@@ -20,211 +16,189 @@ function NamedMapsAdminController(authApi, templateMaps) {
 module.exports = NamedMapsAdminController;
 
 NamedMapsAdminController.prototype.register = function (app) {
+    const { base_url_templated } = app;
+
     app.post(
-        app.base_url_templated + '/',
+        `${base_url_templated}/`,
         cors(),
         userMiddleware,
-        this.create.bind(this)
+        this.checkContentType('POST', 'POST TEMPLATE'),
+        this.authorizedByAPIKey('create', 'POST TEMPLATE'),
+        this.create()
     );
 
     app.put(
-        app.base_url_templated + '/:template_id',
+        `${base_url_templated}/:template_id`,
         cors(),
         userMiddleware,
-        this.update.bind(this)
+        this.checkContentType('PUT', 'PUT TEMPLATE'),
+        this.authorizedByAPIKey('update', 'PUT TEMPLATE'),
+        this.update()
     );
 
     app.get(
-        app.base_url_templated + '/:template_id',
+        `${base_url_templated}/:template_id`,
         cors(),
         userMiddleware,
-        this.retrieve.bind(this)
+        this.authorizedByAPIKey('get', 'GET TEMPLATE'),
+        this.retrieve()
     );
 
     app.delete(
-        app.base_url_templated + '/:template_id',
+        `${base_url_templated}/:template_id`,
         cors(),
         userMiddleware,
-        this.destroy.bind(this)
+        this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'),
+        this.destroy()
     );
 
     app.get(
-        app.base_url_templated + '/',
+        `${base_url_templated}/`,
         cors(),
         userMiddleware,
-        this.list.bind(this)
+        this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'),
+        this.list()
     );
 
     app.options(
-        app.base_url_templated + '/:template_id',
+        `${base_url_templated}/:template_id`,
         cors('Content-Type')
     );
 };
 
-NamedMapsAdminController.prototype.create = function(req, res, next) {
-    var self = this;
+NamedMapsAdminController.prototype.authorizedByAPIKey = function (action, label) {
+    return function authorizedByAPIKeyMiddleware (req, res, next) {
+        const { user } = res.locals;
 
-    var cdbuser = res.locals.user;
+        this.authApi.authorizedByAPIKey(user, req, (err, authenticated) => {
+            if (err) {
+                return next(err);
+            }
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function addTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
-            ifInvalidContentType(req, 'template POST data must be of type application/json');
-            var cfg = req.body;
-            self.templateMaps.addTemplate(cdbuser, cfg, this);
-        },
-        function prepareResponse(err, tpl_id){
-            assert.ifError(err);
-            return { template_id: tpl_id };
-        },
-        finishFn(self, req, res, 'POST TEMPLATE', null, next)
-    );
+            if (!authenticated) {
+                const error = new Error(`Only authenticated user can ${action} templated maps`);
+                error.http_status = 403;
+                error.label = label;
+                return next(error);
+            }
+
+            next();
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.update = function(req, res, next) {
-    var self = this;
-
-    var cdbuser = res.locals.user;
-    var template;
-    var tpl_id;
-
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function updateTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated user can update templated maps');
-            ifInvalidContentType(req, 'template PUT data must be of type application/json');
-
-            template = req.body;
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.updTemplate(cdbuser, tpl_id, template, this);
-        },
-        function prepareResponse(err){
-            assert.ifError(err);
-
-            return { template_id: tpl_id };
-        },
-        finishFn(self, req, res, 'PUT TEMPLATE', null, next)
-    );
+NamedMapsAdminController.prototype.checkContentType = function (action, label) {
+    return function checkContentTypeMiddleware (req, res, next) {
+        if (!req.is('application/json')) {
+            const error = new Error(`template ${action} data must be of type application/json`);
+            error.label = label;
+            return next(error);
+        }
+        next();
+    };
 };
 
-NamedMapsAdminController.prototype.retrieve = function(req, res, next) {
-    var self = this;
+NamedMapsAdminController.prototype.create = function () {
+    return function createTemplateMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const template = req.body;
 
-    req.profiler.start('windshaft-cartodb.get_template');
+        this.templateMaps.addTemplate(user, template, (err, templateId) => {
+            if (err) {
+                return next(err);
+            }
 
-    var cdbuser = res.locals.user;
-    var tpl_id;
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function getTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
+            res.status(200);
 
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.getTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err, tpl_val) {
-            assert.ifError(err);
-            if ( ! tpl_val ) {
-                err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
-                err.http_status = 404;
-                throw err;
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_id: templateId });
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.update = function () {
+    return function updateTemplateMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const template = req.body;
+        const templateId = templateName(req.params.template_id);
+
+        this.templateMaps.updTemplate(user, templateId, template, (err) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_id: templateId });
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.retrieve =  function () {
+    return function retrieveTemplateMiddleware (req, res, next) {
+        req.profiler.start('windshaft-cartodb.get_template');
+
+        const { user } = res.locals;
+        const templateId = templateName(req.params.template_id);
+
+        this.templateMaps.getTemplate(user, templateId, (err, template) => {
+            if (err) {
+                return next(err);
+            }
+
+            if (!template) {
+                const error = new Error(`Cannot find template '${templateId}' of user '${user}'`);
+                error.http_status = 404;
+                return next(error);
             }
             // auth_id was added by ourselves,
             // so we remove it before returning to the user
-            delete tpl_val.auth_id;
-            return { template: tpl_val };
-        },
-        finishFn(self, req, res, 'GET TEMPLATE', null, next)
-    );
+            delete template.auth_id;
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template });
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.destroy = function(req, res, next) {
-    var self = this;
+NamedMapsAdminController.prototype.destroy = function () {
+    return function destroyTemplateMiddleware (req, res, next) {
+        req.profiler.start('windshaft-cartodb.delete_template');
 
-    req.profiler.start('windshaft-cartodb.delete_template');
+        const { user } = res.locals;
+        const templateId = templateName(req.params.template_id);
 
-    var cdbuser = res.locals.user;
-    var tpl_id;
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function deleteTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can delete template maps');
-
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.delTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err/*, tpl_val*/){
-            assert.ifError(err);
-            return '';
-        },
-        finishFn(self, req, res, 'DELETE TEMPLATE', 204, next)
-    );
-};
-
-NamedMapsAdminController.prototype.list = function(req, res, next) {
-    var self = this;
-    req.profiler.start('windshaft-cartodb.get_template_list');
-
-    var cdbuser = res.locals.user;
-
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function listTemplates(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated user can list templated maps');
-
-            self.templateMaps.listTemplates(cdbuser, this);
-        },
-        function prepareResponse(err, tpl_ids){
-            assert.ifError(err);
-            return { template_ids: tpl_ids };
-        },
-        finishFn(self, req, res, 'GET TEMPLATE LIST', null, next)
-    );
-};
-
-function finishFn(controller, req, res, description, status, next) {
-    return function finish(err, body){
-        if (err) {
-            err.label = description;
-            next(err);
-        } else {
-            res.status(status || 200);
-
-            if (req.query && req.query.callback) {
-                res.jsonp(body);
-            } else {
-                res.json(body);
+        this.templateMaps.delTemplate(user, templateId, (err/* , tpl_val */) => {
+            if (err) {
+                return next(err);
             }
-        }
-    };
-}
 
-function ifUnauthenticated(authenticated, description) {
-    if (!authenticated) {
-        var err = new Error(description);
-        err.http_status = 403;
-        throw err;
-    }
-}
+            res.status(204);
 
-function ifInvalidContentType(req, description) {
-    if (!req.is('application/json')) {
-        throw new Error(description);
-    }
-}
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]('');
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.list = function () {
+    return function listTemplatesMiddleware (req, res, next) {
+        req.profiler.start('windshaft-cartodb.get_template_list');
+
+        const { user } = res.locals;
+
+        this.templateMaps.listTemplates(user, (err, templateIds) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_ids: templateIds });
+        });
+    }.bind(this);
+};

lib/cartodb/models/aggregation/aggregation-map-config.js

@@ -1,34 +0,0 @@
-const MapConfig = require('windshaft').model.MapConfig;
-
-module.exports = class AggregationMapConfig extends MapConfig {
-    constructor (config, datasource) {
-        super(config, datasource);
-    }
-
-    isAggregationMapConfig () {
-        return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation();
-    }
-
-    isAggregationLayer (index) {
-        return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index);
-    }
-
-    hasAnyLayerAggregation () {
-        const layers = this.getLayers();
-
-        for (let index = 0; index < layers.length; index++) {
-            if (this.hasLayerAggregation(index)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    hasLayerAggregation (index) {
-        const layer = this.getLayer(index);
-        const { aggregation } = layer.options;
-
-        return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean');
-    }
-};

lib/cartodb/models/aggregation/aggregation-mapconfig.js

@@ -0,0 +1,233 @@
+const MapConfig = require('windshaft').model.MapConfig;
+const aggregationQuery = require('./aggregation-query');
+const aggregationValidator = require('./aggregation-validator');
+const {
+    createPositiveNumberValidator,
+    createIncludesValueValidator,
+    createAggregationColumnsValidator
+} = aggregationValidator;
+
+const SubstitutionTokens = require('../../utils/substitution-tokens');
+
+const removeDuplicates = arr => [...new Set(arr)];
+
+function prepareSql(sql) {
+    return sql && SubstitutionTokens.replace(sql, {
+        bbox: 'ST_MakeEnvelope(0,0,0,0)',
+        scale_denominator: '0',
+        pixel_width: '1',
+        pixel_height: '1'
+    });
+}
+
+module.exports = class AggregationMapConfig extends MapConfig {
+    static get AGGREGATIONS () {
+        return aggregationQuery.SUPPORTED_AGGREGATE_FUNCTIONS;
+    }
+
+    static get PLACEMENTS () {
+        return aggregationQuery.SUPPORTED_PLACEMENTS;
+    }
+
+    static get THRESHOLD () {
+        return 1e5; // 100K
+    }
+
+    static get RESOLUTION () {
+        return 1;
+    }
+
+    static get SUPPORTED_GEOMETRY_TYPES () {
+        return [
+            'ST_Point'
+        ];
+    }
+
+    static supportsGeometryType(geometryType) {
+        return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType);
+    }
+
+    static  getAggregationGeometryColumn() {
+        return aggregationQuery.GEOMETRY_COLUMN;
+    }
+
+    constructor (user, config, connection, datasource) {
+        super(config, datasource);
+
+        const validate = aggregationValidator(this);
+        const positiveNumberValidator = createPositiveNumberValidator(this);
+        const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS);
+        const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS);
+
+        validate('resolution', positiveNumberValidator);
+        validate('placement', includesValidPlacementsValidator);
+        validate('threshold', positiveNumberValidator);
+        validate('columns', aggregationColumnsValidator);
+
+        this.user = user;
+        this.pgConnection = connection;
+    }
+
+    getAggregatedQuery (index) {
+        const { sql_raw, sql } = this.getLayer(index).options;
+        const {
+            // The default aggregation has no placement, columns or dimensions;
+            // this enables the special "full-sample" aggregation.
+            resolution = AggregationMapConfig.RESOLUTION,
+            threshold = AggregationMapConfig.THRESHOLD,
+            placement,
+            columns = {},
+            dimensions = {}
+        } = this.getAggregation(index);
+
+        return aggregationQuery({
+            query: sql_raw || sql,
+            resolution,
+            threshold,
+            placement,
+            columns,
+            dimensions,
+            isDefaultAggregation: this._isDefaultLayerAggregation(index)
+        });
+    }
+
+    isAggregationMapConfig () {
+        return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation();
+    }
+
+    isAggregationLayer (index) {
+        return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index);
+    }
+
+    hasAnyLayerAggregation () {
+        const layers = this.getLayers();
+
+        for (let index = 0; index < layers.length; index++) {
+            if (this.hasLayerAggregation(index)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    hasLayerAggregation (index) {
+        const layer = this.getLayer(index);
+        const { aggregation } = layer.options;
+
+        return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean');
+    }
+
+    getAggregation (index) {
+        if (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) {
+            return {};
+        }
+
+        const { aggregation } = this.getLayer(index).options;
+
+        if (typeof aggregation === 'boolean') {
+            return {};
+        }
+
+        return aggregation;
+    }
+
+    getLayerAggregationColumns (index, callback) {
+        if (this._isDefaultLayerAggregation(index)) {
+            const skipGeoms = true;
+            return this.getLayerColumns(index, skipGeoms, (err, columns) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                return callback(null, columns);
+            });
+        }
+
+        const columns = this._getLayerAggregationRequiredColumns(index);
+
+        return callback(null, columns);
+    }
+
+    _getLayerAggregationRequiredColumns (index) {
+        const { columns, dimensions } = this.getAggregation(index);
+
+        let aggregatedColumns = [];
+        if (columns) {
+            aggregatedColumns = Object.keys(columns)
+                .map(key => columns[key].aggregated_column)
+                .filter(aggregatedColumn => typeof aggregatedColumn === 'string');
+        }
+
+        let dimensionsColumns = [];
+        if (dimensions) {
+            dimensionsColumns = Object.keys(dimensions)
+                .map(key => dimensions[key])
+                .filter(dimension => typeof dimension === 'string');
+        }
+
+        return removeDuplicates(aggregatedColumns.concat(dimensionsColumns));
+    }
+
+    doesLayerReachThreshold(index, featureCount) {
+        const threshold = this.getAggregation(index) && this.getAggregation(index).threshold ?
+            this.getAggregation(index).threshold :
+            AggregationMapConfig.THRESHOLD;
+
+        return featureCount >= threshold;
+    }
+
+    getLayerColumns (index, skipGeoms, callback) {
+        const geomColumns = ['the_geom', 'the_geom_webmercator'];
+        const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
+        const layer = this.getLayer(index);
+
+        this.pgConnection.getConnection(this.user, (err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            const sql = limitedQuery({
+                query: prepareSql(layer.options.sql)
+            });
+
+            connection.query(sql, (err, result) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                let columns = result.fields || [];
+
+                columns = columns.map(({ name }) => name);
+
+                if (skipGeoms) {
+                    columns = columns.filter((column) => !geomColumns.includes(column));
+                }
+
+                return callback(err, columns);
+            });
+        });
+    }
+
+    _isDefaultLayerAggregation (index) {
+        const aggregation = this.getAggregation(index);
+
+        return (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) ||
+            aggregation === true ||
+            this._isDefaultAggregation(aggregation);
+    }
+
+    _isDefaultAggregation (aggregation) {
+        return aggregation.placement === undefined &&
+            aggregation.columns === undefined &&
+            this._isEmptyParameter(aggregation.dimensions);
+    }
+
+    _isEmptyParameter(parameter) {
+        return parameter === undefined || parameter === null || this._isEmptyObject(parameter);
+    }
+
+    _isEmptyObject (parameter) {
+        return typeof parameter === 'object' && Object.keys(parameter).length === 0;
+    }
+};

lib/cartodb/models/aggregation/aggregation-query.js

@@ -1,8 +1,12 @@
+const DEFAULT_PLACEMENT = 'point-sample';
+
 /**
  * Returns a template function (function that accepts template parameters and returns a string)
  * to generate an aggregation query.
  * Valid options to define the query template are:
  * - placement
+ * - columns
+ * - dimensions*
  * The query template parameters taken by the result template function are:
  * - sourceQuery
  * - res
@@ -10,9 +14,12 @@
  * - dimensions
  */
 const templateForOptions = (options) => {
-    let templateFn = aggregationQueryTemplates[options.placement];
-    if (!templateFn) {
-        throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
+    let templateFn = defaultAggregationQueryTemplate;
+    if (!options.isDefaultAggregation) {
+        templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT];
+        if (!templateFn) {
+            throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
+        }
     }
     return templateFn;
 };
@@ -25,6 +32,11 @@ const templateForOptions = (options) => {
  * - columns
  * - placement
  * - dimensions
+ *
+ * The default aggregation (when no explicit placement, columns or dimensions are present) returns
+ * a sample record (with all the original columns and _cdb_feature_count) for each aggregation group.
+ * When placement, columns or dimensions are specified, columns are aggregated as requested
+ * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
  */
 const queryForOptions = (options) => templateForOptions(options)({
     sourceQuery: options.query,
@@ -56,6 +68,8 @@ const SUPPORTED_AGGREGATE_FUNCTIONS = {
     }
 };
 
+module.exports.SUPPORTED_AGGREGATE_FUNCTIONS = Object.keys(SUPPORTED_AGGREGATE_FUNCTIONS);
+
 const sep = (list) => {
     let expr = list.join(', ');
     return expr ? ', ' + expr : expr;
@@ -69,8 +83,13 @@ const aggregateColumns = ctx => {
     }, ctx.columns || {});
 };
 
-const aggregateColumnNames = ctx => {
+const aggregateColumnNames = (ctx, table) => {
     let columns = aggregateColumns(ctx);
+    if (table) {
+        return sep(Object.keys(columns).map(
+            column_name => `${table}.${column_name}`
+        ));
+    }
     return sep(Object.keys(columns));
 };
 
@@ -90,8 +109,14 @@ const aggregateColumnDefs = ctx => {
 
 const aggregateDimensions = ctx => ctx.dimensions || {};
 
-const dimensionNames = ctx => {
-    return sep(Object.keys(aggregateDimensions(ctx)));
+const dimensionNames = (ctx, table) => {
+    let dimensions = aggregateDimensions(ctx);
+    if (table) {
+        return sep(Object.keys(dimensions).map(
+            dimension_name => `${table}.${dimension_name}`
+        ));
+    }
+    return sep(Object.keys(dimensions));
 };
 
 const dimensionDefs = ctx => {
@@ -114,9 +139,38 @@ const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::dou
 //   is only applied after the aggregation.
 // * This queries are used for rendering and the_geom is omitted in the results for better performance
 
+// The special default aggregation includes all the columns of a sample row per grid cell and
+// the count (_cdb_feature_count) of the aggregated rows.
+const defaultAggregationQueryTemplate = ctx => `
+    WITH
+    _cdb_params AS (
+        SELECT
+        ${gridResolution(ctx)} AS res,
+        !bbox! AS bbox
+    ),
+    _cdb_clusters AS (
+        SELECT
+            MIN(cartodb_id) AS cartodb_id
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+        WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+        GROUP BY
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+            ${dimensionNames(ctx)}
+    ) SELECT
+        _cdb_query.*
+        ${aggregateColumnNames(ctx)}
+    FROM
+        _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+        ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
+`;
+
 const aggregationQueryTemplates = {
     'centroid': ctx => `
-        WITH _cdb_params AS (
+        WITH
+        _cdb_params AS (
             SELECT
             ${gridResolution(ctx)} AS res,
             !bbox! AS bbox
@@ -140,34 +194,38 @@ const aggregationQueryTemplates = {
     `,
 
     'point-grid': ctx => `
-    WITH _cdb_params AS (
-        SELECT
-        ${gridResolution(ctx)} AS res,
-        !bbox! AS bbox
-    ),
-    _cdb_clusters AS (
-        SELECT
-        Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
-        Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
-        ${dimensionDefs(ctx)}
-        ${aggregateColumnDefs(ctx)}
-        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
-        WHERE the_geom_webmercator && _cdb_params.bbox
-        GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
-    )
-    SELECT
-        ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
-        ${dimensionNames(ctx)}
-        ${aggregateColumnNames(ctx)}
-        FROM _cdb_clusters, _cdb_params
-    `,
-
-    'point-sample': ctx => `
-        WITH _cdb_params AS (
+        WITH
+        _cdb_params AS (
             SELECT
             ${gridResolution(ctx)} AS res,
             !bbox! AS bbox
-        ), _cdb_clusters AS (
+        ),
+        _cdb_clusters AS (
+            SELECT
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+            WHERE the_geom_webmercator && _cdb_params.bbox
+            GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
+        )
+        SELECT
+            row_number() over() AS cartodb_id,
+            ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
+            ${dimensionNames(ctx)}
+            ${aggregateColumnNames(ctx)}
+            FROM _cdb_clusters, _cdb_params
+    `,
+
+    'point-sample': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        ),
+        _cdb_clusters AS (
             SELECT
                 MIN(cartodb_id) AS cartodb_id
                 ${dimensionDefs(ctx)}
@@ -178,13 +236,18 @@ const aggregationQueryTemplates = {
                 Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
                 Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
                 ${dimensionNames(ctx)}
-        ) SELECT
+        )
+        SELECT
             _cdb_clusters.cartodb_id,
             the_geom, the_geom_webmercator
-            ${dimensionNames(ctx)}
-            ${aggregateColumnNames(ctx)}
+            ${dimensionNames(ctx, '_cdb_query')}
+            ${aggregateColumnNames(ctx, '_cdb_clusters')}
         FROM
             _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
             ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
     `
+
 };
+
+module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
+module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator';

lib/cartodb/models/aggregation/aggregation-validator.js

@@ -0,0 +1,93 @@
+module.exports = function aggregationValidator (mapconfig) {
+    return function validateProperty (key, validator) {
+        for (let index = 0; index < mapconfig.getLayers().length; index++) {
+            const aggregation = mapconfig.getAggregation(index);
+
+            if (aggregation === undefined || aggregation[key] === undefined) {
+                continue;
+            }
+
+            validator(aggregation[key], key, index);
+        }
+    };
+};
+
+module.exports.createIncludesValueValidator = function (mapconfig, validValues) {
+    return function validateIncludesValue (value, key, index) {
+        if (!validValues.includes(value)) {
+            const message = `Invalid ${key}. Valid values: ${validValues.join(', ')}`;
+            throw createLayerError(message, mapconfig, index);
+        }
+    };
+};
+
+module.exports.createPositiveNumberValidator = function (mapconfig) {
+    return function validatePositiveNumber (value, key, index) {
+        if (!Number.isFinite(value) || value <= 0) {
+            const message = `Invalid ${key}, should be a number greather than 0`;
+            throw createLayerError(message, mapconfig, index);
+        }
+    };
+};
+
+module.exports.createAggregationColumnsValidator = function (mapconfig, validAggregatedFunctions) {
+    const validateAggregationColumnNames = createAggregationColumnNamesValidator(mapconfig);
+    const validateAggregateFunction = createAggregateFunctionValidator(mapconfig, validAggregatedFunctions);
+    const validateAggregatedColumn = createAggregatedColumnValidator(mapconfig);
+
+    return function validateAggregationColumns (value, key, index) {
+        validateAggregationColumnNames(value, key, index);
+        validateAggregateFunction(value, key, index);
+        validateAggregatedColumn(value, key, index);
+    };
+};
+
+function createAggregationColumnNamesValidator(mapconfig) {
+    return function validateAggregationColumnNames (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            if (columnName.length <= 0) {
+                const message = `Invalid column name, should be a non empty string`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createAggregateFunctionValidator (mapconfig, validAggregatedFunctions) {
+    return function validateAggregateFunction (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            const { aggregate_function } = value[columnName];
+
+            if (!validAggregatedFunctions.includes(aggregate_function)) {
+                const message = `Unsupported aggregation function ${aggregate_function},` +
+                                ` valid ones: ${validAggregatedFunctions.join(', ')}`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createAggregatedColumnValidator (mapconfig) {
+    return function validateAggregatedColumn (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            const { aggregated_column } = value[columnName];
+
+            if (typeof aggregated_column !== 'string' || aggregated_column <= 0) {
+                const message = `Invalid aggregated column, should be a non empty string`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createLayerError(message, mapconfig, index) {
+    const error = new Error(message);
+    error.type = 'layer';
+    error.layer = {
+        id: mapconfig.getLayerId(index),
+        index: index,
+        type: mapconfig.layerType(index)
+    };
+
+    return error;
+}

lib/cartodb/models/aggregation/aggregation.js

@@ -1,26 +0,0 @@
-const aggregationQuery = require('./aggregation-query');
-
-module.exports = class Aggregation {
-    static get THRESHOLD() {
-        return 1e5; // 100K
-    }
-
-    constructor (mapconfig, query, {
-        resolution = 1,
-        threshold = Aggregation.THRESHOLD,
-        placement = 'centroid',
-        columns = {},
-        dimensions = {}
-    } = {}) {
-        this.mapconfig = mapconfig;
-        this.query = query;
-        this.resolution = resolution;
-        this.threshold = threshold;
-        this.placement = placement;
-        this.columns = columns;
-        this.dimensions = dimensions;
-    }
-    sql () {
-        return aggregationQuery(this);
-    }
-};

lib/cartodb/models/dataview/aggregation.js

@@ -74,7 +74,7 @@ const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
 
 const rankedAggregationQueryTpl = ctx => `
     SELECT
-        CAST(category AS text),
+        category,
         value,
         false as agg,
         nulls_count,
@@ -87,7 +87,7 @@ const rankedAggregationQueryTpl = ctx => `
         WHERE rank < ${ctx.limit}
     UNION ALL
     SELECT
-        'Other' category,
+        null category,
         ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
         true as agg,
         nulls_count,
@@ -109,7 +109,7 @@ const rankedAggregationQueryTpl = ctx => `
 
 const aggregationQueryTpl = ctx => `
     SELECT
-        CAST(${ctx.column} AS text) AS category,
+        ${ctx.column} AS category,
         ${ctx.aggregationFn} AS value,
         false as agg,
         nulls_count,
@@ -120,7 +120,7 @@ const aggregationQueryTpl = ctx => `
         ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
     FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
     GROUP BY
-        category,
+        ${ctx.column},
         nulls_count,
         min_val,
         max_val,
@@ -282,7 +282,7 @@ module.exports = class Aggregation extends BaseDataview {
             max_val = 0,
             categories_count = 0
          } = result.rows[0] || {};
-
+        
         return {
             aggregation: this.aggregation,
             count: count,
@@ -292,7 +292,13 @@ module.exports = class Aggregation extends BaseDataview {
             min: min_val,
             max: max_val,
             categoriesCount: categories_count,
-            categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg }))
+            categories: result.rows.map(({ category, value, agg }) => {
+                return { 
+                    category: agg ? 'Other' : category,
+                    value, 
+                    agg 
+                };
+            })
         };
     }
 

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -56,7 +56,7 @@ __wd_buckets AS
         ${ctx.query}
     ) __source, __wd_tz
     ${condition_str}
-    GROUP BY timestamp, __wd_tz.name
+    GROUP BY 1, __wd_tz.name
 ),`;
 }
 

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -28,7 +28,7 @@ const binsQueryTpl = ctx => `
             ) AS quartile
             FROM __cdb_filtered_source) _cdb_quartiles
             WHERE quartile = 1 or quartile = 3
-            GROUP BY quartile
+            GROUP BY 1
         ) __cdb_iqr
     ),
     __cdb_bins AS (
@@ -137,8 +137,8 @@ FROM
 (
     ${ctx.query}
 ) __cdb_filtered_source_query${extra_tables}
-GROUP BY bin${extra_groupby}
-ORDER BY bin;`;
+GROUP BY 10${extra_groupby}
+ORDER BY 10;`;
     }
 
     _hasOverridenBins (override) {

lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js

@@ -1,9 +1,9 @@
-const Aggregation = require('../../aggregation/aggregation');
-const AggregationMapConfig = require('../../aggregation/aggregation-map-config');
+const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig');
 const queryUtils = require('../../../utils/query-utils');
 
 const unsupportedGeometryTypeErrorMessage = ctx =>
-`Unsupported geometry type: ${ctx.geometryType}. Aggregation is available only for geometry type: ST_Point`;
+`Unsupported geometry type: ${ctx.geometryType}. ` +
+`Aggregation is available only for geometry type: ${AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES}`;
 
 const invalidAggregationParamValueErrorMessage = ctx =>
 `Invalid value for 'aggregation' query param: ${ctx.value}. Valid ones are 'true' or 'false'`;
@@ -14,11 +14,17 @@ module.exports = class AggregationMapConfigAdapter {
     }
 
     getMapConfig (user, requestMapConfig, params, context, callback) {
-        if (!this._isValidAggregationParam(params)) {
+        if (!this._isValidAggregationQueryParam(params)) {
             return callback(new Error(invalidAggregationParamValueErrorMessage({ value: params.aggregation })));
         }
 
-        const mapConfig = new AggregationMapConfig(requestMapConfig);
+        let mapConfig;
+        try {
+            mapConfig = new AggregationMapConfig(user, requestMapConfig, this.pgConnection);
+        } catch (err) {
+            return callback(err);
+        }
+
 
         if (!this._shouldAdapt(mapConfig, params)) {
             return callback(null, requestMapConfig);
@@ -33,7 +39,7 @@ module.exports = class AggregationMapConfigAdapter {
         });
     }
 
-    _isValidAggregationParam (params) {
+    _isValidAggregationQueryParam (params) {
         const { aggregation } = params;
         return aggregation === undefined || aggregation === 'true' || aggregation === 'false';
     }
@@ -83,61 +89,72 @@ module.exports = class AggregationMapConfigAdapter {
                     return reject(err);
                 }
 
-                if (shouldAdapt) {
-                    const sql = layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql;
-                    const aggregation = new Aggregation(mapConfig, sql, layer.options.aggregation);
-                    const sqlQueryWrap = layer.options.sql_wrap;
-
-                    let aggregationSql = aggregation.sql();
-
-                    if (sqlQueryWrap) {
-                        aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
-                    }
-
-                    layer.options.sql = aggregationSql;
+                if (!shouldAdapt) {
+                    return resolve({ layer, index, adapted: shouldAdapt });
                 }
 
-                return resolve({ layer, index, adapted: shouldAdapt });
+                const sqlQueryWrap = layer.options.sql_wrap;
+
+                let aggregationSql = mapConfig.getAggregatedQuery(index);
+
+                if (sqlQueryWrap) {
+                    aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
+                }
+
+                layer.options.sql = aggregationSql;
+
+                mapConfig.getLayerAggregationColumns(index, (err, columns) => {
+                    if (err) {
+                        return reject(err);
+                    }
+
+                    layer.options.columns = columns;
+
+                    return resolve({ layer, index, adapted: shouldAdapt });
+                });
             });
         });
     }
 
     _shouldAdaptLayer (connection, mapConfig, layer, index, callback) {
-        let shouldAdapt = false;
-
         if (!mapConfig.isAggregationLayer(index)) {
-            return callback(null, shouldAdapt);
+            return callback(null, false);
         }
 
         const aggregationMetadata = queryUtils.getAggregationMetadata({
-            query: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql
+            query: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql,
+            geometryColumn: AggregationMapConfig.getAggregationGeometryColumn()
         });
 
         connection.query(aggregationMetadata, (err, res) => {
             if (err) {
-                return callback(null, shouldAdapt);
+                return callback(null, false);
             }
 
             const result = res.rows[0] || {};
-            const estimatedFeatureCount = result.count;
 
-            const threshold = layer.options.aggregation && layer.options.aggregation.threshold ?
-                layer.options.aggregation.threshold :
-                Aggregation.THRESHOLD;
+            if (!mapConfig.isVectorOnlyMapConfig() && !AggregationMapConfig.supportsGeometryType(result.type)) {
+                const message = unsupportedGeometryTypeErrorMessage({ geometryType: result.type });
+                const error = new Error(message);
+                error.type = 'layer';
+                error.layer = {
+                    id: mapConfig.getLayerId(index),
+                    index: index,
+                    type: mapConfig.layerType(index)
+                };
 
-            if (estimatedFeatureCount < threshold) {
-                return callback(null, shouldAdapt);
+                return callback(error);
             }
 
-            const geometryType = result.type;
-
-            if (geometryType !== 'ST_Point') {
-                return callback(new Error(unsupportedGeometryTypeErrorMessage({ geometryType })));
+            if (mapConfig.isVectorOnlyMapConfig() && !AggregationMapConfig.supportsGeometryType(result.type)) {
+                return callback(null, false);
             }
 
-            shouldAdapt = true;
+            if (!mapConfig.doesLayerReachThreshold(index, result.count)) {
+                return callback(null, false);
+            }
 
-            callback(null, shouldAdapt);
+            callback(null, true);
         });
     }
 

lib/cartodb/models/resource-locator.js

@@ -21,69 +21,208 @@ function ResourceLocator(environment) {
 
 module.exports = ResourceLocator;
 
-ResourceLocator.prototype.getUrls = function(username, resource) {
+ResourceLocator.prototype.getTileUrls = function(username, resourcePath) {
     if (this.resourcesUrlTemplates) {
-        return this.getUrlsFromTemplate(username, resource);
-    }
-    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource);
-    if (cdnDomain) {
+        const urls = this.getUrlsFromTemplate(username, new TileResource(resourcePath));
         return {
-            http: 'http://' + cdnDomain.http + '/' + username + '/api/v1/map/' + resource,
-            https: 'https://' + cdnDomain.https + '/' + username + '/api/v1/map/' + resource
+            http: Array.isArray(urls.http) ? urls.http : [urls.http],
+            https: Array.isArray(urls.https) ? urls.https : [urls.https]
         };
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new TileResource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
     } else {
         var port = this.environment.port;
         return {
-            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
+            http: [`http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`]
         };
     }
 };
 
+ResourceLocator.prototype.getTemplateUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, new TemplateResource(resourcePath), true);
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new TemplateResource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: {
+                urlTemplate: `http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`,
+                subdomains: []
+            }
+        };
+    }
+};
 
-ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource) {
-    var urls = {};
-    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+ResourceLocator.prototype.getUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, new Resource(resourcePath));
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new Resource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: `http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`
+        };
+    }
+};
 
-    if (this.resourcesUrlTemplates.http) {
-        urls.http = this.resourcesUrlTemplates.http({
-            cdn_url: cdnDomain.http,
+function urlForTemplate(tpl, username, cdnDomain, resource, templated) {
+    cdnDomain = cdnDomain || {};
+    if (templated) {
+        return {
+            urlTemplate: tpl({
+                cdn_url: (cdnDomain.hasOwnProperty('urlTemplate') ? cdnDomain.urlTemplate : cdnDomain),
+                user: username,
+                port: this.environment.port,
+                resource: resource.getPath()
+            }),
+            subdomains: cdnDomain.subdomains || []
+        };
+    }
+    if (Array.isArray(cdnDomain)) {
+        return cdnDomain.map(d => tpl({
+            cdn_url: d,
             user: username,
             port: this.environment.port,
-            resource: resource
+            resource: resource.getPath()
+        }));
+    } else {
+        return tpl({
+            cdn_url: cdnDomain,
+            user: username,
+            port: this.environment.port,
+            resource: resource.getPath()
         });
     }
+}
 
+ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource, templated) {
+    var urls = {};
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = urlForTemplate(this.resourcesUrlTemplates.http, username, cdnDomain.http, resource, templated);
+    }
     if (this.resourcesUrlTemplates.https) {
-        urls.https = this.resourcesUrlTemplates.https({
-            cdn_url: cdnDomain.https,
-            user: username,
-            port: this.environment.port,
-            resource: resource
-        });
+        urls.https = urlForTemplate(this.resourcesUrlTemplates.https, username, cdnDomain.https, resource, templated);
     }
 
     return urls;
 };
 
+class Resource {
+    constructor (resourcePath) {
+        this.resourcePath = resourcePath;
+    }
+
+    getPath () {
+        return this.resourcePath;
+    }
+
+    getDomain (domain, subdomains) {
+        if (!subdomains) {
+            return domain;
+        }
+        return domain.replace('{s}', subdomain(subdomains, this.resourcePath));
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        let urls = getUrl(baseUrl, username, this.resourcePath);
+        if (subdomains) {
+            urls = urls.replace('{s}', subdomain(subdomains, this.resourcePath));
+        }
+        return urls;
+    }
+}
+
+class TileResource extends Resource {
+    constructor (resourcePath) {
+        super(resourcePath);
+    }
+
+    getDomain (domain, subdomains) {
+        if (!subdomains) {
+            return domain;
+        }
+        return subdomains.map(s => domain.replace('{s}', s));
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        if (!subdomains) {
+            return [super.getUrl(baseUrl, username)];
+        }
+        return subdomains.map(subdomain => {
+            return getUrl(baseUrl, username, this.resourcePath)
+                .replace('{s}', subdomain);
+        });
+    }
+}
+
+class TemplateResource extends Resource {
+    constructor (resourcePath) {
+        super(resourcePath);
+    }
+
+    getDomain (domain, subdomains) {
+        return {
+            urlTemplate: domain,
+            subdomains: subdomains || []
+        };
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        return {
+            urlTemplate: getUrl(baseUrl, username, this.resourcePath),
+            subdomains: subdomains || []
+        };
+    }
+}
+
+function getUrl(baseUrl, username, path) {
+    return `${baseUrl}/${username}/api/v1/map/${path}`;
+}
+
+function getCdnUrls(serverMetadata, username, resource) {
+    if (serverMetadata && serverMetadata.cdn_url) {
+        var cdnUrl = serverMetadata.cdn_url;
+        var httpUrls = resource.getUrl(`http://${cdnUrl.http}`, username);
+        var httpsUrls = resource.getUrl(`https://${cdnUrl.https}`, username);
+        if (cdnUrl.templates) {
+            var templates = cdnUrl.templates;
+            httpUrls = resource.getUrl(templates.http.url, username, templates.http.subdomains);
+            httpsUrls = resource.getUrl(templates.https.url, username, templates.https.subdomains);
+        }
+        return {
+            http: httpUrls,
+            https: httpsUrls,
+        };
+    }
+    return null;
+}
+
 function getCdnDomain(serverMetadata, resource) {
     if (serverMetadata && serverMetadata.cdn_url) {
         var cdnUrl = serverMetadata.cdn_url;
-        var http = cdnUrl.http;
-        var https = cdnUrl.https;
+        var httpDomain = resource.getDomain(cdnUrl.http);
+        var httpsDomain = resource.getDomain(cdnUrl.https);
         if (cdnUrl.templates) {
             var templates = cdnUrl.templates;
             var httpUrlTemplate = templates.http.url;
             var httpsUrlTemplate = templates.https.url;
-            http = httpUrlTemplate
-                .replace(/^(http[s]*:\/\/)/, '')
-                .replace('{s}', subdomain(templates.http.subdomains, resource));
-            https = httpsUrlTemplate
-                .replace(/^(http[s]*:\/\/)/, '')
-                .replace('{s}', subdomain(templates.https.subdomains, resource));
+            httpDomain = httpUrlTemplate.replace(/^(http[s]*:\/\/)/, '');
+            httpDomain = resource.getDomain(httpDomain, templates.http.subdomains);
+            httpsDomain = httpsUrlTemplate.replace(/^(http[s]*:\/\/)/, '');
+            httpsDomain = resource.getDomain(httpsDomain, templates.https.subdomains);
         }
         return {
-            http: http,
-            https: https,
+            http: httpDomain,
+            https: httpsDomain,
         };
     }
     return null;

lib/cartodb/utils/icu_data_env_setter.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const glob = require('glob');
+const path = require('path');
+
+// See https://github.com/CartoDB/support/issues/984
+// CartoCSS properties text-wrap-width/text-wrap-character not working
+function setICUEnvVariable() {
+    if (process.env.ICU_DATA === undefined) {
+        const regexedPath = '/node_modules/mapnik/lib/binding/*/share/mapnik/icu/';
+        const directory = glob.sync(path.join(__dirname, '../../..', regexedPath));
+
+        if (directory && directory.length > 0) {
+            process.env.ICU_DATA = directory[0];
+        }
+    }
+}
+
+module.exports = setICUEnvVariable;

lib/cartodb/utils/query-utils.js

@@ -32,8 +32,8 @@ module.exports.getAggregationMetadata = ctx => `
         ${getQueryRowEstimation(ctx.query)}
     ),
     geometryType AS (
-        SELECT ST_GeometryType(the_geom) as geom_type
-        FROM (${ctx.query}) AS __cdb_query WHERE the_geom IS NOT NULL LIMIT 1
+        SELECT ST_GeometryType(${ctx.geometryColumn}) as geom_type
+        FROM (${ctx.query}) AS __cdb_query WHERE ${ctx.geometryColumn} IS NOT NULL LIMIT 1
     )
     SELECT
         rows AS count,

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "4.5.0",
+  "version": "5.2.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -33,6 +33,7 @@
     "dot": "~1.0.2",
     "express": "~4.16.0",
     "fastly-purge": "~1.0.1",
+    "glob": "^7.1.2",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "2.6.5",
     "lzma": "~2.3.2",
@@ -46,7 +47,7 @@
     "step-profiler": "~0.3.0",
     "turbo-carto": "0.20.2",
     "underscore": "~1.6.0",
-    "windshaft": "4.1.0",
+    "windshaft": "4.3.3",
     "yargs": "~5.0.0"
   },
   "devDependencies": {
@@ -65,9 +66,8 @@
     "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn",
     "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)",
     "docker-pull": "docker pull cartoimages/windshaft-testing",
-    "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh",
-    "docker-bash": "docker run  -it -v `pwd`:/srv cartoimages/windshaft-testing bash",
-    "docker-publish": "docker push cartoimages/windshaft-carto-testing"
+    "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh && docker ps --filter status=dead --filter status=exited -aq | xargs -r docker rm -v",
+    "docker-bash": "docker run  -it -v `pwd`:/srv cartoimages/windshaft-testing bash"
   },
   "engines": {
     "node": ">=6.9",

test/acceptance/aggregation.js

@@ -92,6 +92,14 @@ describe('aggregation', function () {
         }
     `;
 
+    const POINTS_SQL_ONLY_WEBMERCATOR = `
+    select
+        x + 4 as cartodb_id,
+        st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator,
+        x as value
+    from generate_series(-3, 3) x
+    `;
+
     function createVectorMapConfig (layers = [
         {
             type: 'cartodb',
@@ -289,7 +297,8 @@ describe('aggregation', function () {
                         options: {
                             sql: POINTS_SQL_2,
                             aggregation: {
-                                threshold: 1
+                                threshold: 1,
+                                placement: 'centroid'
                             },
                             cartocss: '#layer { marker-width: [value]; }',
                             cartocss_version: '2.3.0'
@@ -309,6 +318,45 @@ describe('aggregation', function () {
                 });
             });
 
+            it('should provide all columns in the default aggregation ',
+            function (done) {
+                const response = {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                };
+
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_2,
+                            aggregation: {
+                                threshold: 1
+                            },
+                            cartocss: '#layer { marker-width: [value]; }',
+                            cartocss_version: '2.3.0'
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+                this.testClient.getLayergroup({ response }, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+
+                    assert.equal(typeof body.metadata, 'object');
+                    assert.ok(Array.isArray(body.metadata.layers));
+
+                    body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt));
+                    body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png));
+                    done();
+                });
+            });
+
             it('should skip aggregation to create a layergroup with aggregation defined already', function (done) {
                 const mapConfig = createVectorMapConfig([
                     {
@@ -345,6 +393,44 @@ describe('aggregation', function () {
                 });
             });
 
+            it('skip default aggregation by setting `aggregation: false` for just one layer', function (done) {
+                const mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                threshold: 1
+                            }
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_2,
+                            aggregation: false
+                        }
+                    }
+
+                ]);
+
+                this.testClient = new TestClient(mapConfig);
+
+                this.testClient.getLayergroup((err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(typeof body.metadata, 'object');
+                    assert.ok(Array.isArray(body.metadata.layers));
+
+                    assert.equal(body.metadata.layers[0].meta.aggregation.mvt, true);
+                    assert.equal(body.metadata.layers[1].meta.aggregation.mvt, false);
+
+                    done();
+                });
+            });
+
             it('when the aggregation param is not valid should respond with error', function (done) {
                 const mapConfig = createVectorMapConfig([
                     {
@@ -431,6 +517,8 @@ describe('aggregation', function () {
                         type: 'cartodb',
                         options: {
                             sql: POLYGONS_SQL_1,
+                            cartocss: '#layer { marker-width: [value]; }',
+                            cartocss_version: '2.3.0',
                             aggregation: {
                                 threshold: 1
                             }
@@ -456,9 +544,14 @@ describe('aggregation', function () {
                                 ' Aggregation is available only for geometry type: ST_Point'
                         ],
                         errors_with_context:[{
-                            type: 'unknown',
+                            type: 'layer',
                             message: 'Unsupported geometry type: ST_Polygon.' +
-                            ' Aggregation is available only for geometry type: ST_Point'
+                            ' Aggregation is available only for geometry type: ST_Point',
+                            layer: {
+                                id: 'layer0',
+                                index: 0,
+                                type: 'mapnik'
+                            }
                         }]
                     });
 
@@ -524,6 +617,198 @@ describe('aggregation', function () {
                 });
             });
 
+            it('when dimensions is provided should return a tile returning the column used as dimensions',
+            function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                threshold: 1,
+                                dimensions: {
+                                    value: "value"
+                                }
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+                const options = {
+                    format: 'mvt'
+                };
+                this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    const tileJSON = tile.toJSON();
+
+                    tileJSON[0].features.forEach(feature => assert.equal(typeof feature.properties.value, 'number'));
+
+                    done();
+                });
+            });
+
+            ['centroid', 'point-sample', 'point-grid'].forEach(placement => {
+                it(`dimensions should work for ${placement} placement`, function(done) {
+                    this.mapConfig = createVectorMapConfig([
+                        {
+                            type: 'cartodb',
+                            options: {
+                                sql: POINTS_SQL_1,
+                                aggregation: {
+                                    placement: placement ,
+                                    threshold: 1,
+                                    dimensions: {
+                                        value: "value"
+                                    }
+                                }
+                            }
+                        }
+                    ]);
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    const options = {
+                        format: 'mvt'
+                    };
+                    this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        const tileJSON = tile.toJSON();
+
+                        tileJSON[0].features.forEach(
+                            feature => assert.equal(typeof feature.properties.value, 'number')
+                        );
+
+                        done();
+                    });
+                });
+            });
+
+            it(`dimensions should trigger non-default aggregation`, function(done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_2,
+                            aggregation: {
+                                threshold: 1,
+                                dimensions: {
+                                    value: "value"
+                                }
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+                const options = {
+                    format: 'mvt'
+                };
+                this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    const tileJSON = tile.toJSON();
+
+                    tileJSON[0].features.forEach(
+                        feature => assert.equal(typeof feature.properties.value, 'number')
+                    );
+                    tileJSON[0].features.forEach(
+                        feature => assert.equal(typeof feature.properties.sqrt_value, 'undefined')
+                    );
+
+                    done();
+                });
+            });
+
+            it(`aggregation columns should trigger non-default aggregation`, function(done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_2,
+                            aggregation: {
+                                threshold: 1,
+                                columns: {
+                                    value: {
+                                        aggregate_function: 'sum',
+                                        aggregated_column: 'value'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+                const options = {
+                    format: 'mvt'
+                };
+                this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    const tileJSON = tile.toJSON();
+
+                    tileJSON[0].features.forEach(
+                        feature => assert.equal(typeof feature.properties.value, 'number')
+                    );
+                    tileJSON[0].features.forEach(
+                        feature => assert.equal(typeof feature.properties.sqrt_value, 'undefined')
+                    );
+
+                    done();
+                });
+            });
+
+            ['centroid', 'point-sample', 'point-grid'].forEach(placement => {
+                it(`aggregations with base column names should work for ${placement} placement`, function(done) {
+                    this.mapConfig = createVectorMapConfig([
+                        {
+                            type: 'cartodb',
+                            options: {
+                                sql: POINTS_SQL_1,
+                                aggregation: {
+                                    placement: placement ,
+                                    threshold: 1,
+                                    columns: {
+                                        value: {
+                                            aggregate_function: 'sum',
+                                            aggregated_column: 'value'
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    ]);
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    const options = {
+                        format: 'mvt'
+                    };
+                    this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        const tileJSON = tile.toJSON();
+
+                        tileJSON[0].features.forEach(
+                            feature => assert.equal(typeof feature.properties.value, 'number')
+                        );
+
+                        done();
+                    });
+                });
+            });
+
             it('should work when the sql has single quotes', function (done) {
                 this.mapConfig = createVectorMapConfig([
                     {
@@ -531,6 +816,7 @@ describe('aggregation', function () {
                         options: {
                             sql: `
                             SELECT
+                                cartodb_id,
                                 the_geom_webmercator,
                                 the_geom,
                                 value,
@@ -686,6 +972,472 @@ describe('aggregation', function () {
                     done();
                 });
             });
+
+            it('aggregates with full-sample placement by default', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            resolution: 256,
+                            aggregation: {
+                                threshold: 1
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                this.testClient.getTile(0, 0, 0, { format: 'mvt' }, function (err, res, mvt) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    const geojsonTile = JSON.parse(mvt.toGeoJSONSync(0));
+
+                    assert.ok(Array.isArray(geojsonTile.features));
+                    assert.ok(geojsonTile.features.length > 0);
+
+                    const feature = geojsonTile.features[0];
+
+                    assert.ok(feature.properties.hasOwnProperty('value'), 'Missing value property');
+
+                    done();
+                });
+            });
+
+            it('should fail with bad resolution', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        id: 'wadus',
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                resolution: 'wadus',
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                const options = {
+                    response: {
+                        status: 400
+                    }
+                };
+
+                this.testClient.getLayergroup(options, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.deepEqual(body, {
+                        errors: [ 'Invalid resolution, should be a number greather than 0' ],
+                        errors_with_context:[{
+                            type: 'layer',
+                            message: 'Invalid resolution, should be a number greather than 0',
+                            layer: {
+                                "id": "wadus",
+                                "index": 0,
+                                "type": "mapnik"
+                            }
+                        }]
+                    });
+                    done();
+                });
+            });
+
+            it('should fail with bad placement', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                placement: 'wadus',
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                const options = {
+                    response: {
+                        status: 400
+                    }
+                };
+
+                this.testClient.getLayergroup(options, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.deepEqual(body, {
+                        errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample'],
+                        errors_with_context:[{
+                            type: 'layer',
+                            message: 'Invalid placement. Valid values: centroid, point-grid, point-sample',
+                            layer: {
+                                id: "layer0",
+                                index: 0,
+                                type: "mapnik",
+                            }
+                        }]
+                    });
+
+                    done();
+                });
+            });
+
+            it('should fail with bad threshold', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                threshold: 'wadus',
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                const options = {
+                    response: {
+                        status: 400
+                    }
+                };
+
+                this.testClient.getLayergroup(options, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.deepEqual(body, {
+                        errors: [ 'Invalid threshold, should be a number greather than 0' ],
+                        errors_with_context:[{
+                            type: 'layer',
+                            message: 'Invalid threshold, should be a number greather than 0',
+                            layer: {
+                                "id": "layer0",
+                                "index": 0,
+                                "type": "mapnik"
+                            }
+                        }]
+                    });
+
+                    done();
+                });
+            });
+
+            it('should fail with bad column name', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                columns : {
+                                    '': {
+                                        aggregate_function: 'count',
+                                        aggregated_column: 'value',
+                                    }
+                                },
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                const options = {
+                    response: {
+                        status: 400
+                    }
+                };
+
+                this.testClient.getLayergroup(options, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.deepEqual(body, {
+                        errors: [ 'Invalid column name, should be a non empty string' ],
+                        errors_with_context:[{
+                            type: 'layer',
+                            message: 'Invalid column name, should be a non empty string',
+                            layer: {
+                                "id": "layer0",
+                                "index": 0,
+                                "type": "mapnik"
+                            }
+                        }]
+                    });
+
+                    done();
+                });
+            });
+
+            it('should fail with bad aggregated function', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                columns : {
+                                    'wadus_function': {
+                                        aggregate_function: 'wadus',
+                                        aggregated_column: 'value',
+                                    }
+                                },
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                const options = {
+                    response: {
+                        status: 400
+                    }
+                };
+
+                this.testClient.getLayergroup(options, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.deepEqual(body, {
+                        errors: [ 'Unsupported aggregation function wadus, ' +
+                                    'valid ones: count, avg, sum, min, max, mode' ],
+                        errors_with_context:[{
+                            type: 'layer',
+                            message: 'Unsupported aggregation function wadus, ' +
+                                    'valid ones: count, avg, sum, min, max, mode',
+                            layer: {
+                                "id": "layer0",
+                                "index": 0,
+                                "type": "mapnik"
+                            }
+                        }]
+                    });
+
+                    done();
+                });
+            });
+
+            it('should fail with bad aggregated columns', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                columns : {
+                                    'total_wadus': {
+                                        aggregate_function: 'sum',
+                                        aggregated_column: '',
+                                    }
+                                },
+                            }
+                        }
+                    }
+                ]);
+                this.testClient = new TestClient(this.mapConfig);
+
+                const options = {
+                    response: {
+                        status: 400
+                    }
+                };
+
+                this.testClient.getLayergroup(options, (err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.deepEqual(body, {
+                        errors: [ 'Invalid aggregated column, should be a non empty string' ],
+                        errors_with_context:[{
+                            type: 'layer',
+                            message: 'Invalid aggregated column, should be a non empty string',
+                            layer: {
+                                "id": "layer0",
+                                "index": 0,
+                                "type": "mapnik"
+                            }
+                        }]
+                    });
+
+                    done();
+                });
+            });
+
+
+            it('should skip aggregation w/o failing when is Vector Only MapConfig and layer has polygons',
+            function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POLYGONS_SQL_1
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                this.testClient.getLayergroup((err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(typeof body.metadata, 'object');
+                    assert.ok(Array.isArray(body.metadata.layers));
+
+                    body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.mvt));
+                    body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.png));
+
+                    const options = {
+                        format: 'mvt'
+                    };
+
+                    this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        const tileJSON = tile.toJSON();
+
+                        assert.equal(tileJSON[0].features.length, 7);
+
+                        done();
+                    });
+                });
+            });
+
+            it('should skip aggregation for polygons (w/o failing) and aggregate when the layer has points',
+            function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POLYGONS_SQL_1
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_1,
+                            aggregation: {
+                                threshold: 1
+                            }
+                        }
+                    }
+                ]);
+
+                this.testClient = new TestClient(this.mapConfig);
+
+                this.testClient.getLayergroup((err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(typeof body.metadata, 'object');
+                    assert.ok(Array.isArray(body.metadata.layers));
+
+                    assert.equal(body.metadata.layers[0].meta.aggregation.mvt, false);
+                    assert.equal(body.metadata.layers[1].meta.aggregation.mvt, true);
+
+                    const options = {
+                        format: 'mvt'
+                    };
+
+                    this.testClient.getTile(0, 0, 0, options, (err, res, tile) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        const tileJSON = tile.toJSON();
+
+                        assert.equal(tileJSON[0].features.length, 7);
+
+                        done();
+                    });
+                });
+            });
+
+            ['centroid', 'point-sample', 'point-grid'].forEach(placement => {
+                it(`cartodb_id should be present in ${placement} aggregation`, function(done) {
+                    this.mapConfig = createVectorMapConfig([
+                        {
+                            type: 'cartodb',
+                            options: {
+                                sql: POINTS_SQL_1,
+                                aggregation: {
+                                    placement: placement,
+                                    threshold: 1
+                                },
+                                cartocss: '#layer { marker-width: 1; }',
+                                cartocss_version: '2.3.0',
+                                interactivity: ['cartodb_id']
+                            }
+                        }
+                    ]);
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    this.testClient.getLayergroup((err, body) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        assert.equal(typeof body.metadata, 'object');
+                        assert.ok(Array.isArray(body.metadata.layers));
+
+                        body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt));
+                        body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png));
+
+                        done();
+                    });
+                });
+            });
+
+            it('should only require the_geom_webmercator for aggregation', function (done) {
+                this.mapConfig = createVectorMapConfig([
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: POINTS_SQL_ONLY_WEBMERCATOR,
+                            aggregation: {
+                                threshold: 1
+                            }
+                        }
+                    }
+                ]);
+                this.testClient = new TestClient(this.mapConfig);
+
+                this.testClient.getLayergroup((err, body) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(typeof body.metadata, 'object');
+                    assert.ok(Array.isArray(body.metadata.layers));
+
+                    body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt));
+                    body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.png));
+
+                    done();
+                });
+            });
         });
     });
 });

test/acceptance/label-wrap.js

@@ -0,0 +1,69 @@
+require('../support/test_helper');
+var TestClient = require('../support/test-client');
+
+var assert = require('../support/assert');
+var IMAGE_TOLERANCE = 5;
+
+describe('CartoCSS wrap', function () {
+    const options = {
+        sql: `
+            SELECT 
+                5 as cartodb_id, 
+                ST_Transform(ST_SetSRID(ST_MakePoint(-57.65625,-15.6230368),4326),3857) as the_geom_webmercator,  
+                ST_SetSRID(ST_MakePoint(-57.65625,-15.62303683),4326) as the_geom, 
+                'South America' as continent
+        `,
+        cartocss: `
+            #continent_points::labels {
+                text-name: [continent];
+                text-face-name: 'DejaVu Sans Book';
+                text-size: 10;
+                text-fill: lighten(#000,40);
+                text-transform: uppercase;
+                text-wrap-width: 30;
+                text-character-spacing: 2;
+                text-placement: point;
+                text-placement-type: dummy;
+                [zoom >= 3]{
+                    text-character-spacing: 2;
+                    text-size: 11;
+                }
+            }
+        `,
+        cartocss_version: '3.0.12'
+    };
+
+    const type = 'mapnik';
+
+    const mapConfig = {
+        version: '1.6.0',
+        layers: [
+            {
+                type,
+                id: 'layerLabel',
+                options
+            }
+        ]
+    };
+
+
+    var keysToDelete;
+
+    beforeEach(function () {
+        keysToDelete = {};
+    });
+
+    afterEach(function (done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        }
+    });
+
+    it("Label should be text-wrapped", function (done) {
+        this.testClient = new TestClient(mapConfig);
+        this.testClient.getTile(1, 0, 1, { layers: [0] }, (err, res, body) => {
+            var textWrapPath = './test/fixtures/text_wrap.png';
+            assert.imageIsSimilarToFile(body, textWrapPath, IMAGE_TOLERANCE, done);
+        });
+    });
+});

test/acceptance/layers-filters.js

@@ -0,0 +1,54 @@
+require('../support/test_helper');
+var TestClient = require('../support/test-client');
+
+describe('layers filters', function() {
+    const type = 'mapnik';
+    const sql = 'select * from populated_places_simple_reduced';
+    const cartocss = `#points {
+        marker-fill-opacity: 1.0;
+        marker-line-color: #FFF;
+        marker-line-width: 0.5;
+        marker-line-opacity: 1.0;
+        marker-placement: point;
+        marker-type: ellipse;
+        marker-width: 8;
+        marker-fill: red;
+        marker-allow-overlap: true;
+    }`;
+    const cartocss_version = '3.0.12';
+    const options = {
+        sql,
+        cartocss,
+        cartocss_version
+    };
+
+    const mapConfig = {
+        version: '1.6.0',
+        layers: [
+            {
+                type,
+                id: 'layerA',
+                options
+            },
+            {
+                type,
+                id: 'layerB',
+                options
+            }
+        ]
+    };
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        }
+    });
+
+    ['layerA', 'layerB'].forEach(layer => {
+        it(`should work for individual layer ids: ${layer}`, function (done) {
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getTile(0, 0, 0, { layers: layer }, done);
+        });
+    });
+
+});

test/acceptance/ported/torque_boundaries.js

@@ -257,6 +257,14 @@ describe('torque boundary points', function() {
                     assert.equal(res.statusCode, 200, res.body);
                     assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
                     var parsed = JSON.parse(res.body);
+                    /* Order the JSON first by descending x__uint8 and ascending
+                     * y__uint8 */
+                    parsed.sort(function(a,b) {
+                        if (a.x__uint8 === b.x__uint8) {
+                            return (a.y__uint8 > b.y__uint8);
+                        }
+                        return (a.x__uint8 < b.x__uint8);
+                    });
 
                     var i = 0;
                     tileRequest.expects.forEach(function(expected) {
@@ -424,7 +432,7 @@ describe('torque boundary points', function() {
 
                 var parsed = JSON.parse(res.body);
 
-                assert.deepEqual(parsed, [
+                assert.deepEqual(parsed.sort(function(a,b){return a.x__uint8 > b.x__uint8;}), [
                     {
                         x__uint8: 47,
                         y__uint8: 127,
@@ -438,7 +446,6 @@ describe('torque boundary points', function() {
                         dates__uint16: [0]
                     }
                 ]);
-
                 done();
             });
         });

test/acceptance/tilejson.js

@@ -0,0 +1,219 @@
+require('../support/test_helper');
+
+const assert = require('../support/assert');
+const TestClient = require('../support/test-client');
+
+describe('tilejson', function() {
+
+    function tilejsonValidation(tilejson, shouldHaveGrid = false) {
+        assert.equal(tilejson.tilejson, '2.2.0');
+
+        assert.ok(Array.isArray(tilejson.tiles), JSON.stringify(tilejson));
+        assert.ok(tilejson.tiles.length > 0);
+
+        if (shouldHaveGrid) {
+            assert.ok(Array.isArray(tilejson.grids));
+            assert.ok(tilejson.grids.length > 0);
+        }
+
+    }
+
+    const sql = 'SELECT * FROM populated_places_simple_reduced';
+    const cartocss = TestClient.CARTOCSS.POINTS;
+    const cartocss_version = '3.0.12';
+
+    const RASTER_LAYER = {
+        options: {
+            sql, cartocss, cartocss_version
+        }
+    };
+    const RASTER_INTERACTIVITY_LAYER = {
+        options: {
+            sql, cartocss, cartocss_version,
+            interactivity: ['cartodb_id']
+        }
+    };
+    const VECTOR_LAYER = {
+        options: {
+            sql
+        }
+    };
+    const PLAIN_LAYER = {
+        type: 'plain',
+        options: {
+            color: '#000000'
+        }
+    };
+
+    function mapConfig(layers) {
+        return {
+            version: '1.7.0',
+            layers: Array.isArray(layers) ? layers : [layers]
+        };
+    }
+
+    describe('per layer', function() {
+        it('should expose raster + vector tilejson for raster layers', function(done) {
+            var testClient = new TestClient(mapConfig(RASTER_LAYER));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                assert.equal(metadata.layers.length, 1);
+
+                const layer = metadata.layers[0];
+                assert.deepEqual(Object.keys(layer.tilejson), ['vector', 'raster']);
+
+                Object.keys(layer.tilejson).forEach(k => {
+                    tilejsonValidation(layer.tilejson[k]);
+                });
+
+                testClient.drain(done);
+            });
+        });
+
+        it('should expose just the vector tilejson vector only layers', function(done) {
+            var testClient = new TestClient(mapConfig(VECTOR_LAYER));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                assert.equal(metadata.layers.length, 1);
+
+                const layer = metadata.layers[0];
+                assert.deepEqual(Object.keys(layer.tilejson), ['vector']);
+
+                Object.keys(layer.tilejson).forEach(k => {
+                    tilejsonValidation(layer.tilejson[k]);
+                });
+
+                testClient.drain(done);
+            });
+        });
+
+        it('should expose just the raster tilejson plain layers', function(done) {
+            var testClient = new TestClient(mapConfig(PLAIN_LAYER));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                assert.equal(metadata.layers.length, 1);
+
+                const layer = metadata.layers[0];
+                assert.deepEqual(Object.keys(layer.tilejson), ['raster']);
+
+                Object.keys(layer.tilejson).forEach(k => {
+                    tilejsonValidation(layer.tilejson[k]);
+                });
+
+                testClient.drain(done);
+            });
+        });
+
+        it('should expose grids for the raster layer with interactivity', function(done) {
+            var testClient = new TestClient(mapConfig(RASTER_INTERACTIVITY_LAYER));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                assert.equal(metadata.layers.length, 1);
+
+                const layer = metadata.layers[0];
+                assert.deepEqual(Object.keys(layer.tilejson), ['vector', 'raster']);
+
+                tilejsonValidation(layer.tilejson.vector);
+                tilejsonValidation(layer.tilejson.raster, true);
+
+                testClient.drain(done);
+            });
+        });
+
+        it('should work with several layers', function(done) {
+            var testClient = new TestClient(mapConfig([RASTER_LAYER, RASTER_INTERACTIVITY_LAYER]));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                assert.equal(metadata.layers.length, 2);
+
+                assert.deepEqual(Object.keys(metadata.layers[0].tilejson), ['vector', 'raster']);
+                tilejsonValidation(metadata.layers[0].tilejson.vector);
+                tilejsonValidation(metadata.layers[0].tilejson.raster);
+
+                assert.deepEqual(Object.keys(metadata.layers[1].tilejson), ['vector', 'raster']);
+                tilejsonValidation(metadata.layers[1].tilejson.vector);
+                tilejsonValidation(metadata.layers[1].tilejson.raster, true);
+
+                testClient.drain(done);
+            });
+        });
+    });
+
+    describe('root tilejson', function() {
+
+        it('should expose just the `vector` tilejson and URL when for vector only mapnik layers', function(done) {
+            var testClient = new TestClient(mapConfig(VECTOR_LAYER));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                const tilejson = metadata.tilejson;
+                assert.deepEqual(Object.keys(tilejson), ['vector']);
+
+                Object.keys(tilejson).forEach(k => {
+                    tilejsonValidation(tilejson[k]);
+                });
+
+                const url = metadata.url;
+                assert.deepEqual(Object.keys(url), ['vector']);
+
+                assert.ok(url.vector.urlTemplate);
+                assert.ok(url.vector.subdomains);
+
+                testClient.drain(done);
+            });
+        });
+
+        it('should expose just the `vector` and `raster` tilejson and urls for mapnik layers', function(done) {
+            var testClient = new TestClient(mapConfig(RASTER_LAYER));
+
+            testClient.getLayergroup(function(err, layergroupResult) {
+                assert.ok(!err, err);
+                const metadata = layergroupResult.metadata;
+                assert.ok(metadata);
+
+                const tilejson = metadata.tilejson;
+                assert.deepEqual(Object.keys(tilejson), ['vector', 'raster']);
+
+                Object.keys(tilejson).forEach(k => {
+                    tilejsonValidation(tilejson[k]);
+                });
+
+                const url = metadata.url;
+                assert.deepEqual(Object.keys(url), ['vector', 'raster']);
+
+                assert.ok(url.vector.urlTemplate);
+                assert.ok(url.vector.subdomains);
+
+                assert.ok(url.raster.urlTemplate);
+                assert.ok(url.raster.subdomains);
+
+                testClient.drain(done);
+            });
+        });
+
+    });
+
+});

test/support/test_helper.js

@@ -15,12 +15,14 @@ var redis = require('redis');
 var nock = require('nock');
 var log4js = require('log4js');
 var pg = require('pg');
+const setICUEnvVariable = require('../../lib/cartodb/utils/icu_data_env_setter');
 
 // set environment specific variables
 global.environment  = require(__dirname + '/../../config/environments/test');
 global.environment.name = 'test';
 process.env.NODE_ENV = 'test';
 
+setICUEnvVariable();
 
 // don't output logs in test environment to reduce noise
 log4js.configure({ appenders: [] });
@@ -75,6 +77,22 @@ function checkSurrogateKey(res, expectedKey) {
     assert.deepEqual(keys, expectedKeys);
 }
 
+var uncaughtExceptions = [];
+process.on('uncaughtException', function(err) {
+    uncaughtExceptions.push(err);
+});
+beforeEach(function() {
+    uncaughtExceptions = [];
+});
+//global afterEach to capture uncaught exceptions
+afterEach(function() {
+    assert.equal(
+        uncaughtExceptions.length,
+        0,
+        'uncaughtException:\n\n' + uncaughtExceptions.map(err => err.stack).join('\n\n'));
+});
+
+
 var redisClient;
 
 beforeEach(function() {

test/unit/cartodb/model/resource-locator.test.js

@@ -3,9 +3,10 @@ require('../../../support/test_helper');
 var assert = require('../../../support/assert');
 var ResourceLocator = require('../../../../lib/cartodb/models/resource-locator');
 
-describe('ResourceLocator.getUrls', function() {
+describe('ResourceLocator', function() {
     var USERNAME = 'username';
     var RESOURCE = 'wadus';
+    var TILE_RESOURCE = 'wadus/{z}/{x}/{y}.png';
     var HTTP_SUBDOMAINS = ['1', '2', '3', '4'];
     var HTTPS_SUBDOMAINS = ['a', 'b', 'c', 'd'];
 
@@ -15,118 +16,292 @@ describe('ResourceLocator.getUrls', function() {
         assert.ok(urls);
     });
 
-    var BASIC_ENVIRONMENT = {
-        serverMetadata: {
-            cdn_url: {
-                http: 'cdn.carto.com',
-                https: 'cdn.ssl.carto.com'
-            }
-        }
-    };
-    it('should return default urls when basic http and https domains are provided', function() {
-        var resourceLocator = new ResourceLocator(BASIC_ENVIRONMENT);
-        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
-        assert.ok(urls);
+    describe('basic', function() {
+
+        var BASIC_ENVIRONMENT = {
+            serverMetadata: {
+                cdn_url: {
+                    http: 'cdn.carto.com',
+                    https: 'cdn.ssl.carto.com'
+                }
+            }
+        };
+
+        describe('getUrls', function() {
+            it('should return default urls when basic http and https domains are provided', function() {
+                var resourceLocator = new ResourceLocator(BASIC_ENVIRONMENT);
+                var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+                assert.ok(urls);
+
+                assert.equal(urls.http, ['http://cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
+                assert.equal(urls.https, ['https://cdn.ssl.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
+            });
+        });
+
+        describe('getTileUrls', function() {
+            it('should return default urls when basic http and https domains are provided', function() {
+                var resourceLocator = new ResourceLocator(BASIC_ENVIRONMENT);
+                var urls = resourceLocator.getTileUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(
+                    urls.http,
+                    [`http://cdn.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`]
+                );
+                assert.deepEqual(
+                    urls.https,
+                    [`https://cdn.ssl.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`]
+                );
+            });
+        });
+
+        describe('getTemplateUrls', function() {
+            it('should return default urls when basic http and https domains are provided', function() {
+                var resourceLocator = new ResourceLocator(BASIC_ENVIRONMENT);
+                var urls = resourceLocator.getTemplateUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(urls.http, {
+                    urlTemplate: `http://cdn.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: []
+                });
+                assert.deepEqual(urls.https, {
+                    urlTemplate: `https://cdn.ssl.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: []
+                });
+            });
+        });
 
-        assert.equal(urls.http, ['http://cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
-        assert.equal(urls.https, ['https://cdn.ssl.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/'));
     });
 
-    var RESOURCE_TEMPLATES_ENVIRONMENT = {
-        serverMetadata: {
-            cdn_url: {
-                http: 'cdn.carto.com',
-                https: 'cdn.ssl.carto.com'
-            }
-        },
-        resources_url_templates: {
-            http: 'http://{{=it.user}}.localhost.lan/api/v1/map',
-            https: 'https://{{=it.user}}.ssl.localhost.lan/api/v1/map'
-        }
-    };
-    it('resources_url_templates should take precedence over http and https domains', function() {
-        var resourceLocator = new ResourceLocator(RESOURCE_TEMPLATES_ENVIRONMENT);
-        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
-        assert.ok(urls);
+    describe('resource templates', function() {
+
+        var RESOURCE_TEMPLATES_ENVIRONMENT = {
+            serverMetadata: {
+                cdn_url: {
+                    http: 'cdn.carto.com',
+                    https: 'cdn.ssl.carto.com'
+                }
+            },
+            resources_url_templates: {
+                http: 'http://{{=it.user}}.localhost.lan/api/v1/map',
+                https: 'https://{{=it.user}}.ssl.localhost.lan/api/v1/map'
+            }
+        };
+
+        describe('getUrls', function() {
+            it('resources_url_templates should take precedence over http and https domains', function() {
+                var resourceLocator = new ResourceLocator(RESOURCE_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+                assert.ok(urls);
+
+                assert.equal(
+                    urls.http,
+                    ['http://' + USERNAME + '.localhost.lan', 'api/v1/map', RESOURCE].join('/')
+                );
+                assert.equal(
+                    urls.https,
+                    ['https://' + USERNAME + '.ssl.localhost.lan', 'api/v1/map', RESOURCE].join('/')
+                );
+            });
+        });
+
+        describe('getTileUrls', function() {
+            it('resources_url_templates should take precedence over http and https domains', function() {
+                var resourceLocator = new ResourceLocator(RESOURCE_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getTileUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(
+                    urls.http,
+                    [`http://${USERNAME}.localhost.lan/api/v1/map/${TILE_RESOURCE}`]
+                );
+                assert.deepEqual(
+                    urls.https,
+                    [`https://${USERNAME}.ssl.localhost.lan/api/v1/map/${TILE_RESOURCE}`]
+                );
+            });
+        });
+
+        describe('getTemplateUrls', function() {
+            it('resources_url_templates should take precedence over http and https domains', function() {
+                var resourceLocator = new ResourceLocator(RESOURCE_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getTemplateUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(urls.http, {
+                    urlTemplate: `http://${USERNAME}.localhost.lan/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: []
+                });
+                assert.deepEqual(urls.https, {
+                    urlTemplate: `https://${USERNAME}.ssl.localhost.lan/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: []
+                });
+            });
+        });
 
-        assert.equal(urls.http, ['http://' + USERNAME + '.localhost.lan', 'api/v1/map', RESOURCE].join('/'));
-        assert.equal(urls.https, ['https://' + USERNAME + '.ssl.localhost.lan', 'api/v1/map', RESOURCE].join('/'));
     });
 
-    var CDN_TEMPLATES_ENVIRONMENT = {
-        serverMetadata: {
-            cdn_url: {
-                http: 'cdn.carto.com',
-                https: 'cdn.ssl.carto.com',
-                templates: {
-                    http: {
-                        url: "http://{s}.cdn.carto.com",
-                        subdomains: HTTP_SUBDOMAINS
-                    },
-                    https: {
-                        url: "https://cdn_{s}.ssl.cdn.carto.com",
-                        subdomains: HTTPS_SUBDOMAINS
+    describe('cdn templates', function() {
+
+        var CDN_TEMPLATES_ENVIRONMENT = {
+            serverMetadata: {
+                cdn_url: {
+                    http: 'cdn.carto.com',
+                    https: 'cdn.ssl.carto.com',
+                    templates: {
+                        http: {
+                            url: "http://{s}.cdn.carto.com",
+                            subdomains: HTTP_SUBDOMAINS
+                        },
+                        https: {
+                            url: "https://cdn_{s}.ssl.cdn.carto.com",
+                            subdomains: HTTPS_SUBDOMAINS
+                        }
                     }
                 }
             }
-        }
-    };
-    it('cdn_url templates should take precedence over http and https domains', function() {
-        var resourceLocator = new ResourceLocator(CDN_TEMPLATES_ENVIRONMENT);
-        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
-        assert.ok(urls);
+        };
 
-        var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
-        var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
+        describe('getUrls', function() {
+            it('cdn_url templates should take precedence over http and https domains', function() {
+                var resourceLocator = new ResourceLocator(CDN_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+                assert.ok(urls);
+
+                var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
+                var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
+
+                assert.equal(
+                    urls.http,
+                    ['http://' + httpSubdomain + '.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
+                );
+                assert.equal(
+                    urls.https,
+                    ['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
+                );
+            });
+        });
+
+        describe('getTileUrls', function() {
+            it('cdn_url templates should take precedence over http and https domains', function() {
+                var resourceLocator = new ResourceLocator(CDN_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getTileUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(
+                    urls.http,
+                    HTTP_SUBDOMAINS
+                        .map(s => `http://${s}.cdn.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`)
+                );
+                assert.deepEqual(
+                    urls.https,
+                    HTTPS_SUBDOMAINS
+                        .map(s => `https://cdn_${s}.ssl.cdn.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`)
+                );
+            });
+        });
+
+        describe('getTemplateUrls', function() {
+            it('cdn_url templates should take precedence over http and https domains', function() {
+                var resourceLocator = new ResourceLocator(CDN_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getTemplateUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(urls.http, {
+                    urlTemplate: `http://{s}.cdn.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: HTTP_SUBDOMAINS
+                });
+                assert.deepEqual(urls.https, {
+                    urlTemplate: `https://cdn_{s}.ssl.cdn.carto.com/${USERNAME}/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: HTTPS_SUBDOMAINS
+                });
+            });
+        });
 
-        assert.equal(
-            urls.http,
-            ['http://' + httpSubdomain + '.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
-        );
-        assert.equal(
-            urls.https,
-            ['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', USERNAME, 'api/v1/map', RESOURCE].join('/')
-        );
     });
 
-    var CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT = {
-        serverMetadata: {
-            cdn_url: {
-                http: 'cdn.carto.com',
-                https: 'cdn.ssl.carto.com',
-                templates: {
-                    http: {
-                        url: "http://{s}.cdn.carto.com",
-                        subdomains: HTTP_SUBDOMAINS
-                    },
-                    https: {
-                        url: "https://cdn_{s}.ssl.cdn.carto.com",
-                        subdomains: HTTPS_SUBDOMAINS
+    describe('cdn and resource templates', function() {
+
+        var CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT = {
+            serverMetadata: {
+                cdn_url: {
+                    http: 'cdn.carto.com',
+                    https: 'cdn.ssl.carto.com',
+                    templates: {
+                        http: {
+                            url: "http://{s}.cdn.carto.com",
+                            subdomains: HTTP_SUBDOMAINS
+                        },
+                        https: {
+                            url: "https://cdn_{s}.ssl.cdn.carto.com",
+                            subdomains: HTTPS_SUBDOMAINS
+                        }
                     }
                 }
+            },
+            resources_url_templates: {
+                http: 'http://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map',
+                https: 'https://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map'
             }
-        },
-        resources_url_templates: {
-            http: 'http://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map',
-            https: 'https://{{=it.cdn_url}}/u/{{=it.user}}/api/v1/map'
-        }
-    };
-    it('should mix cdn_url templates and resources_url_templates', function() {
-        var resourceLocator = new ResourceLocator(CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT);
-        var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
-        assert.ok(urls);
+        };
 
-        var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
-        var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
+        describe('getUrls', function() {
+            it('should mix cdn_url templates and resources_url_templates', function() {
+                var resourceLocator = new ResourceLocator(CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getUrls(USERNAME, RESOURCE);
+                assert.ok(urls);
+
+                var httpSubdomain = ResourceLocator.subdomain(HTTP_SUBDOMAINS, RESOURCE);
+                var httpsSubdomain = ResourceLocator.subdomain(HTTPS_SUBDOMAINS, RESOURCE);
+
+                assert.equal(
+                    urls.http,
+                    ['http://' + httpSubdomain + '.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
+                );
+                assert.equal(
+                    urls.https,
+                    `https://cdn_${httpsSubdomain}.ssl.cdn.carto.com/u/${USERNAME}/api/v1/map/${RESOURCE}`
+                );
+            });
+        });
+
+        describe('getTileUrls', function() {
+            it('should mix cdn_url templates and resources_url_templates', function() {
+                var resourceLocator = new ResourceLocator(CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getTileUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(
+                    urls.http,
+                    HTTP_SUBDOMAINS
+                        .map(s => `http://${s}.cdn.carto.com/u/${USERNAME}/api/v1/map/${TILE_RESOURCE}`)
+                );
+                assert.deepEqual(
+                    urls.https,
+                    HTTPS_SUBDOMAINS
+                        .map(s => `https://cdn_${s}.ssl.cdn.carto.com/u/${USERNAME}/api/v1/map/${TILE_RESOURCE}`)
+                );
+            });
+        });
+
+        describe('getTemplateUrls', function() {
+            it('should mix cdn_url templates and resources_url_templates', function() {
+                var resourceLocator = new ResourceLocator(CDN_URL_AND_RESOURCE_TEMPLATES_ENVIRONMENT);
+                var urls = resourceLocator.getTemplateUrls(USERNAME, TILE_RESOURCE);
+                assert.ok(urls);
+
+                assert.deepEqual(urls.http, {
+                    urlTemplate: `http://{s}.cdn.carto.com/u/${USERNAME}/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: HTTP_SUBDOMAINS
+                });
+                assert.deepEqual(urls.https, {
+                    urlTemplate: `https://cdn_{s}.ssl.cdn.carto.com/u/${USERNAME}/api/v1/map/${TILE_RESOURCE}`,
+                    subdomains: HTTPS_SUBDOMAINS
+                });
+            });
+        });
 
-        assert.equal(
-            urls.http,
-            ['http://' + httpSubdomain + '.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
-        );
-        assert.equal(
-            urls.https,
-            ['https://cdn_' + httpsSubdomain + '.ssl.cdn.carto.com', 'u', USERNAME, 'api/v1/map', RESOURCE].join('/')
-        );
     });
 
 });

test/unit/cartodb/ported/tile_stats.test.js

@@ -11,7 +11,7 @@ describe('tile stats', function() {
     });
 
     afterEach(function() {
-        global.statsClient  = this.statsClient;
+        global.statsClient = this.statsClient;
     });
 
     it('finalizeGetTileOrGrid does not call statsClient when format is not supported', function() {
@@ -84,7 +84,7 @@ describe('tile stats', function() {
     });
 
     function mockStatsClientGetInstance(instance) {
-        global.statsClient = instance;
+        global.statsClient = Object.assign(global.statsClient, instance);
     }
 
 });

yarn.lock

@@ -2,12 +2,33 @@
 # yarn lockfile v1
 
 
-abaculus@cartodb/abaculus#2.0.3-cdb1:
-  version "2.0.3-cdb1"
-  resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf"
+"@carto/mapnik@3.6.2-carto.2", "@carto/mapnik@~3.6.2-carto.0":
+  version "3.6.2-carto.2"
+  resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.2.tgz#45a055fd2d39530a873ef9ce5a325baacc81c196"
   dependencies:
+    mapnik-vector-tile "1.5.0"
+    nan "~2.7.0"
+    node-pre-gyp "~0.6.30"
+    protozero "1.5.1"
+
+"@carto/tilelive-bridge@github:cartodb/tilelive-bridge#2.5.1-cdb1":
+  version "2.5.1-cdb1"
+  resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/b0b5559f948e77b337bc9a9ae0bf6ec4249fba21"
+  dependencies:
+    "@carto/mapnik" "~3.6.2-carto.0"
+    "@mapbox/sphericalmercator" "~1.0.1"
+    mapnik-pool "~0.1.3"
+
+"@mapbox/sphericalmercator@~1.0.1":
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2"
+
+"abaculus@github:cartodb/abaculus#2.0.3-cdb2":
+  version "2.0.3-cdb2"
+  resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc"
+  dependencies:
+    "@carto/mapnik" "~3.6.2-carto.0"
     d3-queue "^2.0.2"
-    mapnik "~3.5.0"
     sphericalmercator "1.0.x"
 
 abbrev@1:
@@ -33,8 +54,8 @@ ajv@^4.9.1:
     json-stable-stringify "^1.0.1"
 
 ajv@^5.1.0:
-  version "5.5.1"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.5.1.tgz#b38bb8876d9e86bee994956a04e721e88b248eb2"
+  version "5.5.2"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.5.2.tgz#73b5eeca3fab653e3d3f9422b341ad42205dc965"
   dependencies:
     co "^4.6.0"
     fast-deep-equal "^1.0.0"
@@ -95,8 +116,8 @@ assert-plus@^0.2.0:
   resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234"
 
 assertion-error@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.0.2.tgz#13ca515d86206da0bac66e834dd397d87581094c"
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b"
 
 async@1.x, async@^1.4.0, async@^1.5.2:
   version "1.5.2"
@@ -226,7 +247,7 @@ camshaft@0.60.0:
     dot "^1.0.3"
     request "^2.69.0"
 
-canvas@cartodb/node-canvas#1.6.2-cdb2:
+"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
   version "1.6.2-cdb2"
   resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
   dependencies:
@@ -244,15 +265,15 @@ carto@0.16.3:
     semver "^5.1.0"
     yargs "^4.2.0"
 
-carto@CartoDB/carto#0.15.1-cdb1:
+"carto@github:cartodb/carto#0.15.1-cdb1":
   version "0.15.1-cdb1"
-  resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
+  resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
   dependencies:
     mapnik-reference "~6.0.2"
     optimist "~0.6.0"
     underscore "~1.6.0"
 
-carto@cartodb/carto#0.15.1-cdb3:
+"carto@github:cartodb/carto#0.15.1-cdb3":
   version "0.15.1-cdb3"
   resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
   dependencies:
@@ -480,10 +501,14 @@ delegates@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
 
-depd@1.1.1, depd@~1.1.1:
+depd@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359"
 
+depd@~1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
+
 destroy@~1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
@@ -549,8 +574,8 @@ ee-first@1.1.1:
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
 
 encodeurl@~1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.1.tgz#79e3d58655346909fe6f0f45a5de68103b294d20"
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
 
 entities@1.0:
   version "1.0.0"
@@ -823,7 +848,7 @@ glob@^6.0.1:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7.0.5, glob@^7.1.1:
+glob@^7.0.5, glob@^7.1.1, glob@^7.1.2:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
   dependencies:
@@ -842,9 +867,9 @@ graceful-fs@^4.1.2:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
 
-grainstore@1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.7.0.tgz#28d78895c82e6201f7d0ff63af1056f3c0fda0d3"
+grainstore@1.8.2:
+  version "1.8.2"
+  resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.8.2.tgz#79dd7a91a098bf8b0ea3189961775c8cc7474319"
   dependencies:
     carto "0.16.3"
     debug "~3.1.0"
@@ -1073,8 +1098,8 @@ istanbul@~0.4.3:
     wordwrap "^1.0.0"
 
 js-base64@^2.1.9:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.4.0.tgz#9e566fee624751a1d720c966cd6226d29d4025aa"
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.4.3.tgz#2e545ec2b0f2957f41356510205214e98fad6582"
 
 js-string-escape@1.0.1:
   version "1.0.1"
@@ -1273,18 +1298,9 @@ mapnik-reference@~8.5.3:
   dependencies:
     semver "^5.1.0"
 
-mapnik-vector-tile@~1.2.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/mapnik-vector-tile/-/mapnik-vector-tile-1.2.2.tgz#42795ca211dd274a9a4af5bf6cfe3b0bfe0ba243"
-
-mapnik@3.5.14, mapnik@~3.5.0:
-  version "3.5.14"
-  resolved "https://registry.yarnpkg.com/mapnik/-/mapnik-3.5.14.tgz#632bd6635c72c0214a707549309ba416594afff7"
-  dependencies:
-    mapnik-vector-tile "~1.2.2"
-    nan "~2.4.0"
-    node-pre-gyp "~0.6.30"
-    protozero "~1.4.2"
+mapnik-vector-tile@1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/mapnik-vector-tile/-/mapnik-vector-tile-1.5.0.tgz#c647bfb8027e9dc40db583505a436f35e2101407"
 
 media-typer@0.3.0:
   version "0.3.0"
@@ -1380,8 +1396,8 @@ mocha@~3.4.1:
     supports-color "3.1.2"
 
 moment@^2.10.6:
-  version "2.19.4"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.4.tgz#17e5e2c6ead8819c8ecfad83a0acccb312e94682"
+  version "2.20.1"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.20.1.tgz#d6eb1a46cbcc14a2b2f9434112c1ff8907f313fd"
 
 moment@~2.18.1:
   version "2.18.1"
@@ -1633,7 +1649,7 @@ pg-types@1.*:
     postgres-date "~1.0.0"
     postgres-interval "^1.1.0"
 
-pg@cartodb/node-postgres#6.1.6-cdb1:
+"pg@github:cartodb/node-postgres#6.1.6-cdb1":
   version "6.1.6"
   resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44"
   dependencies:
@@ -1737,9 +1753,9 @@ propagate@0.3.x:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/propagate/-/propagate-0.3.1.tgz#e3a84404a7ece820dd6bbea9f6d924e3135ae09c"
 
-protozero@~1.4.2:
-  version "1.4.5"
-  resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.4.5.tgz#80eaa80a4f9c751465c4cb2620d8233b50ec1aff"
+protozero@1.5.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.5.1.tgz#5a27df6fb6e1ed743f510812ae76c082f5b16638"
 
 proxy-addr@~2.0.2:
   version "2.0.2"
@@ -1778,8 +1794,8 @@ raw-body@2.3.2:
     unpipe "1.0.0"
 
 rc@^1.1.7:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.2.tgz#d8ce9cb57e8d64d9c7badd9876c7c34cbe3c7077"
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd"
   dependencies:
     deep-extend "~0.4.0"
     ini "~1.3.0"
@@ -1950,8 +1966,8 @@ safe-json-stringify@~1:
   resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911"
 
 "semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0:
-  version "5.4.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e"
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab"
 
 semver@4.3.2:
   version "4.3.2"
@@ -2223,20 +2239,12 @@ through@2:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
 
-tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4:
-  version "2.3.1-cdb4"
-  resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6"
-  dependencies:
-    mapnik "~3.5.0"
-    mapnik-pool "~0.1.3"
-    sphericalmercator "1.0.x"
-
-tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3:
-  version "0.6.18-cdb3"
-  resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01"
+"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb4":
+  version "0.6.18-cdb4"
+  resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/510cfb6f033f7551f973886751643202d4cb5f4a"
   dependencies:
+    "@carto/mapnik" "~3.6.2-carto.0"
     generic-pool "~2.4.0"
-    mapnik "3.5.14"
     mime "~1.3.4"
     sphericalmercator "~1.0.4"
     step "~0.0.5"
@@ -2346,8 +2354,8 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
 
 uuid@^3.0.0, uuid@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14"
 
 validate-npm-package-license@^3.0.1:
   version "3.0.1"
@@ -2392,18 +2400,19 @@ window-size@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
 
-windshaft@4.1.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.1.0.tgz#dc17c8369570c305171d1ab5ca130369bba04d58"
+windshaft@4.3.3:
+  version "4.3.3"
+  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.3.3.tgz#a48fdd6ca05257c103f34c80195722ef52dc0813"
   dependencies:
-    abaculus cartodb/abaculus#2.0.3-cdb1
+    "@carto/mapnik" "3.6.2-carto.2"
+    "@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb1
+    abaculus cartodb/abaculus#2.0.3-cdb2
     canvas cartodb/node-canvas#1.6.2-cdb2
     carto cartodb/carto#0.15.1-cdb3
     cartodb-psql "^0.10.1"
     debug "^3.1.0"
     dot "~1.0.2"
-    grainstore "1.7.0"
-    mapnik "3.5.14"
+    grainstore "1.8.2"
     queue-async "~1.0.7"
     redis-mpool "0.4.1"
     request "^2.83.0"
@@ -2411,8 +2420,7 @@ windshaft@4.1.0:
     sphericalmercator "1.0.4"
     step "~0.0.6"
     tilelive "5.12.2"
-    tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb4
-    tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb3
+    tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb4
     torque.js "~2.11.0"
     underscore "~1.6.0"