Release 2.44.0

Upgrades camshaft to 0.11.0

.travis.yml

@@ -1,18 +1,13 @@
-dist: precise
-
 sudo: false
 
 addons:
   postgresql: "9.3"
   apt:
     packages:
-      - postgresql-9.3-postgis-2.1
-      - postgresql-plpython-9.3
       - pkg-config
       - libcairo2-dev
       - libjpeg8-dev
       - libgif-dev
-      - libpango1.0-dev
 
 before_install:
   - npm install -g npm@2

Makefile

@@ -43,7 +43,7 @@ jshint:
 	@echo "***jshint***"
 	@./node_modules/.bin/jshint lib/ test/ app.js
 
-test-all: jshint test
+test-all: test jshint
 
 coverage:
 	@RUNTESTFLAGS=--with-coverage make test

NEWS.md

@@ -1,12 +1,19 @@
 # Changelog
 
-## 2.43.2
+## 2.44.0
 
-Released 2017-09-01
+Released 2016-05-31
+
+Announcements:
+ - Upgrades camshaft to [0.11.0](https://github.com/CartoDB/camshaft/releases/tag/0.11.0)
+ - Upgrades turbo-carto to [0.10.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.10.0)
+
+New features:
+ - Adds support for sql wrap in all layers
 
 Bug fixes:
- - Adding a new cartodb-psql fixing a pg vulnerability
- 
+ - Fail on turbo-carto invalid quantification methods
+
 
 ## 2.43.1
 

docs/MapConfig-Analyses-extension.md

@@ -0,0 +1,93 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.4.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.4.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension targets layers with `sql` option, including layer types: `cartodb`, `mapnik`, and `torque`.
+
+It extends MapConfig with a new attribute: `analyses`.
+
+## 2.1 Analyses attribute
+
+The new analyses attribute must be an array of analyses as per [camshaft](https://github.com/CartoDB/camshaft). Each
+analysis must adhere to the [camshaft-reference](https://github.com/CartoDB/camshaft/blob/0.8.0/reference/versions/0.7.0/reference.json) specification.
+
+Each node can have an id that can be later references to consume the query from MapConfig's layers.
+
+Basic analyses example:
+
+```javascript
+[
+    {
+        // REQUIRED
+        // string, `id` free identifier that can be reference from any layer
+        "id": "HEAD",
+        // REQUIRED
+        // string, `type` camshaft's analysis type
+        "type": "source",
+        // REQUIRED
+        // object, `params` will depend on `type`, check camshaft-reference for more information
+        "params": {
+            "query": "select * from your_table"
+        }
+    }
+]
+```
+
+# 2.2. Integration with layers
+
+As pointed before an analysis node id can be referenced from layers to consume its output query.
+
+The layer consuming the output must reference it with the following option:
+
+```
+{
+    "options": {
+        // REQUIRED
+        // object, `source` as in the future we might want to have other source options
+        "source": {
+            // REQUIRED
+            // string, `id` the analysis node identifier
+            "id": "HEAD"
+        }
+    }
+}
+```
+
+## 2.3. Complete example
+
+```
+{
+    "version": "1.4.0",
+    "layers": [
+        {
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": "...",
+                "cartocss_version": "2.3.0"
+            }
+        }
+    ],
+    "analyses": [
+        {
+            "id": "HEAD",
+            "type": "source",
+            "params": {
+                "query": "select * from your_table"
+            }
+        }
+    ]
+}
+```
+
+# History
+
+## 1.0.0
+
+ - Initial version

docs/MapConfig-Dataviews-extension.md

@@ -0,0 +1,314 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.4.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.4.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`.
+
+It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms.
+
+## 2.1. Dataview types
+
+### List
+
+A list is a simple result set per row where is possible to retrieve several columns from the original layer query.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the list type
+    “type”: “list”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // array, `columns` to select for the list
+        “columns”: [“name”, “description”]
+    }
+}
+```
+
+Expected output
+```
+{
+  "type": "list",
+  "rows": [
+    {
+      "{columnName1}": "val1",
+      "{columnName2}": 100
+    },
+    {
+      "{columnName1}": "val2",
+      "{columnName2}": 200
+    }
+  ]
+}
+```
+
+### Aggregation
+
+An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the aggregation type
+    “type”: “aggregation”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // string, `column` column name to aggregate by
+        “column”: “country”,
+        // REQUIRED
+        // string, `aggregation` operation to perform
+        “aggregation”: “count”
+        // OPTIONAL
+        // string, `aggregationColumn` column value to aggregate
+        // This param is required when `aggregation` is different than "count"
+        “aggregationColumn”: “population”
+    }
+}
+```
+
+Expected output
+```
+{
+  "type": "aggregation",
+  "categories": [
+    {
+      "category": "foo",
+      "value": 100
+    },
+    {
+      "category": "bar",
+      "value": 200
+    }
+  ]
+}
+```
+
+### Histograms
+
+Histograms represent the data distribution for a column.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the histogram type
+    “type”: “histogram”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // string, `column` column name to aggregate by
+        “column”: “name”,
+        // OPTIONAL
+        // number, `bins` how many buckets the histogram should use
+        “bins”: 10
+    }
+}
+```
+
+Expected output
+```
+{
+  "type": "histogram",
+  "bins": [{"bin": 0, "start": 2, "end": 2, "min": 2, "max": 2, "freq": 1}, null, null, {"bin": 3, "min": 40, "max": 44, "freq": 2}, null],
+  "width": 10
+}
+```
+
+### Formula
+
+Formulas given a final value representing the whole dataset.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the formula type
+    “type”: “formula”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // string, `column` column name to aggregate by
+        “column”: “name”,
+        // REQUIRED
+        // string, `aggregation` operation to perform
+        “operation”: “count”
+    }
+}
+```
+
+Operation must be: “min”, “max”, “count”, “avg”, or “sum”.
+
+Result
+```
+{
+  "type": "formula",
+  "operation": "count",
+  "result": 1000,
+  "nulls": 0
+}
+```
+
+
+## 2.2 Dataviews attribute
+
+The new dataviews attribute must be a dictionary of dataviews.
+
+An analysis node id can be referenced from dataviews to consume its output query.
+
+
+The layer consuming the output must reference it with the following option:
+
+```
+{
+    // REQUIRED
+    // object, `source` as in the future we might want to have other source options
+    "source": {
+        // REQUIRED
+        // string, `id` the analysis node identifier
+        "id": "HEAD"
+    }
+}
+```
+
+## 2.3. Complete example
+
+```
+{
+    "version": "1.4.0",
+    "layers": [
+        {
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": "...",
+                "cartocss_version": "2.3.0"
+            }
+        }
+    ],
+    "dataviews" {
+        "basic_histogram": {
+            "source": {
+                "id": "HEAD"
+            },
+            "type": "histogram",
+            "options": {
+                "column": "pop_max"
+            }
+        }
+    },
+    "analyses": [
+        {
+            "id": "HEAD",
+            "type": "source",
+            "params": {
+                "query": "select * from your_table"
+            }
+        }
+    ]
+}
+```
+
+## 3. Filters
+
+Camshaft's analyses expose a filtering capability and `aggregation` and `histogram` dataviews get them for free with
+ this extension. Filters are available with the very dataview id, so if you have a "basic_histogram" histogram dataview
+ you can filter with a range filter with "basic_histogram" name.
+
+
+## 3.1 Filter types
+
+### Category
+
+Allows to remove results that are not contained within a set of elements.
+Initially this filter can be applied to a `numeric` or `text` columns.
+
+Params
+
+```
+{
+    “accept”: [“Spain”, “Germany”]
+    “reject”: [“Japan”]
+}
+```
+
+### Range filter
+
+Allows to remove results that don’t satisfy numeric min and max values.
+Filter is applied to a numeric column.
+
+Params
+
+```
+{
+    “min”: 0,
+    “max”: 1000
+}
+```
+
+## 3.2. How to apply filters
+
+Filters must be applied at map instantiation time.
+
+With :mapconfig as a valid MapConfig and with :filters (a valid JSON) as:
+
+### Anonymous map
+
+`GET /api/v1/map?config=:mapconfig&filters=:filters`
+
+`POST /api/v1/map?filters=:filters`
+with `BODY=:mapconfig`
+
+If in the future we need to support a bigger filters param and it doesn’t fit in the query string,
+ we might solve it by accepting:
+
+`POST /api/v1/map`
+with `BODY={“config”: :mapconfig, “filters”: :filters}`
+
+### Named map
+
+Assume :params (a valid JSON) as named maps params, like in: `{“color”: “red”}`
+
+`GET /api/v1/named/:name/jsonp?config=:params&filters=:filters&callback=cb`
+
+`POST /api/v1/named/:name?filters=:filters`
+with `BODY=:params`
+
+If, again, in the future we need to support a bigger filters param that doesn’t fit in the query string,
+ we might solve it by accepting:
+
+`POST /api/v1/named/:name`
+with `BODY={“config”: :params, “filters”: :filters}`
+
+
+## 3.3 Bounding box special filter
+
+A bounding box filter allows to remove results that don’t satisfy a geospatial range.
+
+The bounding box special filter is available per dataview and there is no need to create a bounding box definition as
+it’s always possible to apply a bbox filter per dataview.
+
+A dataview can get its result filtered by bounding box by sending a bbox param in the query string,
+param must be in the form `west,south,east,north`.
+
+So applying a bbox filter to a dataview looks like:
+GET /api/v1/map/:layergroupid/dataview/:dataview_name?bbox=-90,-45,90,45
+
+# History
+
+## 1.0.0-alpha
+
+ - WIP document

lib/cartodb/backends/dataview.js

@@ -123,7 +123,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             var queryRewriteData = layer && layer.options.query_rewrite_data;
             if ( queryRewriteData ) {
                 if ( node.type === 'source' ) {
-                    var filters = node.filters; // TODO: node.getFilters() when available in camshaft
+                    var filters = node.getFilters();
                     var filters_disabler = Object.keys(filters).reduce(
                         function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
                         {}

lib/cartodb/cache/named_map_provider_cache.js

@@ -1,24 +1,17 @@
 var _ = require('underscore');
 var dot = require('dot');
-var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
-var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
-var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
+var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
 var templateName = require('../backends/template_maps').templateName;
 var queue = require('queue-async');
 
 var LruCache = require("lru-cache");
 
-function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, analysisBackend, userLimitsApi,
-                               overviewsAdapter, turboCartoAdapter) {
+function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.metadataBackend = metadataBackend;
     this.userLimitsApi = userLimitsApi;
-
-    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
-    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
-    this.overviewsAdapter = overviewsAdapter;
-    this.turboCartoAdapter = turboCartoAdapter;
+    this.mapConfigAdapter = mapConfigAdapter;
 
     this.providerCache = new LruCache({ max: 2000 });
 }
@@ -36,10 +29,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
             this.pgConnection,
             this.metadataBackend,
             this.userLimitsApi,
-            this.namedLayersAdapter,
-            this.overviewsAdapter,
-            this.turboCartoAdapter,
-            this.analysisMapConfigAdapter,
+            this.mapConfigAdapter,
             user,
             templateId,
             config,

lib/cartodb/controllers/layergroup.js

@@ -10,7 +10,7 @@ var userMiddleware = require('../middleware/user');
 var DataviewBackend = require('../backends/dataview');
 var AnalysisStatusBackend = require('../backends/analysis-status');
 
-var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
+var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
 
 var QueryTables = require('cartodb-query-tables');
 

lib/cartodb/controllers/map.js

@@ -15,10 +15,8 @@ var Datasource = windshaft.model.Datasource;
 
 var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
 
-var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
-var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
-var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
-var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
+var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
+var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
 
 /**
  * @param {AuthApi} authApi
@@ -29,14 +27,11 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
- * @param {MapConfigOverviewsAdapter} overviewsAdapter
- * @param {TurboCartoAdapter} turboCartoAdapter
- * @param {AnalysisBackend} analysisBackend
+ * @param {MapConfigAdapter} mapConfigAdapter
  * @constructor
  */
 function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables,
-                       overviewsAdapter, turboCartoAdapter, analysisBackend) {
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
 
     BaseController.call(this, authApi, pgConnection);
 
@@ -47,11 +42,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
-    this.turboCartoAdapter = turboCartoAdapter;
 
-    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
-    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
-    this.overviewsAdapter = overviewsAdapter;
+    this.mapConfigAdapter = mapConfigAdapter;
 }
 
 util.inherits(MapController, BaseController);
@@ -132,16 +124,17 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
     var self = this;
 
     var mapConfig;
-    var analysesResults = [];
+
+    var context = {};
 
     step(
         function setupParams(){
             self.req2params(req, this);
         },
         prepareConfigFn,
-        function prepareAnalysisLayers(err, requestMapConfig) {
+        function prepareAdapterMapConfig(err, requestMapConfig) {
             assert.ifError(err);
-            var analysisConfiguration = {
+            context.analysisConfiguration = {
                 db: {
                     host: req.params.dbhost,
                     port: req.params.dbport,
@@ -154,71 +147,12 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
                     apiKey: req.params.api_key
                 }
             };
-
-            var filters = {};
-            if (req.params.filters) {
-                try {
-                    filters = JSON.parse(req.params.filters);
-                } catch (e) {
-                    // ignore
-                }
-            }
-            self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this);
+            self.mapConfigAdapter.getMapConfig(req.context.user, requestMapConfig, req.params, context, this);
         },
-        function beforeLayergroupCreate(err, requestMapConfig, _analysesResults) {
+        function createLayergroup(err, requestMapConfig) {
             assert.ifError(err);
-            var next = this;
-            analysesResults = _analysesResults;
-            self.namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, self.pgConnection,
-                function(err, layers, datasource) {
-                    if (err) {
-                        return next(err);
-                    }
-
-                    if (layers) {
-                        requestMapConfig.layers = layers;
-                    }
-                    return next(null, requestMapConfig, datasource);
-                }
-            );
-        },
-        function addOverviewsInformation(err, requestMapConfig, datasource) {
-            assert.ifError(err);
-            var next = this;
-            self.overviewsAdapter.getLayers(
-                req.context.user, requestMapConfig.layers, analysesResults,
-                function(err, layers) {
-                    if (err) {
-                        return next(err);
-                    }
-
-                    if (layers) {
-                        requestMapConfig.layers = layers;
-                    }
-
-                    return next(null, requestMapConfig, datasource);
-                }
-            );
-        },
-        function parseTurboCarto(err, requestMapConfig, datasource) {
-            assert.ifError(err);
-
-            var next = this;
-            self.turboCartoAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) {
-                if (err) {
-                    return next(err);
-                }
-
-                if (layers) {
-                    requestMapConfig.layers = layers;
-                }
-
-                return next(null, requestMapConfig, datasource);
-            });
-        },
-        function createLayergroup(err, requestMapConfig, datasource) {
-            assert.ifError(err);
-            mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource());
+            var datasource = context.datasource || Datasource.EmptyDatasource();
+            mapConfig = new MapConfig(requestMapConfig, datasource);
             self.mapBackend.createLayergroup(
                 mapConfig, req.params,
                 new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params),
@@ -227,6 +161,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function afterLayergroupCreate(err, layergroup) {
             assert.ifError(err);
+            var analysesResults = context.analysesResults || [];
             self.afterLayergroupCreate(req, res, mapConfig, analysesResults, layergroup, this);
         },
         function finish(err, layergroup) {
@@ -264,10 +199,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 self.pgConnection,
                 self.metadataBackend,
                 self.userLimitsApi,
-                self.namedLayersAdapter,
-                self.overviewsAdapter,
-                self.turboCartoAdapter,
-                self.analysisMapConfigAdapter,
+                self.mapConfigAdapter,
                 cdbuser,
                 req.params.template_id,
                 templateParams,

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -11,73 +11,25 @@ function AnalysisMapConfigAdapter(analysisBackend) {
 
 module.exports = AnalysisMapConfigAdapter;
 
-var SKIP_COLUMNS = {
-    'the_geom': true,
-    'the_geom_webmercator': true
-};
-
-function skipColumns(columnNames) {
-    return columnNames
-        .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
-}
-
-var layerQueryTemplate = dot.template([
-    'SELECT {{=it._columns}}',
-    'FROM ({{=it._query}}) _cdb_analysis_query'
-].join('\n'));
-
-function layerQuery(node) {
-    if (node.type === 'source') {
-        return node.getQuery();
-    }
-    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
-    return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') });
-}
-
-function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
-    var analyses = requestMapConfig.analyses || [];
-
-    requestMapConfig.analyses = analyses.map(function(analysisDefinition) {
-        var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);
-        var definition = analysisDefinition;
-        Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) {
-            definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] });
-        });
-
-        return definition;
-    });
-
-    return requestMapConfig;
-}
-
-function shouldAdaptLayers(requestMapConfig) {
-    return Array.isArray(requestMapConfig.layers) &&
-        Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0;
-}
-
-var DATAVIEW_TYPE_2_FILTER_TYPE = {
-    aggregation: 'category',
-    histogram: 'range'
-};
-function getFilter(dataview, params) {
-    var type = dataview.type;
-
-    return {
-        type: DATAVIEW_TYPE_2_FILTER_TYPE[type],
-        column: dataview.options.column,
-        params: params
-    };
-}
-
-AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration, requestMapConfig, filters, callback) {
+AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
     // jshint maxcomplexity:7
     var self = this;
-    filters = filters || {};
 
     if (!shouldAdaptLayers(requestMapConfig)) {
         return callback(null, requestMapConfig);
     }
 
+    var analysisConfiguration = context.analysisConfiguration;
+
+    var filters = {};
+    if (params.filters) {
+        try {
+            filters = JSON.parse(params.filters);
+        } catch (e) {
+            // ignore
+        }
+    }
+
     var dataviewsFilters = filters.dataviews || {};
     debug(dataviewsFilters);
     var dataviews = requestMapConfig.dataviews || {};
@@ -145,6 +97,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration
                     var analysisSql = layerQuery(layerNode);
                     var sqlQueryWrap = layer.options.sql_wrap;
                     if (sqlQueryWrap) {
+                        layer.options.sql_raw = analysisSql;
                         analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql);
                     }
                     layer.options.sql = analysisSql;
@@ -167,10 +120,70 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration
             return callback(missingNodesErrors);
         }
 
-        return callback(null, requestMapConfig, analysesResults);
+        context.analysesResults = analysesResults;
+
+        return callback(null, requestMapConfig);
     });
 };
 
+var SKIP_COLUMNS = {
+    'the_geom': true,
+    'the_geom_webmercator': true
+};
+
+function skipColumns(columnNames) {
+    return columnNames
+        .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
+}
+
+var layerQueryTemplate = dot.template([
+    'SELECT {{=it._columns}}',
+    'FROM ({{=it._query}}) _cdb_analysis_query'
+].join('\n'));
+
+function layerQuery(node) {
+    if (node.type === 'source') {
+        return node.getQuery();
+    }
+    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
+    return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') });
+}
+
+function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
+    var analyses = requestMapConfig.analyses || [];
+
+    requestMapConfig.analyses = analyses.map(function(analysisDefinition) {
+        var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);
+        var definition = analysisDefinition;
+        Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) {
+            definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] });
+        });
+
+        return definition;
+    });
+
+    return requestMapConfig;
+}
+
+function shouldAdaptLayers(requestMapConfig) {
+    return Array.isArray(requestMapConfig.layers) &&
+        Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0;
+}
+
+var DATAVIEW_TYPE_2_FILTER_TYPE = {
+    aggregation: 'category',
+    histogram: 'range'
+};
+function getFilter(dataview, params) {
+    var type = dataview.type;
+
+    return {
+        type: DATAVIEW_TYPE_2_FILTER_TYPE[type],
+        column: dataview.options.column,
+        params: params
+    };
+}
+
 function getLayerSourceId(layer) {
     return layer.options.source && layer.options.source.id;
 }

lib/cartodb/models/mapconfig/adapter/index.js

@@ -0,0 +1,26 @@
+'use strict';
+
+function MapConfigAdapter(adapters) {
+    this.adapters = Array.isArray(adapters) ? adapters : Array.apply(null, arguments);
+}
+
+module.exports = MapConfigAdapter;
+
+MapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+    var self = this;
+    var i = 0;
+    var tasksLeft = this.adapters.length;
+
+    function next(err, _requestMapConfig) {
+        if (err) {
+            return callback(err);
+        }
+        if (tasksLeft-- === 0) {
+            return callback(null, _requestMapConfig);
+        }
+        var nextAdapter = self.adapters[i++];
+        nextAdapter.getMapConfig(user, _requestMapConfig, params, context, next);
+    }
+
+    next(null, requestMapConfig);
+};

lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js

@@ -2,17 +2,20 @@ var queue = require('queue-async');
 var _ = require('underscore');
 var Datasource = require('windshaft').model.Datasource;
 
-function MapConfigNamedLayersAdapter(templateMaps) {
+function MapConfigNamedLayersAdapter(templateMaps, pgConnection) {
     this.templateMaps = templateMaps;
+    this.pgConnection = pgConnection;
 }
 
 module.exports = MapConfigNamedLayersAdapter;
 
-MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbMetadata, callback) {
+MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
     var self = this;
 
+    var layers = requestMapConfig.layers;
+
     if (!layers) {
-        return callback(null);
+        return callback(null, requestMapConfig);
     }
 
     var adaptLayersQueue = queue(layers.length);
@@ -28,9 +31,9 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
             var templateConfigParams = layer.options.config || {};
             var templateAuthTokens = layer.options.auth_tokens;
 
-            self.templateMaps.getTemplate(username, templateName, function(err,  template) {
+            self.templateMaps.getTemplate(user, templateName, function(err,  template) {
                 if (err || !template) {
-                    return done(new Error("Template '" + templateName + "' of user '" + username + "' not found"));
+                    return done(new Error("Template '" + templateName + "' of user '" + user + "' not found"));
                 }
 
                 if (self.templateMaps.isAuthorized(template, templateAuthTokens)) {
@@ -96,7 +99,10 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
 
         });
 
-        return callback(null, layers, datasourceBuilder.build());
+        requestMapConfig.layers = layers;
+        context.datasource = datasourceBuilder.build();
+
+        return callback(null, requestMapConfig);
     }
 
 
@@ -104,7 +110,7 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
 
     if (_.some(layers, isNamedTypeLayer)) {
         // Lazy load dbAuth
-        dbMetadata.setDBAuth(username, dbAuth, function(err) {
+        this.pgConnection.setDBAuth(user, dbAuth, function(err) {
             if (err) {
                 return callback(err);
             }
@@ -114,7 +120,8 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
             adaptLayersQueue.awaitAll(layersAdaptQueueFinish);
         });
     } else {
-        return callback(null, layers, datasourceBuilder.build());
+        context.datasource = datasourceBuilder.build();
+        return callback(null, requestMapConfig);
     }
 
 };

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -9,11 +9,14 @@ function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
 
 module.exports = MapConfigOverviewsAdapter;
 
-MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) {
+MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
   var self = this;
 
+  var layers = requestMapConfig.layers;
+  var analysesResults = context.analysesResults;
+
   if (!layers || layers.length === 0) {
-      return callback(null, layers);
+      return callback(null, requestMapConfig);
   }
 
   var augmentLayersQueue = queue(layers.length);
@@ -22,7 +25,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
       if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
           return done(null, layer);
       }
-      self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){
+      self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){
            if (err) {
                done(err, layer);
            } else {
@@ -35,7 +38,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
                            var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
                            if ( node ) {
                                node = node.rootNode;
-                               filters = node.filters; // TODO: node.getFilters() when available in camshaft
+                               filters = node.getFilters();
                                var filters_disabler = Object.keys(filters).reduce(
                                    function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
                                    {}
@@ -51,7 +54,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
                        var next_step = this;
                        if ( filters ) {
                            self.filterStatsApi.getFilterStats(
-                               username,
+                               user,
                                unfiltered_query, filters,
                                function(err, stats) {
                                  if ( !err ) {
@@ -85,7 +88,9 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
         return callback(new Error('Missing layers array from layergroup config'));
     }
 
-    return callback(null, layers);
+    requestMapConfig.layers = layers;
+
+    return callback(null, requestMapConfig);
   }
 
   layers.forEach(function(layer) {

lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js

@@ -0,0 +1,25 @@
+function SqlWrapMapConfigAdapter() {
+}
+
+module.exports = SqlWrapMapConfigAdapter;
+
+
+SqlWrapMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+    if (requestMapConfig && Array.isArray(requestMapConfig.layers)) {
+        requestMapConfig.layers = requestMapConfig.layers.map(function(layer) {
+            if (layer.options) {
+                var sqlQueryWrap = layer.options.sql_wrap;
+                if (sqlQueryWrap) {
+                    var layerSql = layer.options.sql;
+                    if (layerSql) {
+                        layer.options.sql_raw = layerSql;
+                        layer.options.sql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, layerSql);
+                    }
+                }
+            }
+            return layer;
+        });
+    }
+
+    return callback(null, requestMapConfig);
+};

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -0,0 +1,102 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+var queue = require('queue-async');
+var SubstitutionTokens = require('../../../utils/substitution-tokens');
+
+function TurboCartoAdapter(turboCartoParser) {
+    this.turboCartoParser = turboCartoParser;
+}
+
+module.exports = TurboCartoAdapter;
+
+TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
+    var self = this;
+
+    var layers = requestMapConfig.layers;
+
+    if (!layers || layers.length === 0) {
+        return callback(null, requestMapConfig);
+    }
+
+    var parseCartoQueue = queue(layers.length);
+
+    layers.forEach(function(layer) {
+        parseCartoQueue.defer(self._parseCartoCss.bind(self), user, layer);
+    });
+
+    parseCartoQueue.awaitAll(function (err, layers) {
+        if (err) {
+            return callback(err);
+        }
+
+        requestMapConfig.layers = layers;
+
+        return callback(null, requestMapConfig);
+    });
+};
+
+var bboxTemplate = dot.template('(select ST_SetSRID(st_extent(the_geom_webmercator), 3857) from ({{=it._sql}}) __c)');
+var zoomTemplate = dot.template([
+    'GREATEST(',
+    'ceil(log(40075017000 / 256 / GREATEST(',
+    '  st_xmax({{=it._bbox}}) - st_xmin({{=it._bbox}}),',
+    '  st_ymax({{=it._bbox}}) - st_ymin({{=it._bbox}})',
+    '))/log(2)),',
+    '0',
+    ')'
+].join('\n'));
+var pixelSizeTemplate = dot.template('40075017 * cos(ST_Y(ST_Centroid({{=it._bbox}}))) / 2 ^ (({{=it._zoom}}) + 8)');
+var scaleDenominatorTemplate = dot.template('({{=it._pixelSize}} / 0.00028)::numeric');
+
+
+TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) {
+    if (!shouldParseLayerCartocss(layer)) {
+        return callback(null, layer);
+    }
+
+    var tokens = {
+        bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)',
+        scale_denominator: '500000001',
+        pixel_width: '156412',
+        pixel_height: '156412'
+    };
+
+    var layerSql = layer.options.sql;
+    var layerRawSql = layer.options.sql_raw;
+    if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) {
+        var bbox = bboxTemplate({ _sql: layerRawSql });
+        var zoom = zoomTemplate({ _bbox: bbox });
+        var pixelSize = pixelSizeTemplate({ _bbox: bbox, _zoom: zoom });
+        var scaleDenominator = scaleDenominatorTemplate({ _pixelSize: pixelSize });
+
+        tokens = {
+            bbox: bbox,
+            scale_denominator: scaleDenominator,
+            pixel_width: pixelSize,
+            pixel_height: pixelSize
+        };
+    }
+
+    var sql = SubstitutionTokens.replace(layerSql, tokens);
+
+    this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) {
+        // Only return turbo-carto errors
+        if (err && err.name === 'TurboCartoError') {
+            err = new Error('turbo-carto: ' + err.message);
+            err.http_status = 400;
+            return callback(err);
+        }
+
+        // Try to continue in the rest of the cases
+        if (cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+        return callback(null, layer);
+    });
+};
+
+function shouldParseLayerCartocss(layer) {
+    return layer && layer.options && layer.options.cartocss && layer.options.sql;
+}

lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js

@@ -1,7 +1,7 @@
 var assert = require('assert');
 var step = require('step');
 
-var MapStoreMapConfigProvider = require('./map_store_provider');
+var MapStoreMapConfigProvider = require('./map-store-provider');
 
 /**
  * @param {MapConfig} mapConfig

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -4,24 +4,20 @@ var crypto = require('crypto');
 var dot = require('dot');
 var step = require('step');
 var MapConfig = require('windshaft').model.MapConfig;
-var templateName = require('../../backends/template_maps').templateName;
+var templateName = require('../../../backends/template_maps').templateName;
 var QueryTables = require('cartodb-query-tables');
 
 /**
  * @constructor
  * @type {NamedMapMapConfigProvider}
  */
-function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi,
-                                   namedLayersAdapter, overviewsAdapter, turboCartoAdapter, analysisMapConfigAdapter,
+function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter,
                                    owner, templateId, config, authToken, params) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.metadataBackend = metadataBackend;
     this.userLimitsApi = userLimitsApi;
-    this.namedLayersAdapter = namedLayersAdapter;
-    this.turboCartoAdapter = turboCartoAdapter;
-    this.analysisMapConfigAdapter = analysisMapConfigAdapter;
-    this.overviewsAdapter = overviewsAdapter;
+    this.mapConfigAdapter = mapConfigAdapter;
 
     this.owner = owner;
     this.templateName = templateName(templateId);
@@ -54,10 +50,11 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
     var self = this;
 
     var mapConfig = null;
-    var datasource = null;
     var rendererParams;
     var apiKey;
 
+    var context = {};
+
     step(
         function getTemplate() {
             self.getTemplate(this);
@@ -95,9 +92,9 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
             assert.ifError(err);
             return self.templateMaps.instance(self.template, templateParams);
         },
-        function prepareAnalysisLayers(err, requestMapConfig) {
+        function prepareAdapterMapConfig(err, requestMapConfig) {
             assert.ifError(err);
-            var analysisConfiguration = {
+            context.analysisConfiguration = {
                 db: {
                     host: rendererParams.dbhost,
                     port: rendererParams.dbport,
@@ -110,75 +107,17 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
                     apiKey: apiKey
                 }
             };
-
-            var filters = {};
-            if (self.params.filters) {
-                try {
-                    filters = JSON.parse(self.params.filters);
-                } catch (e) {
-                    // ignore
-                }
-            }
-            self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this);
+            self.mapConfigAdapter.getMapConfig(self.owner, requestMapConfig, rendererParams, context, this);
         },
-        function prepareLayergroup(err, _mapConfig, analysesResults) {
-            assert.ifError(err);
-            var next = this;
-            self.analysesResults = analysesResults || [];
-            self.namedLayersAdapter.getLayers(self.owner, _mapConfig.layers, self.pgConnection,
-                function(err, layers, datasource) {
-                    if (err) {
-                        return next(err);
-                    }
-
-                    if (layers) {
-                        _mapConfig.layers = layers;
-                    }
-                    return next(null, _mapConfig, datasource);
-                }
-            );
-        },
-        function addOverviewsInformation(err, _mapConfig, datasource) {
-            assert.ifError(err);
-            var next = this;
-
-            self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) {
-                if (err) {
-                    return next(err);
-                }
-
-                if (layers) {
-                    _mapConfig.layers = layers;
-                }
-
-                return next(null, _mapConfig, datasource);
-            });
-        },
-        function parseTurboCarto(err, _mapConfig, datasource) {
-            assert.ifError(err);
-            var next = this;
-
-            self.turboCartoAdapter.getLayers(self.owner, _mapConfig.layers, function (err, layers) {
-                if (err) {
-                    return next(err);
-                }
-
-                if (layers) {
-                    _mapConfig.layers = layers;
-                }
-
-                return next(null, _mapConfig, datasource);
-            });
-        },
-        function prepareContextLimits(err, _mapConfig, _datasource) {
+        function prepareContextLimits(err, _mapConfig) {
             assert.ifError(err);
             mapConfig = _mapConfig;
-            datasource = _datasource;
             self.userLimitsApi.getRenderLimits(self.owner, this);
         },
         function cacheAndReturnMapConfig(err, renderLimits) {
             self.err = err;
-            self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, datasource);
+            self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
+            self.analysesResults = context.analysesResults || [];
             self.rendererParams = rendererParams;
             self.context.limits = renderLimits || {};
             return callback(self.err, self.mapConfig, self.rendererParams, self.context);

lib/cartodb/server.js

@@ -34,10 +34,14 @@ var AnalysisBackend = require('./backends/analysis');
 var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
-var MapConfigOverviewsAdapter = require('./models/mapconfig_overviews_adapter');
-
 var TurboCartoParser = require('./utils/style/turbo-carto-parser');
-var TurboCartoAdapter = require('./utils/style/turbo-carto-adapter');
+
+var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
+var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
+var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
+var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
+var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
+var MapConfigAdapter = require('./models/mapconfig/adapter');
 
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
@@ -150,19 +154,22 @@ module.exports = function(serverOptions) {
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
-
     var turboCartoParser = new TurboCartoParser(pgQueryRunner);
-    var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);
+
+    var mapConfigAdapter = new MapConfigAdapter(
+        new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
+        new SqlWrapMapConfigAdapter(),
+        new AnalysisMapConfigAdapter(analysisBackend),
+        new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
+        new TurboCartoAdapter(turboCartoParser)
+    );
 
     var namedMapProviderCache = new NamedMapProviderCache(
         templateMaps,
         pgConnection,
         metadataBackend,
-        analysisBackend,
         userLimitsApi,
-        overviewsAdapter,
-        turboCartoAdapter
+        mapConfigAdapter
     );
 
     ['update', 'delete'].forEach(function(eventType) {
@@ -201,9 +208,7 @@ module.exports = function(serverOptions) {
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache,
-        overviewsAdapter,
-        turboCartoAdapter,
-        analysisBackend
+        mapConfigAdapter
     ).register(app);
 
     new controller.NamedMaps(

lib/cartodb/utils/style/postgres-datasource.js

@@ -23,8 +23,35 @@ var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, meth
     return methodTemplates;
 }, {});
 
+methodTemplates.category = dot.template([
+    'WITH',
+    'categories AS (',
+    '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
+    '  FROM ({{=it._sql}}) _cdb_aggregation_all',
+    '  GROUP BY {{=it._column}}',
+    '  ORDER BY 2 DESC',
+    '),',
+    'agg_categories AS (',
+    '  SELECT \'__other\' category',
+    '  FROM categories',
+    '  WHERE rank >= {{=it._buckets}}',
+    '  GROUP BY 1',
+    '  UNION ALL',
+    '  SELECT CAST(category AS text)',
+    '  FROM categories',
+    '  WHERE rank < {{=it._buckets}}',
+    ')',
+    'SELECT array_agg(category) AS category FROM agg_categories'
+].join('\n'));
+
+var STRATEGY = {
+    SPLIT: 'split',
+    EXACT: 'exact'
+};
+
 var method2strategy = {
-    headtails: 'split'
+    headtails: STRATEGY.SPLIT,
+    category: STRATEGY.EXACT
 };
 
 function PostgresDatasource (pgQueryRunner, username, query) {
@@ -38,7 +65,12 @@ PostgresDatasource.prototype.getName = function () {
 };
 
 PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) {
-    var methodName = methods.hasOwnProperty(method) ? method : 'quantiles';
+    if (method && !methodTemplates.hasOwnProperty(method)) {
+        return callback(new Error(
+            'Invalid method "' + method + '", valid methods: [' + Object.keys(methodTemplates).join(',') + ']'
+        ));
+    }
+    var methodName = method || 'quantiles';
     var template = methodTemplates[methodName];
 
     var query = template({ _column: column, _sql: this.query, _buckets: buckets });
@@ -48,11 +80,15 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             return callback(err);
         }
 
-        var ramp = result[0][methodName].sort(function(a, b) {
-            return a - b;
-        });
+        var strategy = method2strategy[methodName];
+        var ramp = result[0][methodName];
+        if (strategy !== STRATEGY.EXACT) {
+            ramp = ramp.sort(function(a, b) {
+                return a - b;
+            });
+        }
 
-        return callback(null, { ramp: ramp, strategy: method2strategy[methodName] });
+        return callback(null, { ramp: ramp, strategy: strategy });
     });
 };
 

lib/cartodb/utils/style/turbo-carto-adapter.js

@@ -1,70 +0,0 @@
-'use strict';
-
-var queue = require('queue-async');
-var SubstitutionTokens = require('../substitution-tokens');
-
-function TurboCartoAdapter(turboCartoParser) {
-    this.turboCartoParser = turboCartoParser;
-}
-
-module.exports = TurboCartoAdapter;
-
-TurboCartoAdapter.prototype.getLayers = function (username, layers, callback) {
-    var self = this;
-
-    if (!layers || layers.length === 0) {
-        return callback(null, layers);
-    }
-
-    var parseCartoQueue = queue(layers.length);
-
-    layers.forEach(function(layer) {
-        parseCartoQueue.defer(self._parseCartoCss.bind(self), username, layer);
-    });
-
-    parseCartoQueue.awaitAll(function (err, layers) {
-        if (err) {
-            return callback(err);
-        }
-
-        return callback(null, layers);
-    });
-};
-
-TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) {
-    if (isNotLayerToParseCartocss(layer)) {
-        return process.nextTick(function () {
-            callback(null, layer);
-        });
-    }
-
-    var sql = SubstitutionTokens.replace(layer.options.sql, {
-        bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)',
-        scale_denominator: '500000001',
-        pixel_width: '156412',
-        pixel_height: '156412'
-    });
-
-    this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) {
-        // Only return turbo-carto errors
-        if (err && err.name === 'TurboCartoError') {
-            err = new Error('turbo-carto: ' + err.message);
-            err.http_status = 400;
-            return callback(err);
-        }
-
-        // Try to continue in the rest of the cases
-        if (cartocss) {
-            layer.options.cartocss = cartocss;
-        }
-        return callback(null, layer);
-    });
-};
-
-function isNotLayerToParseCartocss(layer) {
-    if (!layer || !layer.options || !layer.options.cartocss || !layer.options.sql) {
-        return true;
-    }
-
-    return false;
-}

lib/cartodb/utils/substitution-tokens.js

@@ -6,6 +6,16 @@ var SUBSTITUTION_TOKENS = {
 };
 
 var SubstitutionTokens = {
+    tokens: function(sql) {
+        return Object.keys(SUBSTITUTION_TOKENS).filter(function(tokenName) {
+            return !!sql.match(SUBSTITUTION_TOKENS[tokenName]);
+        });
+    },
+
+    hasTokens: function(sql) {
+        return this.tokens(sql).length > 0;
+    },
+
     replace: function(sql, replaceValues) {
         Object.keys(replaceValues).forEach(function(token) {
             if (SUBSTITUTION_TOKENS[token]) {

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.43.2",
+  "version": "2.44.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,8 +20,8 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.8.0",
-    "cartodb-psql": "0.10.1",
+    "camshaft": "0.11.0",
+    "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "~0.13.0",
     "debug": "~2.2.0",
@@ -37,7 +37,7 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.9.2",
+    "turbo-carto": "0.10.0",
     "underscore": "~1.6.0",
     "windshaft": "1.19.0"
   },

test/acceptance/analysis/named-maps.js

@@ -7,7 +7,7 @@ var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 var TestClient = require('../../support/test-client');
 
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 describe('named-maps analysis', function() {
 

test/acceptance/dynamic-styling-named-maps.js

@@ -1,6 +1,6 @@
 var assert = require('../support/assert');
 var step = require('step');
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 var testHelper = require(__dirname + '/../support/test_helper');
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');

test/acceptance/limits.js

@@ -7,7 +7,7 @@ var redis = require('redis');
 var CartodbWindshaft = require('../../lib/cartodb/server');
 var serverOptions = require('../../lib/cartodb/server_options');
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 describe('render limits', function() {
 

test/acceptance/multilayer.js

@@ -6,7 +6,7 @@ var strftime    = require('strftime');
 var redis_stats_db = 5;
 
 var helper = require(__dirname + '/../support/test_helper');
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 

test/acceptance/multilayer_server.js

@@ -4,7 +4,7 @@ var assert = require('../support/assert');
 
 var _ = require('underscore');
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
 var QueryTables = require('cartodb-query-tables');

test/acceptance/named_layers.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');

test/acceptance/overviews_metadata.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 

test/acceptance/overviews_metadata_named_maps.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 

test/acceptance/ported/attributes.js

@@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var PortedServerOptions = require('./support/ported_server_options');
 var BaseController = require('../../../lib/cartodb/controllers/base');
 
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 
 describe('attributes', function() {

test/acceptance/ported/multilayer.js

@@ -7,7 +7,7 @@ var step = require('step');
 var mapnik = require('windshaft').mapnik;
 var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 var BaseController = require('../../../lib/cartodb/controllers/base');
 
 describe('multilayer', function() {

test/acceptance/ported/multilayer_interactivity.js

@@ -5,7 +5,7 @@ var _             = require('underscore');
 var cartodbServer = require('../../../lib/cartodb/server');
 var getLayerTypeFn = require('windshaft').model.MapConfig.prototype.getType;
 var PortedServerOptions = require('./support/ported_server_options');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
 

test/acceptance/ported/raster.js

@@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 describe('raster', function() {
 

test/acceptance/ported/retina.js

@@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 describe('retina support', function() {
 

test/acceptance/ported/server_png8_format.js

@@ -7,7 +7,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 var IMAGE_EQUALS_TOLERANCE_PER_MIL = 85;
 

test/acceptance/ported/support/ported_server_options.js

@@ -1,6 +1,6 @@
 var _ = require('underscore');
 var serverOptions = require('../../../../lib/cartodb/server_options');
-var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../../support/layergroup-token');
 var mapnik = require('windshaft').mapnik;
 var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter');
 var overviewsQueryRewriter = new OverviewsQueryRewriter({

test/acceptance/ported/support/test_client.js

@@ -1,5 +1,5 @@
 var testHelper = require('../../../support/test_helper');
-var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../../support/layergroup-token');
 
 var step = require('step');
 var assert = require('../../../support/assert');

test/acceptance/ported/torque.js

@@ -7,7 +7,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 describe('torque', function() {
 

test/acceptance/ported/torque_boundaries.js

@@ -5,7 +5,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 describe('torque boundary points', function() {
 

test/acceptance/sql-wrap.js

@@ -0,0 +1,52 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+describe('sql-wrap', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            return done();
+        }
+    });
+
+    it('should use sql_wrap from layer options', function(done) {
+        var mapConfig = {
+            version: '1.5.0',
+            layers: [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "sql": "SELECT * FROM populated_places_simple_reduced",
+                        "sql_wrap": "SELECT * FROM (<%= sql %>) _w WHERE adm0_a3 = 'USA'",
+                        "cartocss": [
+                            "#points {",
+                            "  marker-fill-opacity: 1;",
+                            "  marker-line-color: #FFF;",
+                            "  marker-line-width: 0.5;",
+                            "  marker-line-opacity: 1;",
+                            "  marker-placement: point;",
+                            "  marker-type: ellipse;",
+                            "  marker-width: 8;",
+                            "  marker-fill: red;",
+                            "  marker-allow-overlap: true;",
+                            "}"
+                        ].join('\n'),
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ]
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getTile(0, 0, 0, function(err, tile, img) {
+            assert.ok(!err, err);
+            var fixtureImg = './test/fixtures/sql-wrap-usa-filter.png';
+            assert.imageIsSimilarToFile(img, fixtureImg, 20, done);
+        });
+    });
+
+});

test/acceptance/templates.js

@@ -23,7 +23,7 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 describe('template_api', function() {
     server.layergroupAffectedTablesCache.cache.reset();

test/acceptance/turbo-cartocss/anonymous-maps.js

@@ -53,6 +53,20 @@ describe('turbo-carto for anonymous maps', function() {
             var fixturePath = 'test_turbo_carto_greens_13_4011_3088.png';
             this.testClient.getTile(13, 4011, 3088, imageCompareFn(fixturePath, done));
         });
+
+        it('should work for different char case in quantification names', function(done) {
+            this.testClient = new TestClient(
+                makeMapconfig('#layer { marker-fill: ramp([price], colorbrewer(Greens, 3), jeNkS); }')
+            );
+            this.testClient.getLayergroup(function(err, layergroup) {
+                assert.ok(!err, err);
+
+                assert.ok(layergroup.hasOwnProperty('layergroupid'));
+                assert.ok(!layergroup.hasOwnProperty('errors'));
+
+                done();
+            });
+        });
     });
 
     describe('parsing ramp function with colorbrewer for reds and mapnik renderer', function () {

test/acceptance/turbo-cartocss/error-cases.js

@@ -78,6 +78,21 @@ describe('turbo-carto error cases', function() {
         });
     });
 
+    it('should return invalid method from datasource', function(done) {
+        this.testClient = new TestClient(makeMapconfig(null, 'ramp([wadus_column], (red, green, blue), wadusmethod)'));
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('errors'));
+            assert.equal(layergroup.errors.length, 1);
+            assert.ok(layergroup.errors[0].match(/^turbo-carto/));
+            assert.ok(layergroup.errors[0].match(/unable\sto\scompute\sramp/i));
+            assert.ok(layergroup.errors[0].match(/invalid\smethod\s\"wadusmethod\"/i));
+
+            done();
+        });
+    });
+
     it('should fail by falling back to normal carto parser', function(done) {
         this.testClient = new TestClient(makeMapconfig('ramp([price], (8,24,96), (8,24,96));//(red, green, blue))'));
         this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {

test/acceptance/turbo-cartocss/named-maps.js

@@ -1,6 +1,6 @@
 var assert = require('../../support/assert');
 var step = require('step');
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 var testHelper = require('../../support/test_helper');
 var CartodbWindshaft = require('../../../lib/cartodb/server');
 var serverOptions = require('../../../lib/cartodb/server_options');

test/acceptance/turbo-cartocss/regressions.js

@@ -102,4 +102,69 @@ describe('turbo-carto regressions', function() {
             done();
         });
     });
+
+    it('should work with mapnik substitution tokens and analyses', function(done) {
+        var cartocss = [
+            "#layer {",
+            "  line-width: 2;",
+            "  line-color: #3B3B58;",
+            "  line-opacity: 1;",
+            "  polygon-opacity: 0.7;",
+            "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
+            "}"
+        ].join('\n');
+
+        var sqlWrap = [
+            'WITH hgrid AS (',
+            '  SELECT CDB_HexagonGrid(',
+            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
+            '    greatest(!pixel_width!,!pixel_height!) * 100',
+            '  ) as cell',
+            ')',
+            'SELECT',
+            '  hgrid.cell as the_geom_webmercator,',
+            '  count(1) as points_count,',
+            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
+            '  1 as cartodb_id',
+            'FROM hgrid, (<%= sql %>) i',
+            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
+            'GROUP BY hgrid.cell'
+        ].join('\n');
+
+        var mapConfig = {
+            "version": "1.5.0",
+            "layers": [
+                {
+                    "type": 'mapnik',
+                    "options": {
+                        "cartocss_version": '2.3.0',
+                        "source": {
+                            "id": "head"
+                        },
+                        sql_wrap: sqlWrap,
+                        "cartocss": cartocss
+                    }
+                }
+            ],
+            "analyses": [
+                {
+                    "id": "head",
+                    "type": "source",
+                    "params": {
+                        "query": "SELECT * FROM populated_places_simple_reduced"
+                    }
+                }
+            ]
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getLayergroup(function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('layergroupid'));
+            assert.ok(!layergroup.hasOwnProperty('errors'));
+
+            done();
+        });
+    });
 });

test/acceptance/widgets/named-maps.js

@@ -10,7 +10,7 @@ var CartodbWindshaft = require('../../../lib/cartodb/server');
 var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../../support/layergroup-token');
 
 describe('named-maps widgets', function() {
 

test/acceptance/x_cache_channel.js

@@ -8,7 +8,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('../support/layergroup-token');
 
 describe('get requests x-cache-channel', function() {
 

test/integration/mapconfig_named_layers_datasource.js

@@ -4,7 +4,7 @@ var assert = require('assert');
 var RedisPool = require('redis-mpool');
 var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');
 var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
-var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter');
+var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter');
 
 // configure redis pool instance to use in tests
 var redisPool = new RedisPool(global.environment.redis);
@@ -14,7 +14,7 @@ var templateMaps = new TemplateMaps(redisPool, {
     max_user_templates: global.environment.maxUserTemplates
 });
 
-var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps, pgConnection);
 
 var wadusSql = 'select 1 wadusLayer, null::geometry the_geom_webmercator';
 var wadusLayer = {
@@ -294,9 +294,11 @@ describe('named_layers datasources', function() {
 
     testScenarios.forEach(function(testScenario) {
         it('should return a list of layers ' + testScenario.desc, function(done) {
-            mapConfigNamedLayersAdapter.getLayers(username, testScenario.config.layers, pgConnection,
-                function(err, layers, datasource) {
-                    testScenario.test(err, layers, datasource, done);
+            var params = {};
+            var context = {};
+            mapConfigNamedLayersAdapter.getMapConfig(username, testScenario.config, params, context,
+                function(err, mapConfig) {
+                    testScenario.test(err, mapConfig.layers, context.datasource, done);
                 }
             );
         });

test/integration/mapconfig_named_layers_expanded.js

@@ -4,9 +4,9 @@ var assert = require('assert');
 var RedisPool = require('redis-mpool');
 var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');
 var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
-var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter');
+var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter');
 
-describe('mapconfig_named_layers_adapter', function() {
+describe('mapconfig-named-layers-adapter', function() {
 
     // configure redis pool instance to use in tests
     var redisPool = new RedisPool(global.environment.redis);
@@ -16,7 +16,7 @@ describe('mapconfig_named_layers_adapter', function() {
         max_user_templates: global.environment.maxUserTemplates
     });
 
-    var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps, pgConnection);
 
     var wadusLayer = {
         type: 'cartodb',
@@ -134,6 +134,8 @@ describe('mapconfig_named_layers_adapter', function() {
         };
     }
 
+    var params = {};
+    var context = {};
 
     beforeEach(function(done) {
         templateMaps.addTemplate(username, template, done);
@@ -147,11 +149,11 @@ describe('mapconfig_named_layers_adapter', function() {
         var missingNamedMapLayerConfig = makeNamedMapLayerConfig({
             config: {}
         });
-        mapConfigNamedLayersAdapter.getLayers(username, missingNamedMapLayerConfig.layers, pgConnection,
-            function(err, layers, datasource) {
+        mapConfigNamedLayersAdapter.getMapConfig(username, missingNamedMapLayerConfig, params, context,
+            function(err, mapConfig) {
                 assert.ok(err);
-                assert.ok(!layers);
-                assert.ok(!datasource);
+                assert.ok(!mapConfig);
+                assert.ok(!context.datasource);
                 assert.equal(err.message, 'Missing Named Map `name` in layer options');
 
                 done();
@@ -164,11 +166,11 @@ describe('mapconfig_named_layers_adapter', function() {
         var nonExistentNamedMapLayerConfig = makeNamedMapLayerConfig({
             name: missingTemplateName
         });
-        mapConfigNamedLayersAdapter.getLayers(username, nonExistentNamedMapLayerConfig.layers, pgConnection,
-            function(err, layers, datasource) {
+        mapConfigNamedLayersAdapter.getMapConfig(username, nonExistentNamedMapLayerConfig, params, context,
+            function(err, mapConfig) {
                 assert.ok(err);
-                assert.ok(!layers);
-                assert.ok(!datasource);
+                assert.ok(!mapConfig);
+                assert.ok(!context.datasource);
                 assert.equal(
                     err.message, "Template '" + missingTemplateName + "' of user '" + username + "' not found"
                 );
@@ -187,11 +189,11 @@ describe('mapconfig_named_layers_adapter', function() {
             var nonAuthTokensNamedMapLayerConfig = makeNamedMapLayerConfig({
                 name: tokenAuthTemplateName
             });
-            mapConfigNamedLayersAdapter.getLayers(username, nonAuthTokensNamedMapLayerConfig.layers, pgConnection,
-                function(err, layers, datasource) {
+            mapConfigNamedLayersAdapter.getMapConfig(username, nonAuthTokensNamedMapLayerConfig, params, context,
+                function(err, mapConfig) {
                     assert.ok(err);
-                    assert.ok(!layers);
-                    assert.ok(!datasource);
+                    assert.ok(!mapConfig);
+                    assert.ok(!context.datasource);
                     assert.equal(err.message, "Unauthorized '" + tokenAuthTemplateName + "' template instantiation");
 
                     templateMaps.delTemplate(username, tokenAuthTemplateName, done);
@@ -209,11 +211,11 @@ describe('mapconfig_named_layers_adapter', function() {
             var nestedNamedMapLayerConfig = makeNamedMapLayerConfig({
                 name: nestedNamedMapTemplateName
             });
-            mapConfigNamedLayersAdapter.getLayers(username, nestedNamedMapLayerConfig.layers, pgConnection,
-                function(err, layers, datasource) {
+            mapConfigNamedLayersAdapter.getMapConfig(username, nestedNamedMapLayerConfig, params, context,
+                function(err, mapConfig) {
                     assert.ok(err);
-                    assert.ok(!layers);
-                    assert.ok(!datasource);
+                    assert.ok(!mapConfig);
+                    assert.ok(!context.datasource);
                     assert.equal(err.message, 'Nested named layers are not allowed');
 
                     templateMaps.delTemplate(username, nestedNamedMapTemplateName, done);
@@ -226,12 +228,13 @@ describe('mapconfig_named_layers_adapter', function() {
         var validNamedMapMapLayerConfig = makeNamedMapLayerConfig({
             name: templateName
         });
-        mapConfigNamedLayersAdapter.getLayers(username, validNamedMapMapLayerConfig.layers, pgConnection,
-            function(err, layers, datasource) {
+        mapConfigNamedLayersAdapter.getMapConfig(username, validNamedMapMapLayerConfig, params, context,
+            function(err, mapConfig) {
                 assert.ok(!err);
+                var layers = mapConfig.layers;
                 assert.ok(layers.length, 1);
                 assert.ok(layers[0].type, 'cartodb');
-                assert.notEqual(datasource.getLayerDatasource(0), undefined);
+                assert.notEqual(context.datasource.getLayerDatasource(0), undefined);
 
                 done();
             }
@@ -248,11 +251,12 @@ describe('mapconfig_named_layers_adapter', function() {
                 name: tokenAuthTemplateName,
                 auth_tokens: ['valid1']
             });
-            mapConfigNamedLayersAdapter.getLayers(username, validAuthTokensNamedMapLayerConfig.layers, pgConnection,
-                function(err, layers, datasource) {
+            mapConfigNamedLayersAdapter.getMapConfig(username, validAuthTokensNamedMapLayerConfig, params, context,
+                function(err, mapConfig) {
                     assert.ok(!err);
+                    var layers = mapConfig.layers;
                     assert.equal(layers.length, 1);
-                    assert.notEqual(datasource.getLayerDatasource(0), undefined);
+                    assert.notEqual(context.datasource.getLayerDatasource(0), undefined);
 
                     templateMaps.delTemplate(username, tokenAuthTemplateName, done);
                 }
@@ -270,18 +274,19 @@ describe('mapconfig_named_layers_adapter', function() {
                 name: multipleLayersTemplateName,
                 auth_tokens: ['valid2']
             });
-            mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection,
-                function(err, layers, datasource) {
+            mapConfigNamedLayersAdapter.getMapConfig(username, multipleLayersNamedMapLayerConfig, params, context,
+                function(err, mapConfig) {
                     assert.ok(!err);
+                    var layers = mapConfig.layers;
                     assert.equal(layers.length, 2);
 
                     assert.equal(layers[0].type, 'mapnik');
                     assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: green; }');
-                    assert.notEqual(datasource.getLayerDatasource(0), undefined);
+                    assert.notEqual(context.datasource.getLayerDatasource(0), undefined);
 
                     assert.equal(layers[1].type, 'cartodb');
                     assert.equal(layers[1].options.cartocss, '#layer { marker-fill: red; }');
-                    assert.notEqual(datasource.getLayerDatasource(1), undefined);
+                    assert.notEqual(context.datasource.getLayerDatasource(1), undefined);
 
                     templateMaps.delTemplate(username, multipleLayersTemplateName, done);
                 }
@@ -306,18 +311,19 @@ describe('mapconfig_named_layers_adapter', function() {
                 },
                 auth_tokens: ['valid2']
             });
-            mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection,
-                function(err, layers, datasource) {
+            mapConfigNamedLayersAdapter.getMapConfig(username, multipleLayersNamedMapLayerConfig, params, context,
+                function(err, mapConfig) {
                     assert.ok(!err);
+                    var layers = mapConfig.layers;
                     assert.equal(layers.length, 2);
 
                     assert.equal(layers[0].type, 'mapnik');
                     assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: ' + polygonColor + '; }');
-                    assert.notEqual(datasource.getLayerDatasource(0), undefined);
+                    assert.notEqual(context.datasource.getLayerDatasource(0), undefined);
 
                     assert.equal(layers[1].type, 'cartodb');
                     assert.equal(layers[1].options.cartocss, '#layer { marker-fill: ' + color + '; }');
-                    assert.notEqual(datasource.getLayerDatasource(1), undefined);
+                    assert.notEqual(context.datasource.getLayerDatasource(1), undefined);
 
                     templateMaps.delTemplate(username, multipleLayersTemplateName, done);
                 }

test/integration/mapconfig_overviews_adapter.js

@@ -7,11 +7,7 @@ var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connectio
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
 var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
 var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api');
-var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
-
-// configure redis pool instance to use in tests
-var redisPool = new RedisPool(global.environment.redis);
-var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool }));
+var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter');
 
 var redisPool = new RedisPool(global.environment.redis);
 var metadataBackend = cartodbRedis({pool: redisPool});
@@ -37,8 +33,16 @@ describe('MapConfigOverviewsAdapter', function() {
             }
         };
 
-        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
+        var _mapConfig = {
+            layers: [layer_without_overviews]
+        };
+
+        var params = {};
+        var context = {};
+
+        mapConfigOverviewsAdapter.getMapConfig('localhost', _mapConfig, params, context, function(err, mapConfig) {
             assert.ok(!err);
+            var layers = mapConfig.layers;
             assert.equal(layers.length, 1);
             assert.equal(layers[0].type, 'cartodb');
             assert.equal(layers[0].options.sql, sql);
@@ -56,7 +60,7 @@ describe('MapConfigOverviewsAdapter', function() {
         var sql = 'SELECT * FROM test_table_overviews';
         var cartocss = '#layer { marker-fill: black; }';
         var cartocss_version = '2.3.0';
-        var layer_without_overviews = {
+        var layer_with_overviews = {
             type: 'cartodb',
             options: {
                 sql: sql,
@@ -65,8 +69,16 @@ describe('MapConfigOverviewsAdapter', function() {
             }
         };
 
-        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
+        var _mapConfig = {
+            layers: [layer_with_overviews]
+        };
+
+        var params = {};
+        var context = {};
+
+        mapConfigOverviewsAdapter.getMapConfig('localhost', _mapConfig, params, context, function(err, mapConfig) {
             assert.ok(!err);
+            var layers = mapConfig.layers;
             assert.equal(layers.length, 1);
             assert.equal(layers[0].type, 'cartodb');
             assert.equal(layers[0].options.sql, sql);

test/support/prepare_db.sh

@@ -83,7 +83,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
 
   cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
-  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
+  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
   for i in ${SQL_SCRIPTS}
   do
     curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql

test/support/test-client.js

@@ -6,7 +6,7 @@ var urlParser = require('url');
 
 var mapnik = require('windshaft').mapnik;
 
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var LayergroupToken = require('./layergroup-token');
 
 var assert = require('./assert');
 var helper = require('./test_helper');

test/unit/cartodb/mapconfig/adapter.test.js

@@ -0,0 +1,93 @@
+//require('../../../support/test_helper');
+var assert = require('assert');
+
+var MapConfigAdapter = require('../../../../lib/cartodb/models/mapconfig/adapter');
+
+describe('MapConfigAdapter', function() {
+    var user = 'wadus';
+    function requestMapConfig() {
+        return {
+            val: 0
+        };
+    }
+    function params() {
+        return {};
+    }
+    function context() {
+        return {};
+    }
+
+    function createAdapter(valOperatorFn) {
+        return function ValMapConfigAdapter() {
+            this.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+                requestMapConfig.val = valOperatorFn(requestMapConfig.val);
+                return callback(null, requestMapConfig);
+            };
+        };
+    }
+    var IncValMapConfigAdapter = createAdapter(function(val) { return val + 1; });
+    var Mul2ValMapConfigAdapter = createAdapter(function(val) { return val * 2; });
+
+    function validateMapConfig(adapter, expectedNumAdapters, expectedVal, callback) {
+        assert.equal(adapter.adapters.length, expectedNumAdapters);
+        adapter.getMapConfig(user, requestMapConfig(), params(), context(), function(err, mapConfig) {
+            assert.equal(mapConfig.val, expectedVal);
+            return callback(err);
+        });
+    }
+
+    it('works with no adapters', function(done) {
+        var adapter = new MapConfigAdapter();
+        validateMapConfig(adapter, 0, 0, done);
+    });
+
+    it('works with no adapters as empty array', function(done) {
+        var adapter = new MapConfigAdapter([]);
+        validateMapConfig(adapter, 0, 0, done);
+    });
+
+    it('works with basic adapter', function(done) {
+        var adapter = new MapConfigAdapter(new IncValMapConfigAdapter());
+        validateMapConfig(adapter, 1, 1, done);
+    });
+
+    it('works with basic adapter as array', function(done) {
+        var adapter = new MapConfigAdapter([new IncValMapConfigAdapter()]);
+        validateMapConfig(adapter, 1, 1, done);
+    });
+
+    it('works with several adapters', function(done) {
+        var adapter = new MapConfigAdapter(new IncValMapConfigAdapter(), new IncValMapConfigAdapter());
+        validateMapConfig(adapter, 2, 2, done);
+    });
+
+    it('works with several adapters as array', function(done) {
+        var adapter = new MapConfigAdapter([new IncValMapConfigAdapter(), new IncValMapConfigAdapter()]);
+        validateMapConfig(adapter, 2, 2, done);
+    });
+
+    it('should execute in order 1', function(done) {
+        var adapter = new MapConfigAdapter([new Mul2ValMapConfigAdapter(), new IncValMapConfigAdapter()]);
+        validateMapConfig(adapter, 2, 1, done);
+    });
+
+    it('should execute in order 2', function(done) {
+        var adapter = new MapConfigAdapter([new IncValMapConfigAdapter(), new Mul2ValMapConfigAdapter()]);
+        validateMapConfig(adapter, 2, 2, done);
+    });
+
+    it('should execute in order 3', function(done) {
+        var adapter = new MapConfigAdapter([new Mul2ValMapConfigAdapter(), new Mul2ValMapConfigAdapter()]);
+        validateMapConfig(adapter, 2, 0, done);
+    });
+
+    it('should execute in order 4', function(done) {
+        var Mul5ValMapConfigAdapter = createAdapter(function(val) { return val * 5; });
+        var adapter = new MapConfigAdapter(
+            new IncValMapConfigAdapter(),
+            new Mul2ValMapConfigAdapter(),
+            new Mul5ValMapConfigAdapter()
+        );
+        validateMapConfig(adapter, 3, 10, done);
+    });
+});

test/unit/cartodb/substitution-tokens.test.js

@@ -0,0 +1,40 @@
+var assert = require('assert');
+var SubstitutionTokens = require('../../../lib/cartodb/utils/substitution-tokens');
+
+describe('SubstitutionTokens', function() {
+
+    var sql = [
+        'WITH hgrid AS (',
+        '  SELECT CDB_HexagonGrid(',
+        '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
+        '    greatest(!pixel_width!,!pixel_height!) * 100',
+        '  ) as cell',
+        ')',
+        'SELECT',
+        '  hgrid.cell as the_geom_webmercator,',
+        '  count(1) as points_count,',
+        '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
+        '  1 as cartodb_id',
+        'FROM hgrid, (select * from table) i',
+        'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
+        'GROUP BY hgrid.cell'
+    ].join('\n');
+
+    it('should return tokens present in sql', function() {
+        assert.deepEqual(SubstitutionTokens.tokens(sql), ['bbox', 'scale_denominator', 'pixel_width', 'pixel_height']);
+    });
+
+    it('should return just one token', function() {
+        assert.deepEqual(SubstitutionTokens.tokens('select !bbox! from wadus'), ['bbox']);
+    });
+
+    it('should not return other tokens', function() {
+        assert.deepEqual(SubstitutionTokens.tokens('select !wadus! from wadus'), []);
+    });
+
+    it('should report sql has tokens', function() {
+        assert.equal(SubstitutionTokens.hasTokens(sql), true);
+        assert.equal(SubstitutionTokens.hasTokens('select !bbox! from wadus'), true);
+        assert.equal(SubstitutionTokens.hasTokens('select !wadus! from wadus'), false);
+    });
+});