Release 2.28.0

Move turbo cartocss

.travis.yml

@@ -1,18 +1,18 @@
 sudo: false
 
 addons:
-  postgresql: "9.4"
+  postgresql: "9.3"
   apt:
     packages:
-    - postgresql-plpython-9.4
-    - pkg-config
-    - libcairo2-dev
-    - libjpeg8-dev
-    - libgif-dev
+      - pkg-config
+      - libcairo2-dev
+      - libjpeg8-dev
+      - libgif-dev
 
 before_install:
   - npm install -g npm@2
   - createdb template_postgis
+  - createuser publicuser
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
 env:

INSTALL.md

@@ -16,10 +16,6 @@ Make sure that you have the requirements needed. These are
   - CartoDB 0.9.5+ (for `CDB_QueryTables`)
   - Varnish (http://www.varnish-cache.org)
 
-- For running the testsuite
-  - ImageMagick (http://www.imagemagick.org)
-
-
 On Ubuntu 14.04 the dependencies can be installed with
 
 ```shell

NEWS.md

@@ -1,5 +1,115 @@
 # Changelog
 
+
+## 2.28.0
+
+Released 2016-03-14
+
+New features:
+ - Added [turbo-cartocss](https://github.com/CartoDB/turbo-cartocss) to preprocess CartoCSS.
+
+## 2.27.0
+
+Released 2016-03-09
+
+New features:
+ - Add [Surrogate-Key](https://github.com/CartoDB/cartodb/wiki/CartoDB-Surrogate-Keys) headers to responses
+
+Enhancements:
+ - Use new `node-cartodb-query-tables` library to obtain affected tables in queries
+
+Announcements:
+ - Remove deprecated tools directory
+
+
+## 2.26.3
+
+Released 2016-03-03
+
+Improvements:
+ - Optimize overviews queries for efficient spatial filtering in PostgreSQL
+
+
+## 2.26.2
+
+Released 2016-02-25
+
+Announcements:
+ - Upgrades windshaft to [1.13.2](https://github.com/CartoDB/Windshaft/releases/tag/1.13.2)
+
+
+## 2.26.1
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.1](https://github.com/CartoDB/Windshaft/releases/tag/1.13.1)
+
+
+## 2.26.0
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.0](https://github.com/CartoDB/Windshaft/releases/tag/1.13.0)
+
+
+## 2.25.2
+
+Released 2016-02-22
+
+Bug fixes:
+ - Correct URLs for widgets in named maps #381
+
+
+## 2.25.1
+
+Released 2016-02-22
+
+Announcements:
+ - Upgrades windshaft to [1.11.1](https://github.com/CartoDB/Windshaft/releases/tag/1.11.1)
+
+
+## 2.25.0
+
+Released 2016-02-18
+
+Announcements:
+ - Upgrades windshaft to [1.11.0](https://github.com/CartoDB/Windshaft/releases/tag/1.11.0)
+
+
+## 2.24.0
+
+Released 2016-02-15
+
+Announcements:
+ - Upgrades windshaft to [1.10.1](https://github.com/CartoDB/Windshaft/releases/tag/1.10.1)
+
+
+## 2.23.0
+
+Released 2016-02-10
+
+Improvements:
+- Support for overviews
+
+
+## 2.22.0
+
+Released 2016-02-08
+
+Announcements:
+ - Upgrades windshaft to [1.8.3](https://github.com/CartoDB/Windshaft/releases/tag/1.8.3)
+
+
+## 2.21.1
+
+Released 2016-02-05
+
+Bug fixes:
+ - Added default config for geojson renderer
+
+
 ## 2.21.0
 
 Released 2016-02-04

README.md

@@ -67,3 +67,16 @@ Contributing
 ---
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+### Developing with a custom windshaft version
+
+If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
+to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
+
+**Quick start**:
+
+```shell
+~/windshaft-directory $ npm install
+~/windshaft-directory $ npm link
+~/windshaft-cartodb-directory $ npm link windshaft
+```

docs/named_maps.md

@@ -461,6 +461,9 @@ cartodb.createLayer('map_dom_id',layerSource)
 [CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js/) has methods for accessing your named maps.
 
 1. [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
+
+    **Note:** The CartoDB.js `layer.setParams()` function is not supported when using Named maps for Torque.
+
 2. [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
 
 ### Complete Examples of Named Maps created with CartoDB.js

lib/cartodb/api/overviews_metadata_api.js

@@ -0,0 +1,43 @@
+function OverviewsMetadataApi(pgQueryRunner) {
+   this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = OverviewsMetadataApi;
+
+// TODO: share this with QueryTablesApi? ... or maintain independence?
+var affectedTableRegexCache = {
+    bbox: /!bbox!/g,
+    scale_denominator: /!scale_denominator!/g,
+    pixel_width: /!pixel_width!/g,
+    pixel_height: /!pixel_height!/g
+};
+
+function prepareSql(sql) {
+    return sql
+        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+        .replace(affectedTableRegexCache.scale_denominator, '0')
+        .replace(affectedTableRegexCache.pixel_width, '1')
+        .replace(affectedTableRegexCache.pixel_height, '1')
+    ;
+}
+
+OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) {
+    var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
+    this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
+        if (err){
+            callback(err);
+            return;
+        }
+        var metadata = {};
+        rows.forEach(function(row) {
+            var table = row.base_table;
+            var table_metadata = metadata[table];
+            if ( !table_metadata ) {
+              table_metadata = metadata[table] = {};
+            }
+            table_metadata[row.z] = { table: row.overview_table };
+        });
+        return callback(null, metadata);
+    });
+
+};

lib/cartodb/api/query_tables_api.js

@@ -1,96 +0,0 @@
-function QueryTablesApi(pgQueryRunner) {
-    this.pgQueryRunner = pgQueryRunner;
-}
-
-var affectedTableRegexCache = {
-    bbox: /!bbox!/g,
-    scale_denominator: /!scale_denominator!/g,
-    pixel_width: /!pixel_width!/g,
-    pixel_height: /!pixel_height!/g
-};
-
-module.exports = QueryTablesApi;
-
-
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
-    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
-
-    this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
-        if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch source tables: ' + msg));
-            return;
-        }
-
-        // This is an Array, so no need to split into parts
-        var tableNames = rows[0].cdb_querytablestext;
-        return callback(null, tableNames);
-    });
-};
-
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
-    var query = [
-        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
-        ')',
-        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m',
-        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
-        if (err || rows.length === 0) {
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-            return;
-        }
-
-        var result = rows[0];
-
-        // This is an Array, so no need to split into parts
-        var tableNames = result.tablenames;
-
-        var lastUpdatedTime = result.max || 0;
-
-        callback(null, {
-            affectedTables: tableNames,
-            lastUpdatedTime: lastUpdatedTime * 1000
-        });
-    });
-};
-
-QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
-    if (!Array.isArray(tableNames) || tableNames.length === 0) {
-        return callback(null, 0);
-    }
-
-    var query = [
-        'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
-            tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
-        '])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
-        if (err) {
-            var msg = err.message ? err.message : err;
-            return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-        }
-        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
-        var lastUpdated = 0;
-        if (rows.length !== 0) {
-            lastUpdated = rows[0].max || 0;
-        }
-
-        return callback(null, lastUpdated*1000);
-    });
-};
-
-function prepareSql(sql) {
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
-}

lib/cartodb/api/tables_extent_api.js

@@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
  *  `table_name` format as valid input
  * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
  */
-TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) {
-    var estimatedExtentSQLs = tableNames.map(function(tableName) {
-        var schemaTable = tableName.split('.');
-        if (schemaTable.length > 1) {
-            return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
-        }
-        return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
+TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
+    var estimatedExtentSQLs = tables.map(function(table) {
+        return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
     });
 
     var query = [

lib/cartodb/backends/pg_connection.js

@@ -1,5 +1,6 @@
 var assert = require('assert');
 var step = require('step');
+var PSQL = require('cartodb-psql');
 var _ = require('underscore');
 
 function PgConnection(metadataBackend) {
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
         }
     );
 };
+
+
+/**
+ * Returns a `cartodb-psql` object for a given username.
+ * @param {String} username
+ * @param {Function} callback function({Error}, {PSQL})
+ */
+
+PgConnection.prototype.getConnection = function(username, callback) {
+    var self = this;
+
+    var params = {};
+
+    require('debug')('cachechan')("getConn1");
+    step(
+        function setAuth() {
+            self.setDBAuth(username, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.setDBConn(username, params, this);
+        },
+        function openConnection(err) {
+            assert.ifError(err);
+            return callback(err, new PSQL({
+                user: params.dbuser,
+                pass: params.dbpass,
+                host: params.dbhost,
+                port: params.dbport,
+                dbname: params.dbname
+            }));
+        }
+    );
+};

lib/cartodb/cache/model/database_tables_entry.js

@@ -1,24 +0,0 @@
-var crypto = require('crypto');
-
-function DatabaseTables(dbName, tableNames) {
-    this.namespace = 't';
-    this.dbName = dbName;
-    this.tableNames = tableNames;
-}
-
-module.exports = DatabaseTables;
-
-
-DatabaseTables.prototype.key = function() {
-    return this.tableNames.map(function(tableName) {
-        return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
-    }.bind(this));
-};
-
-DatabaseTables.prototype.getCacheChannel = function() {
-    return this.dbName + ':' + this.tableNames.join(',');
-};
-
-function shortHashKey(target) {
-    return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
-}

lib/cartodb/cache/named_map_provider_cache.js

@@ -7,13 +7,13 @@ var queue = require('queue-async');
 
 var LruCache = require("lru-cache");
 
-function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
+function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, turboCartocssAdapter) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.turboCartocssAdapter = turboCartocssAdapter;
 
     this.providerCache = new LruCache({ max: 2000 });
 }
@@ -30,8 +30,8 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
             this.templateMaps,
             this.pgConnection,
             this.userLimitsApi,
-            this.queryTablesApi,
             this.namedLayersAdapter,
+            this.turboCartocssAdapter,
             user,
             templateId,
             config,

lib/cartodb/controllers/base.js

@@ -239,6 +239,7 @@ module.exports.findStatusCode = findStatusCode;
 
 function statusFromErrorMessage(errMsg) {
     // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
     var statusCode = 400;
     if ( -1 !== errMsg.indexOf('permission denied') ) {
         statusCode = 403;
@@ -252,6 +253,8 @@ function statusFromErrorMessage(errMsg) {
     else if ( -1 !== errMsg.indexOf('does not exist') ) {
         if ( -1 !== errMsg.indexOf(' role ') ) {
             statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
         } else {
             statusCode = 404;
         }

lib/cartodb/controllers/layergroup.js

@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
 var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
+
+var QueryTables = require('cartodb-query-tables');
 
 /**
  * @param {AuthApi} authApi
@@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
  * @param {WidgetBackend} widgetBackend
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
- * @param {QueryTablesApi} queryTablesApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
 function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
-                          widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
+                          widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
     BaseController.call(this, authApi, pgConnection);
 
+    this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
@@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
     this.widgetBackend = widgetBackend;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
 }
 
@@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 global.logger.warn('ERROR generating cache channel: ' + err);
             }
             if (!!affectedTables) {
-                var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
-                self.surrogateKeysCache.tag(res, tablesCacheEntry);
+                res.set('X-Cache-Channel', affectedTables.getCacheChannel());
+                self.surrogateKeysCache.tag(res, affectedTables);
             }
             self.send(req, res, body, status, headers);
         }
@@ -366,17 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 throw new Error("this request doesn't need an X-Cache-Channel generated");
             }
 
-            self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(user, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
-        function buildCacheChannel(err, tableNames) {
+        function buildCacheChannel(err, tables) {
             assert.ifError(err);
+            self.layergroupAffectedTables.set(dbName, layergroupId, tables);
 
-            self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
-
-            return tableNames;
+            return tables;
         },
-        function finish(err, affectedTables) {
-            callback(err, affectedTables);
-        }
+        callback
     );
 };

lib/cartodb/controllers/map.js

@@ -2,6 +2,7 @@ var _ = require('underscore');
 var assert = require('assert');
 var step = require('step');
 var windshaft = require('windshaft');
+var QueryTables = require('cartodb-query-tables');
 
 var util = require('util');
 var BaseController = require('./base');
@@ -13,11 +14,11 @@ var MapConfig = windshaft.model.MapConfig;
 var Datasource = windshaft.model.Datasource;
 
 var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
 
 var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
 var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
 var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
+var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter');
 
 /**
  * @param {AuthApi} authApi
@@ -25,14 +26,15 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
  * @param {TemplateMaps} templateMaps
  * @param {MapBackend} mapBackend
  * @param metadataBackend
- * @param {QueryTablesApi} queryTablesApi
+ * @param {OverviewsMetadataApi} overviewsMetadataApi
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
-function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi,
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
+function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
+                       overviewsMetadataApi,
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, turboCartoCssAdapter) {
 
     BaseController.call(this, authApi, pgConnection);
 
@@ -40,12 +42,14 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.templateMaps = templateMaps;
     this.mapBackend = mapBackend;
     this.metadataBackend = metadataBackend;
-    this.queryTablesApi = queryTablesApi;
+    this.overviewsMetadataApi = overviewsMetadataApi;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
+    this.turboCartoCssAdapter = turboCartoCssAdapter;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi);
 }
 
 util.inherits(MapController, BaseController);
@@ -148,6 +152,37 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
                 }
             );
         },
+        function addOverviewsInformation(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+            self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
+        },
+        function parseTurboCartoCss(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+
+            var next = this;
+            self.turboCartoCssAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
+        },
         function createLayergroup(err, requestMapConfig, datasource) {
             assert.ifError(err);
             mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource());
@@ -165,6 +200,8 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
             if (err) {
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
+                addWidgetsUrl(req.context.user, layergroup);
+
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -193,8 +230,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 self.templateMaps,
                 self.pgConnection,
                 self.userLimitsApi,
-                self.queryTablesApi,
                 self.namedLayersAdapter,
+                self.turboCartoCssAdapter,
                 cdbuser,
                 req.params.template_id,
                 templateParams,
@@ -203,7 +240,23 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
             );
             mapConfigProvider.getMapConfig(this);
         },
-        function createLayergroup(err, mapConfig_, rendererParams/*, context*/) {
+        function addOverviewsInformation(err, requestMapConfig, rendererParams/*, context*/) {
+          assert.ifError(err);
+          var next = this;
+          self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
+              function(err, layers) {
+                  if (err) {
+                      return next(err);
+                  }
+
+                  if (layers) {
+                      requestMapConfig.layers = layers;
+                  }
+                  return next(null, requestMapConfig, rendererParams);
+              }
+          );
+        },
+        function createLayergroup(err, mapConfig_, rendererParams) {
             assert.ifError(err);
             mapConfig = mapConfig_;
             self.mapBackend.createLayergroup(
@@ -223,6 +276,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
+                addWidgetsUrl(req.context.user, layergroup);
+
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
 
@@ -277,46 +332,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
     var layergroupId = layergroup.layergroupid;
 
     step(
-        function checkCachedAffectedTables() {
-            return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
+        function getPgConnection() {
+            self.pgConnection.getConnection(username, this);
         },
-        function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
+        function getAffectedTablesAndLastUpdatedTime(err, connection) {
             assert.ifError(err);
-            if (hasCache) {
-                var next = this;
-                var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
-                self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
-                    if (err) {
-                        return next(err);
-                    }
-                    return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
-                });
-            } else {
-                self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
-            }
+            QueryTables.getAffectedTablesFromQuery(connection, sql, this);
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
             if (req.profiler) {
                 req.profiler.done('queryTablesAndLastUpdated');
             }
             assert.ifError(err);
-            self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
+            // feed affected tables cache so it can be reused from, for instance, layergroup controller
+            self.layergroupAffectedTables.set(dbName, layergroupId, result);
 
             // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
-            layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
+            layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
+            layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
 
             // TODO this should take into account several URL patterns
             addWidgetsUrl(username, layergroup);
-
             if (req.method === 'GET') {
-                var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
                 var ttl = global.environment.varnish.layergroupTtl || 86400;
                 res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
                 res.set('Last-Modified', (new Date()).toUTCString());
-                res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
-                if (result.affectedTables && result.affectedTables.length > 0) {
-                    self.surrogateKeysCache.tag(res, tableCacheEntry);
+                res.set('X-Cache-Channel', result.getCacheChannel());
+                if (result.tables && result.tables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
                 }
             }
 

lib/cartodb/controllers/named_maps.js

@@ -9,8 +9,6 @@ var BaseController = require('./base');
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
-
 function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
     BaseController.call(this, authApi, pgConnection);
@@ -44,7 +42,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
 
     var self = this;
 
-    var dbName = req.params.dbname;
     step(
         function getAffectedTablesAndLastUpdatedTime() {
             namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
@@ -54,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
             if (err) {
                 global.logger.log('ERROR generating cache channel: ' + err);
             }
-            if (!result || !!result.affectedTables) {
+            if (!result || !!result.tables) {
                 // we increase cache control as we can invalidate it
                 res.set('Cache-Control', 'public,max-age=31536000');
 
                 var lastModifiedDate;
                 if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.lastUpdatedTime);
+                    lastModifiedDate = new Date(result.getLastUpdatedAt());
                 } else {
                     lastModifiedDate = new Date();
                 }
                 res.set('Last-Modified', lastModifiedDate.toUTCString());
 
-                var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
-                if (result.affectedTables.length > 0) {
-                    self.surrogateKeysCache.tag(res, tablesCacheEntry);
+                res.set('X-Cache-Channel', result.getCacheChannel());
+                if (result.tables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
                 }
             }
             self.send(req, res, resource, 200);
@@ -231,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
                     return next(null);
                 }
 
-                var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
+                var affectedTables = affectedTablesAndLastUpdate.tables || [];
 
                 if (affectedTables.length === 0) {
                     return next(null);

lib/cartodb/controllers/named_maps_admin.js

@@ -65,6 +65,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
     var cdbuser = req.context.user;
     var template;
     var tpl_id;
+
     step(
         function checkPerms(){
             self.authApi.authorizedByAPIKey(cdbuser, req, this);

lib/cartodb/models/mapconfig/named_map_provider.js

@@ -5,18 +5,19 @@ var dot = require('dot');
 var step = require('step');
 var MapConfig = require('windshaft').model.MapConfig;
 var templateName = require('../../backends/template_maps').templateName;
+var QueryTables = require('cartodb-query-tables');
 
 /**
  * @constructor
  * @type {NamedMapMapConfigProvider}
  */
-function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
-                                   owner, templateId, config, authToken, params) {
+function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter,
+                                   turboCartoCssAdapter, owner, templateId, config, authToken, params) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
     this.namedLayersAdapter = namedLayersAdapter;
+    this.turboCartoCssAdapter = turboCartoCssAdapter;
 
     this.owner = owner;
     this.templateName = templateName(templateId);
@@ -91,6 +92,18 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
                 }
             );
         },
+        function parseTurboCartoCss(err, _mapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+
+            self.turboCartoCssAdapter.getLayers(self.owner, _mapConfig.layers, function (err, layers) {
+                if (!err && layers) {
+                    _mapConfig.layers = layers;
+                }
+
+                return next(null, _mapConfig, datasource);
+            });
+        },
         function beforeLayergroupCreate(err, _mapConfig, _datasource) {
             assert.ifError(err);
             mapConfig = _mapConfig;
@@ -256,7 +269,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
         },
         function getAffectedTables(err, sql) {
             assert.ifError(err);
-            self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(self.owner, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
         function finish(err, result) {
             self.affectedTablesAndLastUpdate = result;

lib/cartodb/models/mapconfig_overviews_adapter.js

@@ -0,0 +1,53 @@
+var queue = require('queue-async');
+var _ = require('underscore');
+
+function MapConfigOverviewsAdapter(overviewsMetadataApi) {
+    this.overviewsMetadataApi = overviewsMetadataApi;
+}
+
+module.exports = MapConfigOverviewsAdapter;
+
+MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
+  var self = this;
+
+  if (!layers || layers.length === 0) {
+      return callback(null, layers);
+  }
+
+  var augmentLayersQueue = queue(layers.length);
+
+  function augmentLayer(layer, done) {
+      if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
+          return done(null, layer);
+      }
+      self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){
+           if (err) {
+               done(err, layer);
+           } else {
+               if ( !_.isEmpty(metadata) ) {
+                   layer = _.extend({}, layer);
+                   layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
+               }
+               done(null, layer);
+           }
+      });
+  }
+
+  function layersAugmentQueueFinish(err, layers) {
+    if (err) {
+        return callback(err);
+    }
+
+    if (!layers || layers.length === 0) {
+        return callback(new Error('Missing layers array from layergroup config'));
+    }
+
+    return callback(null, layers);
+  }
+
+  layers.forEach(function(layer) {
+      augmentLayersQueue.defer(augmentLayer, layer);
+  });
+  augmentLayersQueue.awaitAll(layersAugmentQueueFinish);
+
+};

lib/cartodb/server.js

@@ -19,7 +19,7 @@ var windshaft = require('windshaft');
 var mapnik = windshaft.mapnik;
 
 var TemplateMaps = require('./backends/template_maps.js');
-var QueryTablesApi = require('./api/query_tables_api');
+var OverviewsMetadataApi = require('./api/overviews_metadata_api');
 var UserLimitsApi = require('./api/user_limits_api');
 var AuthApi = require('./api/auth_api');
 var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
@@ -30,6 +30,8 @@ var PgConnection = require('./backends/pg_connection');
 var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
+var TurboCartocssParser = require('./utils/style/turbo-cartocss-parser');
+var TurboCartocssAdapter = require('./utils/style/turbo-cartocss-adapter');
 
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
@@ -51,7 +53,7 @@ module.exports = function(serverOptions) {
     var metadataBackend = cartodbRedis({pool: redisPool});
     var pgConnection = new PgConnection(metadataBackend);
     var pgQueryRunner = new PgQueryRunner(pgConnection);
-    var queryTablesApi = new QueryTablesApi(pgQueryRunner);
+    var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
     var userLimitsApi = new UserLimitsApi(metadataBackend, {
         limits: {
             cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
@@ -140,7 +142,16 @@ module.exports = function(serverOptions) {
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
+    var turboCartoCssParser = new TurboCartocssParser(pgQueryRunner);
+    var turboCartocssAdapter = new TurboCartocssAdapter(turboCartoCssParser);
+
+    var namedMapProviderCache = new NamedMapProviderCache(
+        templateMaps,
+        pgConnection,
+        userLimitsApi,
+        turboCartocssAdapter
+    );
+
     ['update', 'delete'].forEach(function(eventType) {
         templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
     });
@@ -164,7 +175,6 @@ module.exports = function(serverOptions) {
         new windshaft.backend.Widget(),
         surrogateKeysCache,
         userLimitsApi,
-        queryTablesApi,
         layergroupAffectedTablesCache
     ).register(app);
 
@@ -174,10 +184,11 @@ module.exports = function(serverOptions) {
         templateMaps,
         mapBackend,
         metadataBackend,
-        queryTablesApi,
+        overviewsMetadataApi,
         surrogateKeysCache,
         userLimitsApi,
-        layergroupAffectedTablesCache
+        layergroupAffectedTablesCache,
+        turboCartocssAdapter
     ).register(app);
 
     new controller.NamedMaps(

lib/cartodb/server_options.js

@@ -1,5 +1,10 @@
 var os = require('os');
 var _ = require('underscore');
+var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter');
+
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+  zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
+});
 
 var rendererConfig = _.defaults(global.environment.renderer || {}, {
     cache_ttl: 60000, // milliseconds
@@ -15,6 +20,8 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, {
     http: {}
 });
 
+rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter;
+
 // Perform keyword substitution in statsd
 // See https://github.com/CartoDB/Windshaft-cartodb/issues/153
 if ( global.environment.statsd ) {
@@ -61,7 +68,16 @@ module.exports = {
             statsInterval: rendererConfig.statsInterval
         },
         renderer: {
-            mapnik: rendererConfig.mapnik,
+            mapnik: _.defaults(rendererConfig.mapnik, {
+                geojson: {
+                    dbPoolParams: {
+                          size: 16,
+                          idleTimeout: 3000,
+                          reapInterval: 1000
+                    },
+                    clipByBox2d: false,
+                }
+            }),
             torque: rendererConfig.torque,
             http: rendererConfig.http
         },

lib/cartodb/utils/overviews_query_rewriter.js

@@ -0,0 +1,178 @@
+var TableNameParser = require('./table_name_parser');
+
+function OverviewsQueryRewriter(options) {
+
+    this.options = options;
+}
+
+module.exports = OverviewsQueryRewriter;
+
+// TODO: some names are introudced in the queries, and the
+// '_vovw_' (for vector overviews) is used in them, but no check
+// is performed for conflicts with existing identifiers in the query.
+
+// Build UNION expression to replace table, using overviews metadata
+// overviews metadata: { 1: 'table_ov1', ... }
+// assume table and overview names include schema if necessary and are quoted as needed
+function overviews_view_for_table(table, overviews_metadata, indent) {
+    var condition, i, len, ov_table, overview_layers, selects, z_hi, z_lo;
+    var parsed_table = TableNameParser.parse(table);
+
+    var sorted_overviews = []; // [[1, 'table_ov1'], ...]
+
+    indent = indent || '    ';
+    for (var z in overviews_metadata) {
+        if (overviews_metadata.hasOwnProperty(z)) {
+            sorted_overviews.push([z, overviews_metadata[z].table]);
+        }
+    }
+    sorted_overviews.sort(function(a, b){ return a[0]-b[0]; });
+
+    overview_layers = [];
+    z_lo = null;
+    for (i = 0, len = sorted_overviews.length; i < len; i++) {
+        z_hi = parseInt(sorted_overviews[i][0]);
+        ov_table = sorted_overviews[i][1];
+        overview_layers.push([overview_z_condition(z_lo, z_hi), ov_table]);
+        z_lo = z_hi;
+    }
+    overview_layers.push(["_vovw_z > " + z_lo, table]);
+
+    selects = overview_layers.map(function(condition_table) {
+      condition = condition_table[0];
+      ov_table = TableNameParser.parse(condition_table[1]);
+      ov_table.schema = ov_table.schema || parsed_table.schema;
+      var ov_identifier = TableNameParser.table_identifier(ov_table);
+      return indent + "SELECT * FROM " + ov_identifier + ", _vovw_scale WHERE " + condition;
+    });
+
+    return selects.join("\n"+indent+"UNION ALL\n");
+}
+
+function overview_z_condition(z_lo, z_hi) {
+    if (z_lo !== null) {
+        if (z_lo === z_hi - 1) {
+            return "_vovw_z = " + z_hi;
+        } else {
+            return "_vovw_z > " + z_lo + " AND _vovw_z <= " + z_hi;
+        }
+    } else {
+        if (z_hi === 0) {
+            return "_vovw_z = " + z_hi;
+        } else {
+            return "_vovw_z <= " + z_hi;
+        }
+    }
+}
+
+// name to be used for the view of the table using overviews
+function overviews_view_name(table) {
+    var parsed_table = TableNameParser.parse(table);
+    parsed_table.table = '_vovw_' + parsed_table.table;
+    parsed_table.schema = null;
+    return TableNameParser.table_identifier(parsed_table);
+}
+
+// replace a table name in a query by anoter name
+function replace_table_in_query(sql, old_table_name, replacement) {
+    var old_table = TableNameParser.parse(old_table_name);
+    var old_table_ident = TableNameParser.table_identifier(old_table);
+
+    // regular expression prefix (beginning) to match a table name
+    function pattern_prefix(schema, identifier) {
+        if ( schema ) {
+            // to match a table name including schema prefix
+            // name should not be part of another name, so we require
+            // to start a at a word boundary
+            if ( identifier[0] !== '"' ) {
+                return '\\b';
+            } else {
+                return '';
+            }
+        } else {
+           // to match a table name without schema
+           // name should not begin right after a dot (i.e. have a explicit schema)
+           // nor be part of another name
+           // since the pattern matches the first character of the table
+           // it must be put back in the replacement text
+           replacement = '$01'+replacement;
+           return '([^\.a-z0-9_]|^)';
+        }
+    }
+
+    // regular expression suffix (ending) to match a table name
+    function pattern_suffix(identifier) {
+        // name shouldn't be the prefix of a longer name
+        if ( identifier[identifier.length-1] !== '"' ) {
+            return '\\b';
+        } else {
+            return '';
+        }
+    }
+
+    // regular expression to match a table name
+    var regexp = pattern_prefix(old_table.schema, old_table_ident) +
+                 old_table_ident +
+                 pattern_suffix(old_table_ident);
+
+    // replace all occurrences of the table pattern
+    return sql.replace(new RegExp(regexp, 'g'), replacement);
+}
+
+function overviews_query(query, overviews, zoom_level_expression) {
+    var replaced_query = query;
+    var sql = "WITH\n  _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
+    var replacement;
+    for ( var table in overviews ) {
+        if (overviews.hasOwnProperty(table)) {
+            var table_overviews = overviews[table];
+            var table_view = overviews_view_name(table);
+            replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
+            replaced_query = replace_table_in_query(replaced_query, table, replacement);
+        }
+    }
+    if ( replaced_query !== query ) {
+        sql += "\n";
+        sql += replaced_query;
+    } else {
+        sql = query;
+    }
+    return sql;
+}
+
+// Transform an SQL query so that it uses overviews.
+// overviews contains metadata about the overviews to be used:
+//     { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
+//
+// For a given query `SELECT * FROM table`,  if any of tables in it
+// has overviews as defined by the provided metadat, the query will
+// be transform into something similar to this:
+//
+//     WITH _vovw_scale AS ( ... ), -- define scale level
+//     WITH _vovw_table AS ( ... ), -- define union of overviews and base table
+//     SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
+//
+// This transformation can in principle be applied to arbitrary queries
+// (except for the case of queries that include the name of tables with
+// overviews inside text literals: at the current table name substitution
+// doesnn't prevent substitution inside literals).
+// But the transformation will currently only be applied to simple queries
+// of the form detected by the overviews_supported_query function.
+OverviewsQueryRewriter.prototype.query = function(query, data) {
+    var overviews = this.overviews_metadata(data);
+    if ( !overviews || !this.is_supported_query(query)) {
+        return query;
+    }
+    var zoom_level_expression = this.options.zoom_level || '0';
+    return overviews_query(query, overviews, zoom_level_expression);
+};
+
+OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
+  return !!sql.match(
+    /^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
+  );
+};
+
+OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
+  return data && data.overviews;
+};

lib/cartodb/utils/style/postgres-datasource.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function createTemplate(method) {
+    return dot.template([
+        'SELECT',
+        method,
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
+    ].join('\n'));
+}
+
+var methods = {
+    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
+    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+};
+
+var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {
+    methodTemplates[methodName] = createTemplate(methods[methodName]);
+    return methodTemplates;
+}, {});
+
+function PostgresDatasource (pgQueryRunner, username, query) {
+    this.pgQueryRunner = pgQueryRunner;
+    this.username = username;
+    this.query = query;
+}
+
+PostgresDatasource.prototype.getName = function () {
+    return 'PostgresDatasource';
+};
+
+PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) {
+    var methodName = methods.hasOwnProperty(method) ? method : 'quantiles';
+    var template = methodTemplates[methodName];
+
+    var query = template({ _column: column, _sql: this.query, _buckets: buckets });
+
+    this.pgQueryRunner.run(this.username, query, function (err, result) {
+        if (err) {
+            return callback(err);
+        }
+
+        var ramp = result[0][methodName].sort(function(a, b) {
+            return a - b;
+        });
+
+        return callback(null, ramp);
+    });
+};
+
+module.exports = PostgresDatasource;

lib/cartodb/utils/style/turbo-cartocss-adapter.js

@@ -0,0 +1,56 @@
+'use strict';
+
+var queue = require('queue-async');
+
+function TurboCartocssAdapter(turboCartocssParser) {
+    this.turboCartocssParser = turboCartocssParser;
+}
+
+module.exports = TurboCartocssAdapter;
+
+TurboCartocssAdapter.prototype.getLayers = function (username, layers, callback) {
+    var self = this;
+
+    if (!layers || layers.length === 0) {
+        return callback(null, layers);
+    }
+
+    var parseCartoCssQueue = queue(layers.length);
+
+    layers.forEach(function(layer) {
+        parseCartoCssQueue.defer(self._parseCartoCss.bind(self), username, layer);
+    });
+
+    parseCartoCssQueue.awaitAll(function (err, layers) {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, layers);
+    });
+};
+
+TurboCartocssAdapter.prototype._parseCartoCss = function (username, layer, callback) {
+    if (isNotLayerToParseCartocss(layer)) {
+        return process.nextTick(function () {
+            callback(null, layer);
+        });
+    }
+
+    this.turboCartocssParser.process(username, layer.options.cartocss, layer.options.sql, function (err, cartocss) {
+        // Ignore turbo-cartocss errors and continue
+        if (!err && cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+
+        callback(null, layer);
+    });
+};
+
+function isNotLayerToParseCartocss(layer) {
+    if (!layer || !layer.options || !layer.options.cartocss || !layer.options.sql) {
+        return true;
+    }
+
+    return false;
+}

lib/cartodb/utils/style/turbo-cartocss-parser.js

@@ -0,0 +1,15 @@
+'use strict';
+
+var turboCartoCss = require('turbo-cartocss');
+var PostgresDatasource = require('./postgres-datasource');
+
+function TurboCartocssParser (pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = TurboCartocssParser;
+
+TurboCartocssParser.prototype.process = function (username, cartocss, sql, callback) {
+    var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql);
+    turboCartoCss(cartocss, datasource, callback);
+};

lib/cartodb/utils/table_name_parser.js

@@ -0,0 +1,106 @@
+// Quote an PostgreSQL identifier if ncecessary
+function quote_identifier_if_needed(txt) {
+    if ( txt && !txt.match(/^[a-z_][a-z_0-9]*$/)) {
+        return '"' + txt.replace(/\"/g, '""') + '"';
+    } else {
+        return txt;
+    }
+}
+
+// Parse PostgreSQL table name (possibly quoted and with optional schema).+
+// Returns { schema: 'schema_name', table: 'table_name' }
+function parse_table_name(table) {
+
+    function split_as_quoted_parts(table_name) {
+      // parse table into 'parts' that may be quoted, each part
+      // in the parts array being an object { part: 'text', quoted: false/true }
+     var parts = [];
+     var splitted = table_name.split(/\"/);
+     for (var i=0; i<splitted.length; i++ ) {
+       if ( splitted[i] === '' ) {
+         if ( parts.length > 0 && i < splitted.length-1 ) {
+             i++;
+             parts[parts.length - 1].part += '"' + splitted[i];
+         }
+       }
+       else {
+         var is_quoted = (i > 0 && splitted[i-1] === '') ||
+                         (i < splitted.length - 1 && splitted[i+1] === '');
+             parts.push({ part: splitted[i], quoted: is_quoted });
+       }
+     }
+     return parts;
+    }
+
+    var parts = split_as_quoted_parts(table);
+
+    function split_single_part(part) {
+        var schema_part = null;
+        var table_part = null;
+        if ( part.quoted ) {
+            table_part = part.part;
+        } else {
+            var parts = part.part.split('.');
+            if ( parts.length === 1 ) {
+                schema_part = null;
+                table_part = parts[0];
+            } else if ( parts.length === 2 ) {
+                schema_part = parts[0];
+                table_part = parts[1];
+            } // else invalid table name
+        }
+        return {
+            schema: schema_part,
+            table:  table_part
+        };
+    }
+
+    function split_two_parts(part1, part2) {
+        var schema_part = null;
+        var table_part = null;
+        if ( part1.quoted && !part2.quoted ) {
+            if ( part2.part[0] === '.' ) {
+                schema_part = part1.part;
+                table_part = part2.part.slice(1);
+            } // else invalid table name (missing dot)
+        } else if ( !part1.quoted && part2.quoted ) {
+            if ( part1.part[part1.part.length - 1] === '.' ) {
+                schema_part = part1.part.slice(0, -1);
+                table_part = part2.part;
+            } // else invalid table name (missing dot)
+        } // else invalid table name (missing dot)
+        return {
+            schema: schema_part,
+            table:  table_part
+        };
+    }
+
+    if ( parts.length === 1 ) {
+        return split_single_part(parts[0]);
+    } else if ( parts.length === 2 ) {
+        return split_two_parts(parts[0], parts[1]);
+    } else if ( parts.length === 3 && parts[1].part === '.' ) {
+        return {
+            schema: parts[0].part,
+            table:  parts[2].part
+        };
+    } // else invalid table name
+}
+
+function table_identifier(parsed_name) {
+    if ( parsed_name && parsed_name.table ) {
+        if ( parsed_name.schema ) {
+            return quote_identifier_if_needed(parsed_name.schema) + '.' + quote_identifier_if_needed(parsed_name.table);
+        } else {
+            return quote_identifier_if_needed(parsed_name.table);
+        }
+    } else {
+        return null;
+    }
+}
+
+module.exports = {
+  parse: parse_table_name,
+  quote: quote_identifier_if_needed,
+  table_identifier: table_identifier
+};

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.21.0",
+    "version": "2.28.0",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -26,7 +26,7 @@
         "node-statsd": "~0.0.7",
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "1.8.2",
+        "windshaft": "1.13.2",
         "step": "~0.0.6",
         "queue-async": "~1.0.7",
         "request": "~2.62.0",
@@ -36,7 +36,9 @@
         "redis-mpool": "~0.4.0",
         "lru-cache": "2.6.5",
         "lzma": "~1.3.7",
-        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
+        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
+        "cartodb-query-tables": "~0.1.0",
+        "turbo-cartocss": "0.4.0"
     },
     "devDependencies": {
         "istanbul": "~0.3.6",

test/acceptance/limits.js

@@ -292,7 +292,8 @@ describe('render limits', function() {
                                 if (err) {
                                     done(err);
                                 }
-                                assert.imageEqualsFile(res.body, './test/fixtures/render-timeout-fallback.png', 25,
+                                var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
+                                assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
                                     function(imgErr/*, similarity*/) {
                                         done(imgErr);
                                     }

test/acceptance/multilayer.js

@@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
+var QueryTables = require('cartodb-query-tables');
 
 ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
 
@@ -122,7 +122,7 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png',
+              assert.imageBufferIsSimilarToFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png',
                   IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
                       next(err);
                   }
@@ -190,38 +190,50 @@ describe(suiteName, function() {
     });
 
 
-    it("should include serverMedata in the response", function(done) {
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
-      var layergroup =  {
-        version: '1.0.0',
-        layers: [
-           { options: {
-               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
-                   ' from test_table limit 2',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
-               cartocss_version: '2.0.1'
-             } }
-        ]
-      };
+    describe('server-metadata', function() {
+        var serverMetadata;
+        beforeEach(function() {
+            serverMetadata = global.environment.serverMetadata;
+            global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
+        });
+
+        afterEach(function() {
+            global.environment.serverMetadata = serverMetadata;
+        });
+
+        it("should include serverMedata in the response", function(done) {
+          var layergroup =  {
+            version: '1.0.0',
+            layers: [
+               { options: {
+                   sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
+                       ' from test_table limit 2',
+                   cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+                   cartocss_version: '2.0.1'
+                 } }
+            ]
+          };
+
+          step(
+            function do_create_get()
+            {
+              var next = this;
+              assert.response(server, {
+                  url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+                  method: 'GET',
+                  headers: {host: 'localhost'}
+              }, {}, function(res, err) { next(err, res); });
+            },
+            function do_check_create(err, res) {
+              var parsed = JSON.parse(res.body);
+              keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
+              keysToDelete['user:localhost:mapviews:global'] = 5;
+              assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+              done();
+            }
+          );
+        });
 
-      step(
-        function do_create_get()
-        {
-          var next = this;
-          assert.response(server, {
-              url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
-              method: 'GET',
-              headers: {host: 'localhost'}
-          }, {}, function(res, err) { next(err, res); });
-        },
-        function do_check_create(err, res) {
-          var parsed = JSON.parse(res.body);
-          keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-          keysToDelete['user:localhost:mapviews:global'] = 5;
-          assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-          done();
-        }
-      );
     });
 
 
@@ -262,9 +274,9 @@ describe(suiteName, function() {
           var parsedBody = JSON.parse(res.body);
           expected_token = parsedBody.layergroupid.split(':')[0];
           helper.checkCache(res);
-          helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
-              'public.test_table',
-              'public.test_table_2'
+          helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
+            {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
+            {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
           ]).key().join(' '));
 
 
@@ -392,7 +404,8 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = 'test/fixtures/test_multilayer_bbox.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err/*, similarity*/) {
                   next(err);
               });
@@ -431,7 +444,8 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = 'test/fixtures/test_multilayer_bbox.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err/*, similarity*/) {
                   next(err);
               });
@@ -576,7 +590,7 @@ describe(suiteName, function() {
         version: '1.0.0',
         layers: [
            { options: {
-               sql: 'select 1 as cartodb_id, !pixel_height! as h' +
+               sql: 'select 1 as cartodb_id, !pixel_height! as h,' +
                    'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
                cartocss: '#layer { polygon-fit:red; }',
                cartocss_version: '2.0.1'
@@ -617,7 +631,7 @@ describe(suiteName, function() {
           headers: {host: 'localhost', 'Content-Type': 'application/json' },
           data: JSON.stringify(layergroup)
       }, {}, function(res) {
-          assert.equal(res.statusCode, 404, res.statusCode + ": " + res.body);
+          assert.equal(res.statusCode, 400, res.statusCode + ": " + res.body);
           var parsed = JSON.parse(res.body);
           var msg = parsed.errors[0];
           assert.ok(msg.match(/bogus.*exist/), msg);
@@ -1007,7 +1021,7 @@ describe(suiteName, function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
+              assert.imageBufferIsSimilarToFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
                       next(err);
                   }
@@ -1319,6 +1333,39 @@ describe(suiteName, function() {
         );
     });
 
+    it('should response to empty layers mapconfig', function(done) {
+        var layergroup =  {
+            layers: []
+        };
+
+        assert.response(
+            server,
+            {
+                url: '/api/v1/map',
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(layergroup)
+            },
+            {
+                status: 200
+            },
+            function(res, err) {
+                assert.ok(!err);
+
+                var parsedBody = JSON.parse(res.body);
+                assert.ok(parsedBody.layergroupid);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                done();
+            }
+        );
+    });
+
 });
 
 });

test/acceptance/multilayer_server.js

@@ -7,6 +7,7 @@ var _ = require('underscore');
 var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var QueryTables = require('cartodb-query-tables');
 var CartodbWindshaft = require('../../lib/cartodb/server');
 var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
@@ -332,13 +333,9 @@ describe('tests from old api translated to multilayer', function() {
 
                 assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
 
-                // TODO when affected tables query makes the request to fail layergroup should be removed
-                keysToDelete['map_cfg|4fb7bd7008322ce66f22d20aebba1ab0'] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
-
                 var parsed = JSON.parse(res.body);
                 assert.deepEqual(parsed, {
-                    errors: ["Error: could not fetch affected tables or last updated time: fake error message"]
+                    errors: ["fake error message"]
                 });
 
                 done();
@@ -364,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
                 keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
                 keysToDelete['user:localhost:mapviews:global'] = 5;
 
-                var runQueryFn = PgQueryRunner.prototype.run;
-                PgQueryRunner.prototype.run = function(username, query, callback) {
-                    return callback(new Error('failed to query database for affected tables'), []);
+                var affectedFn = QueryTables.getAffectedTablesFromQuery;
+                QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
+                    affectedFn({query: function(query, callback) {
+                        return callback(new Error('fake error message'), []);
+                    }}, username, query, callback);
                 };
 
                 // reset internal cacheChannel cache
@@ -391,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
                     },
                     function(res) {
                         assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
-                        PgQueryRunner.prototype.run = runQueryFn;
+                        QueryTables.getAffectedTablesFromQuery = affectedFn;
                         done();
                     }
                 );

test/acceptance/overviews_metadata.js

@@ -0,0 +1,110 @@
+var test_helper = require('../support/test_helper');
+
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+
+var RedisPool = require('redis-mpool');
+
+var step = require('step');
+
+var windshaft = require('windshaft');
+
+
+describe('overviews metadata', function() {
+    // configure redis pool instance to use in tests
+    var redisPool = new RedisPool(global.environment.redis);
+
+    var overviews_layer = {
+        type: 'cartodb',
+        options: {
+            sql: 'SELECT * FROM test_table_overviews',
+            cartocss: '#layer { marker-fill: black; }',
+            cartocss_version: '2.3.0'
+        }
+    };
+
+    var non_overviews_layer = {
+        type: 'cartodb',
+        options: {
+            sql: 'SELECT * FROM test_table',
+            cartocss: '#layer { marker-fill: black; }',
+            cartocss_version: '2.3.0'
+        }
+    };
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        test_helper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    it("layers with and without overviews", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [overviews_layer, non_overviews_layer]
+      };
+
+      var layergroup_url = '/api/v1/map';
+
+      var expected_token;
+      step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: layergroup_url,
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
+              expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_mapconfig(err)
+        {
+            assert.ifError(err);
+            var next = this;
+
+            var mapStore  = new windshaft.storage.MapStore({
+                pool: redisPool,
+                expire_time: 500000
+            });
+            mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) {
+              assert.ifError(err);
+              assert.deepEqual(non_overviews_layer, mapConfig._cfg.layers[1]);
+              assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
+              assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
+              var expected_data = {
+                overviews: {
+                  test_table_overviews: {
+                    1: { table: '_vovw_1_test_table_overviews' },
+                    2: { table: '_vovw_2_test_table_overviews' }
+                  }
+                }
+              };
+              assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
+            });
+
+            next(err);
+        },
+        function finish(err) {
+            keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
+            keysToDelete['user:localhost:mapviews:global'] = 5;
+            done(err);
+        }
+      );
+    });
+});

test/acceptance/overviews_queries.js

@@ -0,0 +1,73 @@
+var testHelper = require('../support/test_helper');
+var assert = require('../support/assert');
+
+var cartodbServer = require('../../lib/cartodb/server');
+var ServerOptions = require('./ported/support/ported_server_options');
+var testClient = require('./ported/support/test_client');
+var BaseController = require('../../lib/cartodb/controllers/base');
+
+describe('overviews_queries', function() {
+
+    var server = cartodbServer(ServerOptions);
+    server.setMaxListeners(0);
+
+    var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2;
+
+    var req2paramsFn;
+    before(function() {
+        req2paramsFn = BaseController.prototype.req2params;
+        BaseController.prototype.req2params = ServerOptions.req2params;
+    });
+
+    after(function() {
+        BaseController.prototype.req2params = req2paramsFn;
+
+        testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir);
+    });
+
+    function imageCompareFn(fixture, done) {
+        return function(err, tile) {
+            if (err) {
+                return done(err);
+            }
+            var referenceImagePath = './test/fixtures/' + fixture;
+            assert.imageBufferIsSimilarToFile(tile.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+        };
+    }
+
+    it("should not use overview for tables without overviews", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table'), 1, 0, 0,
+          imageCompareFn('test_table_1_0_0.png', done)
+      );
+    });
+
+    it("should not use overview for tables without overviews at z=2", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table'), 2, 1, 1,
+          imageCompareFn('test_table_2_1_1.png', done)
+      );
+    });
+
+    it("should not use overview for tables without overviews at z=2", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table'), 3, 3, 3,
+          imageCompareFn('test_table_3_3_3.png', done)
+      );
+    });
+
+    it("should use overview for zoom level 1", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 1, 0, 0,
+          imageCompareFn('_vovw_1_test_table_1_0_0.png', done)
+      );
+    });
+
+    it("should use overview for zoom level 1", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 2, 1, 1,
+          imageCompareFn('_vovw_2_test_table_2_1_1.png', done)
+      );
+    });
+
+    it("should not use overview for zoom level 3", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 3, 3, 3,
+          imageCompareFn('test_table_3_3_3.png', done)
+      );
+    });
+});

test/acceptance/ported/blend.js

@@ -95,10 +95,12 @@ describe('blend png renderer', function() {
         var zxy = [tileRequest.z, tileRequest.x, tileRequest.y];
         it('tile all/' + zxy.join('/') + '.png', function (done) {
             testClient.getTileLayer(plainTorqueMapConfig(testScenario.plainColor), tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(zxy), IMAGE_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(zxy), IMAGE_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/blend_filtering.js

@@ -156,10 +156,12 @@ describe('blend layer filtering', function() {
 
         it('should filter on ' + layerFilter + '/1/0/0.png', function (done) {
             testClient.getTileLayer(mapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/blend_http_fallback.js

@@ -111,10 +111,12 @@ describe('blend http fallback', function() {
 
         it('should fallback on http error while blending layers ' + layerFilter + '/1/0/0.png', function (done) {
             testClient.getTileLayer(mapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err, err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err, err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/external_resources.js

@@ -57,7 +57,8 @@ describe('external resources', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            var referenceImagePath = './test/fixtures/' + fixture;
+            assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
         };
     }
 

test/acceptance/ported/limits.js

@@ -84,11 +84,13 @@ describe.skip('render limits', function() {
         testClient.withLayergroup(slowQueryMapConfig, options, function(err, requestTile, finish) {
             var tileUrl = '/0/0/0.png';
             requestTile(tileUrl, options, function(err, res) {
-                assert.imageEqualsFile(res.body, fixtureImage, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
-                    finish(function(finishErr) {
-                        done(err || finishErr);
-                    });
-                });
+                assert.imageBufferIsSimilarToFile(res.body, fixtureImage, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                    function(err) {
+                        finish(function(finishErr) {
+                            done(err || finishErr);
+                        });
+                    }
+                );
             });
         });
     });

test/acceptance/ported/multilayer.js

@@ -117,7 +117,8 @@ describe('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
               checkCORSHeaders(res);
-              assert.imageEqualsFile(res.body, './test/fixtures/test_bigpoint_red.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = './test/fixtures/test_bigpoint_red.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                   function(err) {
                   next(err);
               });
@@ -191,7 +192,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -302,7 +304,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -425,7 +428,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -541,7 +545,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -727,7 +732,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer2.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer2.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -761,7 +767,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer3.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer3.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -856,7 +863,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer4.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer4.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -1259,7 +1267,7 @@ describe('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
               checkCORSHeaders(res);
-              assert.imageEqualsFile(res.body, './test/fixtures/test_bigpoint_red.png',
+              assert.imageBufferIsSimilarToFile(res.body, './test/fixtures/test_bigpoint_red.png',
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });

test/acceptance/ported/raster.js

@@ -87,7 +87,7 @@ describe('raster', function() {
           assert.equal(res.statusCode, 200, res.body);
           assert.deepEqual(res.headers['content-type'], "image/png");
           var next = this;
-          assert.imageEqualsFile(res.body,
+          assert.imageBufferIsSimilarToFile(res.body,
             './test/fixtures/raster_gray_rect.png',
             IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
               try {

test/acceptance/ported/server_gettile.js

@@ -34,7 +34,9 @@ describe('server_gettile', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            assert.imageBufferIsSimilarToFile(
+                res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done
+            );
         };
     }
 
@@ -113,12 +115,13 @@ describe('server_gettile', function() {
                 assert.ok(res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time");
                 assert.ok(res.headers['x-windshaft-cache'] >= 0);
 
-                assert.imageEqualsFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
-
-                    finish(function(finishErr) {
-                        done(err || finishErr);
-                    });
-                });
+                assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                    function(err) {
+                        finish(function(finishErr) {
+                            done(err || finishErr);
+                        });
+                    }
+                );
             });
         });
     });

test/acceptance/ported/server_png8_format.js

@@ -108,7 +108,7 @@ describe('server_png8_format', function() {
                             assert.equal(responsePng8.headers['content-type'], "image/png");
                             bufferPng8 = responsePng8.body;
                             assert.ok(bufferPng8.length < bufferPng32.length);
-                            assert.imageBuffersAreEqual(bufferPng32, bufferPng8, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                            assert.imageBuffersAreSimilar(bufferPng32, bufferPng8, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                                 function(err, imagePaths, similarity) {
 
                                     keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;

test/acceptance/ported/support/ported_server_options.js

@@ -2,6 +2,10 @@ var _ = require('underscore');
 var serverOptions = require('../../../../lib/cartodb/server_options');
 var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token');
 var mapnik = require('windshaft').mapnik;
+var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter');
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+  zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
+});
 
 module.exports = _.extend({}, serverOptions, {
     base_url: '/database/:dbname/table/:table',
@@ -26,7 +30,8 @@ module.exports = _.extend({}, serverOptions, {
             limits: {
                 render: 0,
                 cacheOnTimeout: true
-            }
+            },
+            queryRewriter: overviewsQueryRewriter
         },
         http: {
             timeout: 5000,

test/acceptance/ported/torque_png.js

@@ -83,10 +83,12 @@ describe('torque png renderer', function() {
         var zxy = [z, x, y];
         it('tile ' + zxy.join('/') + '.torque.png', function (done) {
             testClient.getTileLayer(torquePngPointsMapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, torquePngFixture(zxy), IMAGE_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, torquePngFixture(zxy), IMAGE_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/wrap.js

@@ -107,10 +107,12 @@ describe('wrap x coordinate', function() {
             var fixtureZxy = [testScenario.fixture.z, testScenario.fixture.x, testScenario.fixture.y];
             it('tile all/' + zxy.join('/') + '.png', function (done) {
                 testClient.getTileLayer(plainTorqueMapConfig(testScenario.plainColor), tileRequest, function(err, res) {
-                    assert.imageEqualsFile(res.body, blendPngFixture(fixtureZxy), IMG_TOLERANCE_PER_MIL, function(err) {
-                        assert.ok(!err);
-                        done();
-                    });
+                    assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(fixtureZxy), IMG_TOLERANCE_PER_MIL,
+                        function(err) {
+                            assert.ok(!err);
+                            done();
+                        }
+                    );
                 });
             });
         });

test/acceptance/templates.js

@@ -3,8 +3,8 @@ var _           = require('underscore');
 var redis       = require('redis');
 var step        = require('step');
 var strftime    = require('strftime');
+var QueryTables = require('cartodb-query-tables');
 var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
 var redis_stats_db = 5;
 
 // Pollute the PG environment to make sure
@@ -313,51 +313,63 @@ describe('template_api', function() {
       });
     });
 
-    it("instance endpoint should return server metadata", function(done){
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
-      var tmpl = _.clone(template_acceptance1);
-      tmpl.name = "rambotemplate2";
+    describe('server-metadata', function() {
+        var serverMetadata;
+        beforeEach(function() {
+            serverMetadata = global.environment.serverMetadata;
+            global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
+        });
 
-      step(function postTemplate1() {
-          var next = this;
-          var post_request = {
-              url: '/api/v1/map/named?api_key=1234',
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' },
-              data: JSON.stringify(tmpl)
-          };
-          assert.response(server, post_request, {}, function(res) {
-            next(null, res);
-          });
-        },
-        function testCORS() {
-          var next = this;
-          assert.response(server, {
-              url: '/api/v1/map/named/' + tmpl.name,
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          },{
-              status: 200
-          }, function(res) {
-            var parsed = JSON.parse(res.body);
-            keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-            keysToDelete['user:localhost:mapviews:global'] = 5;
-            assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-            next(null);
-          });
-        },
-        function deleteTemplate(err) {
-            assert.ifError(err);
-          var del_request = {
-              url: '/api/v1/map/named/' + tmpl.name + '?api_key=1234',
-              method: 'DELETE',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          };
-          assert.response(server, del_request, {}, function() {
-              done();
-          });
-        }
-      );
+        afterEach(function() {
+            global.environment.serverMetadata = serverMetadata;
+        });
+
+
+        it("instance endpoint should return server metadata", function(done){
+          var tmpl = _.clone(template_acceptance1);
+          tmpl.name = "rambotemplate2";
+
+          step(function postTemplate1() {
+              var next = this;
+              var post_request = {
+                  url: '/api/v1/map/named?api_key=1234',
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' },
+                  data: JSON.stringify(tmpl)
+              };
+              assert.response(server, post_request, {}, function(res) {
+                next(null, res);
+              });
+            },
+            function testCORS() {
+              var next = this;
+              assert.response(server, {
+                  url: '/api/v1/map/named/' + tmpl.name,
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' }
+              },{
+                  status: 200
+              }, function(res) {
+                var parsed = JSON.parse(res.body);
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+                next(null);
+              });
+            },
+            function deleteTemplate(err) {
+                assert.ifError(err);
+              var del_request = {
+                  url: '/api/v1/map/named/' + tmpl.name + '?api_key=1234',
+                  method: 'DELETE',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' }
+              };
+              assert.response(server, del_request, {}, function() {
+                  done();
+              });
+            }
+          );
+        });
     });
 
 
@@ -1393,7 +1405,8 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-              new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+                                   table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
           helper.checkSurrogateKey(res, expectedSurrogateKey);
@@ -1476,7 +1489,8 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-              new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+                                   table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
           helper.checkSurrogateKey(res, expectedSurrogateKey);

test/acceptance/turbo-cartocss-anonymous-maps.js

@@ -0,0 +1,128 @@
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+var IMAGE_TOLERANCE_PER_MIL = 20;
+
+function imageCompareFn(fixture, done) {
+    return function(err, res, image) {
+        assert.ok(!err, err);
+        assert.imageIsSimilarToFile(image, './test/fixtures/' + fixture, IMAGE_TOLERANCE_PER_MIL, done);
+    };
+}
+
+function makeMapconfig(cartocss) {
+    return {
+        "version": "1.4.0",
+        "layers": [
+            {
+                "type": 'mapnik',
+                "options": {
+                    "cartocss_version": '2.3.0',
+                    "sql": [
+                        'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
+                        '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                        '  UNION',
+                        '  SELECT 2, 10.50',
+                        '  UNION',
+                        '  SELECT 3, 11.00',
+                        '  UNION',
+                        '  SELECT 4, 12.00',
+                        '  UNION',
+                        '  SELECT 5, 21.00',
+                        ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
+                    ].join('\n'),
+                    "cartocss": cartocss
+                }
+            }
+        ]
+    };
+}
+
+describe('turbo-cartocss for anonymous maps', function() {
+    describe('parsing ramp function with colorbrewer for greens and mapnik renderer', function () {
+        beforeEach(function () {
+            var turboCartocss = '#layer { marker-fill: ramp([price], colorbrewer(Greens)); }';
+            this.testClient = new TestClient(makeMapconfig(turboCartocss));
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+
+        it('should get a tile with turbo-cartocss parsed properly', function (done) {
+            var fixturePath = 'test_turbo_cartocss_greens_13_4011_3088.png';
+            this.testClient.getTile(13, 4011, 3088, imageCompareFn(fixturePath, done));
+        });
+    });
+
+    describe('parsing ramp function with colorbrewer for reds and mapnik renderer', function () {
+        beforeEach(function () {
+            var turboCartocss = '#layer { marker-fill: ramp([price], colorbrewer(Reds)); }';
+            this.testClient = new TestClient(makeMapconfig(turboCartocss));
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+
+        it('should get a tile with turbo-cartocss parsed properly', function (done) {
+            var fixtureFileName = 'test_turbo_cartocss_reds_13_4011_3088.png';
+            this.testClient.getTile(13, 4011, 3088, imageCompareFn(fixtureFileName, done));
+        });
+    });
+
+    describe('parsing ramp function with colorbrewer for greens and toque renderer', function () {
+        var mapConfig = {
+            version: '1.2.0',
+            layers: [
+                {
+                    type: 'torque',
+                    options: {
+                        sql: "SELECT * FROM populated_places_simple_reduced where the_geom" +
+                            " && ST_MakeEnvelope(-90, 0, 90, 65)",
+                        cartocss: [
+                            'Map {',
+                            '    buffer-size:0;',
+                            '    -torque-frame-count:1;',
+                            '    -torque-animation-duration:30;',
+                            '    -torque-time-attribute:"cartodb_id";',
+                            '    -torque-aggregation-function:"count(cartodb_id)";',
+                            '    -torque-resolution:1;',
+                            '    -torque-data-aggregation:linear;',
+                            '};',
+                            '#populated_places_simple_reduced {',
+                            '    comp-op: multiply;',
+                            '    marker-fill-opacity: 1;',
+                            '    marker-line-color: #FFF;',
+                            '    marker-line-width: 0;',
+                            '    marker-line-opacity: 1;',
+                            '    marker-type: rectangle;',
+                            '    marker-width: 3;',
+                            '    marker-fill: ramp([pop_max], colorbrewer(Greens));',
+                            '};'
+                        ].join(' '),
+                        cartocss_version: '2.3.0'
+                    }
+                }
+            ]
+        };
+
+        beforeEach(function () {
+            this.testClient = new TestClient(mapConfig);
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+
+        it('should get a tile with turbo-cartocss parsed properly', function (done) {
+            var z = 2;
+            var x = 2;
+            var y = 1;
+
+            var pngFixture = 'torque/populated_places_simple_reduced-turbo-cartocss-' + [z, x, y].join('.') + '.png';
+
+            this.testClient.getTile(z, x, y, { layers: 0, format: 'torque.png' }, imageCompareFn(pngFixture, done));
+        });
+    });
+});

test/acceptance/turbo-cartocss-named-maps.js

@@ -0,0 +1,236 @@
+var assert = require('../support/assert');
+var step = require('step');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var testHelper = require(__dirname + '/../support/test_helper');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+var mapnik = require('windshaft').mapnik;
+var IMAGE_TOLERANCE_PER_MIL = 10;
+
+describe('turbo-cartocss for named maps', function() {
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        testHelper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    var templateId = 'turbo-cartocss-template-1';
+
+    var template = {
+        version: '0.0.1',
+        name: templateId,
+        auth: { method: 'open' },
+        placeholders: {
+            color: {
+                type: "css_color",
+                default: "Reds"
+            }
+        },
+        layergroup:  {
+            version: '1.0.0',
+                layers: [{
+                    options: {
+                        sql: [
+                            'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
+                            '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                            '  UNION',
+                            '  SELECT 2, 10.50',
+                            '  UNION',
+                            '  SELECT 3, 11.00',
+                            '  UNION',
+                            '  SELECT 4, 12.00',
+                            '  UNION',
+                            '  SELECT 5, 21.00',
+                            ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
+                        ].join('\n'),
+                        cartocss: [
+                            '#layer {',
+                            '  marker-fill: ramp([price], colorbrewer(<%= color %>));',
+                            '  marker-allow-overlap:true;',
+                            '}'
+                        ].join('\n'),
+                        cartocss_version: '2.0.2'
+                    }
+                }
+            ]
+        }
+    };
+
+    var templateParamsReds = { color: 'Reds' };
+    var templateParamsBlues = { color: 'Blues' };
+
+    it('should create a template with turbo-cartocss parsed properly', function (done) {
+        step(
+            function postTemplate() {
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/named?api_key=1234',
+                    method: 'POST',
+                    headers: { host: 'localhost', 'Content-Type': 'application/json' },
+                    data: JSON.stringify(template)
+                }, {},
+                function (res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTemplate(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 200);
+                assert.deepEqual(JSON.parse(res.body), {
+                    template_id: templateId
+                });
+
+                return null;
+            },
+            function instantiateTemplateWithReds(err) {
+                assert.ifError(err);
+
+                var next = this;
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(templateParamsReds)
+                }, {},
+                function(res, err) {
+                    return next(err, res);
+                });
+            },
+            function checkInstanciationWithReds(err, res) {
+                assert.ifError(err);
+
+                assert.equal(res.statusCode, 200);
+
+                var parsedBody = JSON.parse(res.body);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                assert.ok(parsedBody.layergroupid);
+                assert.ok(parsedBody.last_updated);
+
+                return parsedBody.layergroupid;
+            },
+            function requestTileReds(err, layergroupId) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
+                    method: 'GET',
+                    headers: { host: 'localhost' },
+                    encoding: 'binary'
+                }, {},
+                function(res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTileReds(err, res) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.equal(res.statusCode, 200);
+                assert.equal(res.headers['content-type'], 'image/png');
+
+                var fixturePath = './test/fixtures/turbo-cartocss-named-maps-reds.png';
+                var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+
+                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
+            },
+            function instantiateTemplateWithBlues(err) {
+                assert.ifError(err);
+
+                var next = this;
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(templateParamsBlues)
+                }, {},
+                function(res, err) {
+                    return next(err, res);
+                });
+            },
+            function checkInstanciationWithBlues(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 200);
+
+                var parsedBody = JSON.parse(res.body);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                assert.ok(parsedBody.layergroupid);
+                assert.ok(parsedBody.last_updated);
+
+                return parsedBody.layergroupid;
+            },
+            function requestTileBlues(err, layergroupId) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
+                    method: 'GET',
+                    headers: { host: 'localhost' },
+                    encoding: 'binary'
+                }, {},
+                function(res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTileBlues(err, res) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.equal(res.statusCode, 200);
+                assert.equal(res.headers['content-type'], 'image/png');
+
+                var fixturePath = './test/fixtures/turbo-cartocss-named-maps-blues.png';
+                var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+
+                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
+            },
+            function deleteTemplate(err) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId + '?api_key=1234',
+                    method: 'DELETE',
+                    headers: { host: 'localhost' }
+                }, {}, function (res, err) {
+                    next(err, res);
+                });
+            },
+            function checkDeleteTemplate(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 204);
+                assert.ok(!res.body);
+
+                return null;
+            },
+            function finish(err) {
+                done(err);
+            }
+        );
+    });
+});

test/acceptance/turbo-cartocss-regressions.js

@@ -0,0 +1,75 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+function makeMapconfig(cartocss) {
+    return {
+        "version": "1.4.0",
+        "layers": [
+            {
+                "type": 'mapnik',
+                "options": {
+                    "cartocss_version": '2.3.0',
+                    "sql": [
+                        'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
+                        '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                        '  UNION',
+                        '  SELECT 2, 10.50',
+                        '  UNION',
+                        '  SELECT 3, 11.00',
+                        '  UNION',
+                        '  SELECT 4, 12.00',
+                        '  UNION',
+                        '  SELECT 5, 21.00',
+                        ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
+                    ].join('\n'),
+                    "cartocss": cartocss
+                }
+            }
+        ]
+    };
+}
+
+describe('turbo-cartocss regressions', function() {
+
+    var cartocss = [
+        "/** simple visualization */",
+        "",
+        "Map {",
+        "  buffer-size: 256;",
+        "}",
+        "",
+        "#county_points_with_population{",
+        "  marker-fill-opacity: 0.1;",
+        "  marker-line-color:#FFFFFF;//#CF1C90;",
+        "  marker-line-width: 0;",
+        "  marker-line-opacity: 0.3;",
+        "  marker-placement: point;",
+        "  marker-type: ellipse;",
+        "  //marker-comp-op: overlay;",
+        "  marker-width: [price];",
+        "  [zoom=5]{marker-width: [price]*2;}",
+        "  [zoom=6]{marker-width: [price]*4;}",
+        "  marker-fill: #000000;",
+        "  marker-allow-overlap: true;",
+        "  ",
+        "",
+        "}"
+    ].join('\n');
+
+    beforeEach(function () {
+        this.testClient = new TestClient(makeMapconfig(cartocss));
+    });
+
+    afterEach(function (done) {
+        this.testClient.drain(done);
+    });
+
+    it('should accept // comments', function(done) {
+        this.testClient.getTile(0, 0, 0, function(err) {
+            assert.ok(!err, err);
+            done();
+        });
+    });
+});

test/acceptance/widgets.js

@@ -1,479 +0,0 @@
-var assert = require('../support/assert');
-var step = require('step');
-var qs = require('querystring');
-
-var helper = require(__dirname + '/../support/test_helper');
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
-
-var CartodbWindshaft = require('../../lib/cartodb/server');
-var serverOptions = require('../../lib/cartodb/server_options');
-var server = new CartodbWindshaft(serverOptions);
-server.setMaxListeners(0);
-
-
-describe('widgets', function() {
-
-    var keysToDelete;
-
-    beforeEach(function() {
-        keysToDelete = {};
-    });
-
-    afterEach(function(done) {
-        helper.deleteRedisKeys(keysToDelete, done);
-    });
-
-    function getWidget(mapConfig, widgetName, params, callback) {
-        if (!callback) {
-            callback = params;
-            params = {};
-        }
-
-        var url = '/api/v1/map';
-        if (params && params.filters) {
-            url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
-        }
-
-        var layergroupId;
-        step(
-            function createLayergroup() {
-                var next = this;
-                assert.response(server,
-                    {
-                        url: url,
-                        method: 'POST',
-                        headers: {
-                            host: 'localhost',
-                            'Content-Type': 'application/json'
-                        },
-                        data: JSON.stringify(mapConfig)
-                    },
-                    {
-                        status: 200,
-                        headers: {
-                            'Content-Type': 'application/json; charset=utf-8'
-                        }
-                    },
-                    function(res, err) {
-                        if (err) {
-                            return next(err);
-                        }
-                        var parsedBody = JSON.parse(res.body);
-                        var expectedWidgetURLS = {
-                            http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
-                        };
-                        assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
-                        assert.ok(
-                            parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
-                        );
-                        return next(null, parsedBody.layergroupid);
-                    }
-                );
-            },
-            function getWidgetResult(err, _layergroupId) {
-                assert.ifError(err);
-
-                var next = this;
-                layergroupId = _layergroupId;
-
-                var urlParams = {
-                    own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
-                };
-                if (params && params.bbox) {
-                    urlParams.bbox = params.bbox;
-                }
-                url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
-
-                assert.response(server,
-                    {
-                        url: url,
-                        method: 'GET',
-                        headers: {
-                            host: 'localhost'
-                        }
-                    },
-                    {
-                        status: 200,
-                        headers: {
-                            'Content-Type': 'application/json; charset=utf-8'
-                        }
-                    },
-                    function(res, err) {
-                        if (err) {
-                            return next(err);
-                        }
-
-                        next(null, res);
-                    }
-                );
-            },
-            function finish(err, res) {
-                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
-                return callback(err, res);
-            }
-        );
-    }
-
-
-    it("should expose layer list", function(done) {
-
-        var listWidgetMapConfig =  {
-            version: '1.5.0',
-            layers: [
-                {
-                    type: 'mapnik',
-                    options: {
-                        sql: 'select * from test_table',
-                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                        cartocss_version: '2.3.0',
-                        widgets: {
-                            names: {
-                                type: 'list',
-                                options: {
-                                    columns: ['name']
-                                }
-                            }
-                        }
-                    }
-                }
-            ]
-        };
-
-        getWidget(listWidgetMapConfig, 'names', function(err, res) {
-            if (err) {
-                return done(err);
-            }
-
-            var expectedList = [
-                {name:"Hawai"},
-                {name:"El Estocolmo"},
-                {name:"El Rey del Tallarín"},
-                {name:"El Lacón"},
-                {name:"El Pico"}
-            ];
-            assert.deepEqual(JSON.parse(res.body).rows, expectedList);
-
-            done();
-        });
-    });
-
-    it("should expose layer histogram", function(done) {
-        var histogramMapConfig =  {
-            version: '1.5.0',
-            layers: [
-                {
-                    type: 'mapnik',
-                    options: {
-                        sql: 'select * from populated_places_simple_reduced',
-                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                        cartocss_version: '2.3.0',
-                        widgets: {
-                            pop_max: {
-                                type: 'histogram',
-                                options: {
-                                    column: 'pop_max'
-                                }
-                            }
-                        }
-                    }
-                }
-            ]
-        };
-        getWidget(histogramMapConfig, 'pop_max', function(err, res) {
-            if (err) {
-                return done(err);
-            }
-
-            var histogram = JSON.parse(res.body);
-            assert.ok(histogram.bins.length);
-
-            done();
-        });
-    });
-
-    describe('filters', function() {
-
-        describe('category', function() {
-            var aggregationMapConfig =  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        type: 'mapnik',
-                        options: {
-                            sql: 'select * from populated_places_simple_reduced',
-                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                            cartocss_version: '2.3.0',
-                            widgets: {
-                                country_places_count: {
-                                    type: 'aggregation',
-                                    options: {
-                                        column: 'adm0_a3',
-                                        aggregation: 'count'
-                                    }
-                                }
-                            }
-                        }
-                    }
-                ]
-            };
-
-            it("should expose an aggregation", function(done) {
-                getWidget(aggregationMapConfig, 'country_places_count', { own_filter: 0 }, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-                    assert.equal(aggregation.categories.length, 6);
-                    assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false });
-
-                    done();
-                });
-            });
-
-            it("should expose a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {country_places_count: {accept: ['CAN']}}
-                        ]
-                    }
-                };
-                getWidget(aggregationMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-                    assert.equal(aggregation.categories.length, 1);
-                    assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false });
-
-                    done();
-                });
-            });
-        });
-
-        describe('range', function() {
-            var histogramMapConfig =  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        type: 'mapnik',
-                        options: {
-                            sql: 'select * from populated_places_simple_reduced',
-                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                            cartocss_version: '2.3.0',
-                            widgets: {
-                                country_places_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'pop_max'
-                                    }
-                                }
-                            }
-                        }
-                    }
-                ]
-            };
-
-            it("should expose an histogram", function(done) {
-                getWidget(histogramMapConfig, 'country_places_histogram', { own_filter: 0 }, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var histogram = JSON.parse(res.body);
-                    // notice min value
-                    assert.deepEqual(
-                        histogram.bins[0],
-                        { bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 }
-                    );
-
-                    done();
-                });
-            });
-
-            it("should expose a filtered histogram", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_histogram: { min: 4000000 }
-                            }
-                        ]
-                    }
-                };
-                getWidget(histogramMapConfig, 'country_places_histogram', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var histogram = JSON.parse(res.body);
-                    // notice min value
-                    assert.deepEqual(histogram.bins[0], {
-                        bin: 0,
-                        freq: 62,
-                        min: 4000000,
-                        max: 9276403,
-                        avg: 5815009.596774193
-                    });
-
-                    done();
-                });
-            });
-        });
-
-        describe('combine widget filters', function() {
-            var combinedWidgetsMapConfig =  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        type: 'mapnik',
-                        options: {
-                            sql: 'select * from populated_places_simple_reduced',
-                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                            cartocss_version: '2.3.0',
-                            widgets: {
-                                country_places_count: {
-                                    type: 'aggregation',
-                                    options: {
-                                        column: 'adm0_a3',
-                                        aggregation: 'count'
-                                    }
-                                },
-                                country_places_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'pop_max'
-                                    }
-                                }
-                            }
-                        }
-                    }
-                ]
-            };
-
-            it("should expose a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_count: { reject: ['CHN'] }
-                            }
-                        ]
-                    }
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-
-            it("should expose a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_count: { reject: ['CHN'] },
-                                country_places_histogram: { min: 7000000 }
-                            }
-                        ]
-                    }
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-
-            it("should allow to filter by bounding box a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_histogram: { min: 50000 }
-                            }
-                        ]
-                    },
-                    bbox: '-20,0,45,60'
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-
-            it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_count: { reject: ['RUS'] },
-                                country_places_histogram: { min: 50000 }
-                            }
-                        ]
-                    },
-                    bbox: '-20,0,45,60'
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-        });
-    });
-
-});

test/acceptance/widgets/aggregation.js

@@ -0,0 +1,74 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('aggregation widgets', function() {
+
+    var aggregationMapConfig =  {
+        version: '1.5.0',
+        layers: [
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                    cartocss_version: '2.3.0',
+                    widgets: {
+                        country_places_count: {
+                            type: 'aggregation',
+                            options: {
+                                column: 'adm0_a3',
+                                aggregation: 'count'
+                            }
+                        }
+                    }
+                }
+            }
+        ]
+    };
+
+    it("should expose an aggregation", function(done) {
+        var testClient = new TestClient(aggregationMapConfig);
+        testClient.getWidget('country_places_count', { own_filter: 0 }, function(err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            var aggregation = JSON.parse(res.body);
+            assert.equal(aggregation.categories.length, 6);
+            assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false });
+
+            testClient.drain(done);
+        });
+    });
+
+    describe('filters', function() {
+
+        describe('category', function () {
+
+            it("should expose a filtered aggregation", function (done) {
+                var params = {
+                    filters: {
+                        layers: [
+                            {country_places_count: {accept: ['CAN']}}
+                        ]
+                    }
+                };
+                var testClient = new TestClient(aggregationMapConfig);
+                testClient.getWidget('country_places_count', params, function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var aggregation = JSON.parse(res.body);
+                    assert.equal(aggregation.categories.length, 1);
+                    assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false });
+
+                    testClient.drain(done);
+                });
+            });
+        });
+
+    });
+});

test/acceptance/widgets/histogram.js

@@ -0,0 +1,120 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('histogram widgets', function() {
+
+    it("should expose layer histogram", function(done) {
+        var histogramMapConfig =  {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            pop_max: {
+                                type: 'histogram',
+                                options: {
+                                    column: 'pop_max'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        var testClient = new TestClient(histogramMapConfig);
+
+        testClient.getWidget('pop_max', function(err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            var histogram = JSON.parse(res.body);
+            assert.ok(histogram.bins.length);
+
+            testClient.drain(done);
+        });
+    });
+
+    describe('filters', function() {
+
+        describe('range', function() {
+            var histogramMapConfig =  {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                            cartocss_version: '2.3.0',
+                            widgets: {
+                                country_places_histogram: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            };
+
+            it("should expose an histogram", function(done) {
+                var testClient = new TestClient(histogramMapConfig);
+                testClient.getWidget('country_places_histogram', { own_filter: 0 }, function(err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var histogram = JSON.parse(res.body);
+                    // notice min value
+                    assert.deepEqual(
+                        histogram.bins[0],
+                        { bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 }
+                    );
+
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered histogram", function(done) {
+                var params = {
+                    filters: {
+                        layers: [
+                            {
+                                country_places_histogram: { min: 4000000 }
+                            }
+                        ]
+                    }
+                };
+                var testClient = new TestClient(histogramMapConfig);
+                testClient.getWidget('country_places_histogram', params, function(err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var histogram = JSON.parse(res.body);
+                    // notice min value
+                    assert.deepEqual(histogram.bins[0], {
+                        bin: 0,
+                        freq: 62,
+                        min: 4000000,
+                        max: 9276403,
+                        avg: 5815009.596774193
+                    });
+
+                    testClient.drain(done);
+                });
+            });
+        });
+    });
+
+});

test/acceptance/widgets/list.js

@@ -0,0 +1,51 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('list widgets', function() {
+
+    it("should expose layer list", function(done) {
+
+        var listWidgetMapConfig =  {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from test_table',
+                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            names: {
+                                type: 'list',
+                                options: {
+                                    columns: ['name']
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        var testClient = new TestClient(listWidgetMapConfig);
+
+        testClient.getWidget('names', function(err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            var expectedList = [
+                {name:"Hawai"},
+                {name:"El Estocolmo"},
+                {name:"El Rey del Tallarín"},
+                {name:"El Lacón"},
+                {name:"El Pico"}
+            ];
+            assert.deepEqual(JSON.parse(res.body).rows, expectedList);
+
+            testClient.drain(done);
+        });
+    });
+});

test/acceptance/widgets/named-maps.js

@@ -0,0 +1,250 @@
+var assert = require('../../support/assert');
+var step = require('step');
+
+var url = require('url');
+var queue = require('queue-async');
+
+var helper = require('../../support/test_helper');
+
+var CartodbWindshaft = require('../../../lib/cartodb/server');
+var serverOptions = require('../../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+
+describe('named-maps widgets', function() {
+
+    var username = 'localhost';
+    var widgetsTemplateName = 'widgets-template';
+
+    var layergroupid;
+    var layergroup;
+    var keysToDelete;
+
+    beforeEach(function(done) {
+        keysToDelete = {};
+
+        var widgetsTemplate =  {
+            version: '0.0.1',
+            name: widgetsTemplateName,
+            layergroup:  {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: "select * from populated_places_simple_reduced_private",
+                            cartocss: '#layer { marker-fill: blue; }',
+                            cartocss_version: '2.3.0',
+                            widgets: {
+                                pop_max_formula_sum: {
+                                    type: 'formula',
+                                    options: {
+                                        column: 'pop_max',
+                                        operation: 'sum'
+                                    }
+                                },
+                                country_places_count: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm0_a3',
+                                        aggregation: 'count'
+                                    }
+                                },
+                                pop_max: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        };
+
+        var template_params = {};
+
+        step(
+            function createTemplate()
+            {
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named?api_key=1234',
+                        method: 'POST',
+                        headers: {
+                            host: username,
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(widgetsTemplate)
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res, err) {
+                        next(err, res);
+                    }
+                );
+            },
+            function instantiateTemplate(err, res) {
+                assert.ifError(err);
+
+                assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName });
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + widgetsTemplateName,
+                        method: 'POST',
+                        headers: {
+                            host: username,
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(template_params)
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        next(null, res);
+                    }
+                );
+            },
+            function fetchTile(err, res) {
+                assert.ifError(err);
+
+                layergroup = JSON.parse(res.body);
+                assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body);
+                layergroupid = layergroup.layergroupid;
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                return done();
+            }
+        );
+
+    });
+
+    afterEach(function(done) {
+        step(
+            function deleteTemplate(err) {
+                assert.ifError(err);
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234',
+                        method: 'DELETE',
+                        headers: {
+                            host: username
+                        }
+                    },
+                    {
+                        status: 204
+                    },
+                    function(res, err) {
+                        next(err, res);
+                    }
+                );
+            },
+            function deleteRedisKeys(err) {
+                assert.ifError(err);
+                helper.deleteRedisKeys(keysToDelete, done);
+            }
+        );
+    });
+
+    function getWidget(widgetName, callback) {
+        assert.response(
+            server,
+            {
+                url: '/api/v1/map/' + layergroupid + '/0/widget/' + widgetName,
+                method: 'GET',
+                headers: {
+                    host: username
+                }
+            },
+            {
+                status: 200
+            },
+            function(res, err) {
+                if (err) {
+                    return callback(err);
+                }
+                var parsedBody = JSON.parse(res.body);
+                return callback(err, res, parsedBody);
+            }
+        );
+    }
+
+    it('should be able to retrieve widgets from all URLs', function(done) {
+        var widgetsPaths = layergroup.metadata.layers.reduce(function(paths, layer) {
+            var widgets = layer.widgets || {};
+            Object.keys(widgets).forEach(function(widget) {
+                paths.push(url.parse(widgets[widget].url.http).path);
+            });
+
+            return paths;
+        }, []);
+
+        var widgetsQueue = queue(widgetsPaths.length);
+
+        widgetsPaths.forEach(function(path) {
+            widgetsQueue.defer(function(path, done) {
+                assert.response(
+                    server,
+                    {
+                        url: path,
+                        method: 'GET',
+                        headers: {
+                            host: username
+                        }
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res, err) {
+                        if (err) {
+                            return done(err);
+                        }
+                        var parsedBody = JSON.parse(res.body);
+                        return done(null, parsedBody);
+                    }
+                );
+            }, path);
+        });
+
+        widgetsQueue.awaitAll(function(err, results) {
+            assert.equal(results.length, 3);
+            done(err);
+        });
+    });
+
+
+    it("should retrieve aggregation", function(done) {
+        getWidget('country_places_count', function(err, response, aggregation) {
+            assert.ok(!err, err);
+
+            assert.equal(aggregation.type, 'aggregation');
+            assert.equal(aggregation.max, 769);
+
+            return done();
+        });
+    });
+
+    it("should retrieve histogram", function(done) {
+        getWidget('pop_max', function(err, response, histogram) {
+            assert.ok(!err, err);
+
+            assert.equal(histogram.type, 'histogram');
+            assert.equal(histogram.bin_width, 743250);
+
+            return done();
+        });
+    });
+
+});

test/acceptance/widgets/widgets.js

@@ -0,0 +1,162 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('widget filters', function() {
+
+    describe('combine widget filters', function() {
+        var combinedWidgetsMapConfig =  {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            country_places_count: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0_a3',
+                                    aggregation: 'count'
+                                }
+                            },
+                            country_places_histogram: {
+                                type: 'histogram',
+                                options: {
+                                    column: 'pop_max'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        it("should expose a filtered aggregation", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_count: { reject: ['CHN'] }
+                        }
+                    ]
+                }
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+
+        it("should expose a filtered aggregation", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_count: { reject: ['CHN'] },
+                            country_places_histogram: { min: 7000000 }
+                        }
+                    ]
+                }
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+
+        it("should allow to filter by bounding box a filtered aggregation", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_histogram: { min: 50000 }
+                        }
+                    ]
+                },
+                bbox: '-20,0,45,60'
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+
+        it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_count: { reject: ['RUS'] },
+                            country_places_histogram: { min: 50000 }
+                        }
+                    ]
+                },
+                bbox: '-20,0,45,60'
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+    });
+});

test/integration/mapconfig_overviews_adapter.js

@@ -0,0 +1,87 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
+var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
+var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
+
+// configure redis pool instance to use in tests
+var redisPool = new RedisPool(global.environment.redis);
+var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool }));
+
+var redisPool = new RedisPool(global.environment.redis);
+var metadataBackend = cartodbRedis({pool: redisPool});
+var pgConnection = new PgConnection(metadataBackend);
+var pgQueryRunner = new PgQueryRunner(pgConnection);
+var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+
+
+var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
+
+describe('MapConfigOverviewsAdapter', function() {
+
+    it('should not modify layers for which no overviews are available', function(done) {
+        var sql = 'SELECT * FROM test_table';
+        var cartocss = '#layer { marker-fill: black; }';
+        var cartocss_version = '2.3.0';
+        var layer_without_overviews = {
+            type: 'cartodb',
+            options: {
+                sql: sql,
+                cartocss: cartocss,
+                cartocss_version: cartocss_version
+            }
+        };
+
+        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
+            assert.ok(!err);
+            assert.equal(layers.length, 1);
+            assert.equal(layers[0].type, 'cartodb');
+            assert.equal(layers[0].options.sql, sql);
+            assert.equal(layers[0].options.cartocss, cartocss);
+            assert.equal(layers[0].options.cartocss_version, cartocss_version);
+            assert.equal(layers[0].options.overviews, undefined);
+            done();
+        });
+    });
+});
+
+describe('MapConfigOverviewsAdapter', function() {
+
+    it('should add overviews metadata for layers using tables with overviews', function(done) {
+        var sql = 'SELECT * FROM test_table_overviews';
+        var cartocss = '#layer { marker-fill: black; }';
+        var cartocss_version = '2.3.0';
+        var layer_without_overviews = {
+            type: 'cartodb',
+            options: {
+                sql: sql,
+                cartocss: cartocss,
+                cartocss_version: cartocss_version
+            }
+        };
+
+        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
+            assert.ok(!err);
+            assert.equal(layers.length, 1);
+            assert.equal(layers[0].type, 'cartodb');
+            assert.equal(layers[0].options.sql, sql);
+            assert.equal(layers[0].options.cartocss, cartocss);
+            assert.equal(layers[0].options.cartocss_version, cartocss_version);
+            assert.ok(layers[0].options.query_rewrite_data);
+            var expected_data = {
+              overviews: {
+                test_table_overviews: {
+                  1: { table: '_vovw_1_test_table_overviews' },
+                  2: { table: '_vovw_2_test_table_overviews' }
+                }
+              }
+            };
+            assert.deepEqual(layers[0].options.query_rewrite_data, expected_data);
+            done();
+        });
+    });
+});

test/integration/overviews-metadata-api.js

@@ -0,0 +1,52 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var PgConnection = require('../../lib/cartodb/backends/pg_connection');
+var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
+
+
+describe('OverviewsMetadataApi', function() {
+
+    var overviewsMetadataApi;
+
+    before(function() {
+        var redisPool = new RedisPool(global.environment.redis);
+        var metadataBackend = cartodbRedis({pool: redisPool});
+        var pgConnection = new PgConnection(metadataBackend);
+        var pgQueryRunner = new PgQueryRunner(pgConnection);
+        overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+    });
+
+    it('should return an empty relation for tables that have no overviews', function(done) {
+        var query = 'select * from test_table';
+        overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.deepEqual(result, {});
+
+            done();
+        });
+    });
+
+    it('should return overviews metadata', function(done) {
+        var query = 'select * from test_table_overviews';
+        overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.deepEqual(result, {
+                'test_table_overviews': {
+                  1: { table: '_vovw_1_test_table_overviews' },
+                  2: { table: '_vovw_2_test_table_overviews' }
+                }
+            });
+
+            done();
+        });
+    });
+
+});

test/integration/query-tables-api.js

@@ -1,55 +0,0 @@
-require('../support/test_helper');
-
-var assert = require('assert');
-
-var RedisPool = require('redis-mpool');
-var cartodbRedis = require('cartodb-redis');
-
-var PgConnection = require('../../lib/cartodb/backends/pg_connection');
-var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
-var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
-
-
-describe('QueryTablesApi', function() {
-
-    var queryTablesApi;
-
-    before(function() {
-        var redisPool = new RedisPool(global.environment.redis);
-        var metadataBackend = cartodbRedis({pool: redisPool});
-        var pgConnection = new PgConnection(metadataBackend);
-        var pgQueryRunner = new PgQueryRunner(pgConnection);
-        queryTablesApi = new QueryTablesApi(pgQueryRunner);
-    });
-
-    // Check test/support/sql/windshaft.test.sql to understand where the values come from.
-
-    it('should return an object with affected tables array and last updated time', function(done) {
-        var query = 'select * from test_table';
-        queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
-            assert.ok(!err, err);
-
-            assert.deepEqual(result, {
-                affectedTables: [ 'public.test_table' ],
-                lastUpdatedTime: 1234567890123
-            });
-
-            done();
-        });
-    });
-
-    it('should work with private tables', function(done) {
-        var query = 'select * from test_table_private_1';
-        queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
-            assert.ok(!err, err);
-
-            assert.deepEqual(result, {
-                affectedTables: [ 'public.test_table_private_1' ],
-                lastUpdatedTime: 1234567890123
-            });
-
-            done();
-        });
-    });
-
-});

test/integration/query-tables.js

@@ -0,0 +1,71 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var PgConnection = require('../../lib/cartodb/backends/pg_connection');
+
+var QueryTables = require('cartodb-query-tables');
+
+
+describe('QueryTables', function() {
+
+    var connection;
+
+    before(function(done) {
+        var redisPool = new RedisPool(global.environment.redis);
+        var metadataBackend = cartodbRedis({pool: redisPool});
+        var pgConnection = new PgConnection(metadataBackend);
+        pgConnection.getConnection('localhost', function(err, pgConnection) {
+            if (err) {
+                return done(err);
+            }
+            connection = pgConnection;
+
+            return done();
+        });
+    });
+
+    // Check test/support/sql/windshaft.test.sql to understand where the values come from.
+
+    it('should return an object with affected tables array and last updated time', function(done) {
+        var query = 'select * from test_table';
+        QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.equal(result.getLastUpdatedAt(), 1234567890123);
+
+            assert.equal(result.tables.length, 1);
+            assert.deepEqual(result.tables[0], {
+                dbname: 'test_windshaft_cartodb_user_1_db',
+                schema_name: 'public',
+                table_name: 'test_table',
+                updated_at: new Date(1234567890123)
+            });
+
+            done();
+        });
+    });
+
+    it('should work with private tables', function(done) {
+        var query = 'select * from test_table_private_1';
+        QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.equal(result.getLastUpdatedAt(), 1234567890123);
+
+            assert.equal(result.tables.length, 1);
+            assert.deepEqual(result.tables[0], {
+                dbname: 'test_windshaft_cartodb_user_1_db',
+                schema_name: 'public',
+                table_name: 'test_table_private_1',
+                updated_at: new Date(1234567890123)
+            });
+
+            done();
+        });
+    });
+
+});

test/support/assert.js

@@ -1,7 +1,6 @@
 // Cribbed from the ever prolific Konstantin Kaefer
 // https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js
 
-var exec = require('child_process').exec;
 var fs = require('fs');
 var path = require('path');
 var util = require('util');
@@ -13,7 +12,7 @@ var request = require('request');
 var assert = module.exports = exports = require('assert');
 
 /**
- * Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
+ * Takes an image data as an input and an image path and compare them using mapnik.Image.compare mechanism, in case the
  * similarity is not within the tolerance limit it will callback with an error.
  *
  * @param buffer The image data to compare from
@@ -22,70 +21,39 @@ var assert = module.exports = exports = require('assert');
  * @param {function} callback Will call to home with null in case there is no error, otherwise with the error itself
  * @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
  */
-assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
+assert.imageBufferIsSimilarToFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
     callback = callback || function(err) { assert.ifError(err); };
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
-        testImageFilePath = createImageFromBuffer(buffer, 'test');
 
-    imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, function(err) {
-        fs.unlinkSync(testImageFilePath);
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
+    var referenceImageBuffer = fs.readFileSync(referenceImageFilePath, { encoding: null });
+
+    assert.imageBuffersAreSimilar(buffer, referenceImageBuffer, tolerance, callback);
+};
+
+assert.imageBuffersAreSimilar = function(bufferA, bufferB, tolerance, callback) {
+    var testImage = mapnik.Image.fromBytes(Buffer.isBuffer(bufferA) ? bufferA : new Buffer(bufferA, 'binary'));
+    var referenceImage = mapnik.Image.fromBytes(Buffer.isBuffer(bufferB) ? bufferB : new Buffer(bufferB, 'binary'));
+
+    imagesAreSimilar(testImage, referenceImage, tolerance, callback);
+};
+
+assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
+    callback = callback || function(err) { assert.ifError(err); };
+
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
+
+    var referenceImage = mapnik.Image.fromBytes(fs.readFileSync(referenceImageFilePath,  { encoding: null }));
+
+    imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
+        if (err) {
+            var testImageFilePath = randomImagePath();
+            testImage.save(testImageFilePath);
+        }
         callback(err);
     });
 };
 
-assert.imageBuffersAreEqual = function(bufferA, bufferB, tolerance, callback) {
-    var randStr = (Math.random() * 1e16).toString().substring(0, 8);
-    var imageFilePathA = createImageFromBuffer(bufferA, randStr + '-a'),
-        imageFilePathB = createImageFromBuffer(bufferB, randStr + '-b');
-
-    imageFilesAreEqual(imageFilePathA, imageFilePathB, tolerance, function(err, similarity) {
-        callback(err, [imageFilePathA, imageFilePathB], similarity);
-    });
-};
-
-function createImageFromBuffer(buffer, nameHint) {
-    var imageFilePath = path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
-    var err = fs.writeFileSync(imageFilePath, buffer, 'binary');
-    assert.ifError(err);
-    return imageFilePath;
-}
-
-function imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, callback) {
-    var resultFilePath = path.resolve(util.format('/tmp/windshaft-result-%s-diff.png', Date.now()));
-    var imageMagickCmd = util.format(
-        'compare -metric fuzz "%s" "%s" "%s"',
-        testImageFilePath, referenceImageFilePath, resultFilePath
-    );
-
-    exec(imageMagickCmd, function(err, stdout, stderr) {
-        if (err) {
-            fs.unlinkSync(testImageFilePath);
-            callback(err);
-        } else {
-            stderr = stderr.trim();
-            var metrics = stderr.match(/([0-9]*) \((.*)\)/);
-            if ( ! metrics ) {
-                callback(new Error("No match for " + stderr));
-                return;
-            }
-            var similarity = parseFloat(metrics[2]),
-                tolerancePerMil = (tolerance / 1000);
-            if (similarity > tolerancePerMil) {
-                err = new Error(util.format(
-                        'Images %s and %s are not equal (got %d similarity, expected %d). Result %s',
-                        testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil, resultFilePath)
-                );
-                err.similarity = similarity;
-                callback(err, similarity);
-            } else {
-                fs.unlinkSync(resultFilePath);
-                callback(null, similarity);
-            }
-        }
-    });
-}
-
-assert.imagesAreSimilar = function(testImage, referenceImage, tolerance, callback) {
+function imagesAreSimilar(testImage, referenceImage, tolerance, callback) {
     if (testImage.width() !== referenceImage.width() || testImage.height() !== referenceImage.height()) {
         return callback(new Error('Images are not the same size'));
     }
@@ -103,27 +71,10 @@ assert.imagesAreSimilar = function(testImage, referenceImage, tolerance, callbac
     } else {
         callback(null, similarity);
     }
-};
+}
 
-assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
-    callback = callback || function(err) { assert.ifError(err); };
-
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
-
-    var referenceImage = mapnik.Image.fromBytes(fs.readFileSync(referenceImageFilePath,  { encoding: null }));
-
-    assert.imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
-        if (err) {
-            var testImageFilePath = randomImagePath();
-            testImage.save(testImageFilePath);
-        }
-        callback(err);
-    });
-};
-
-function randomImagePath(nameHint) {
-    nameHint = nameHint || 'test';
-    return path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
+function randomImagePath() {
+    return path.resolve('test/results/png/image-test-' + Date.now() + '.png');
 }
 
 // jshint maxcomplexity:9

test/support/prepare_db.sh

@@ -78,11 +78,15 @@ if test x"$PREPARE_PGSQL" = xyes; then
     sed "s/:TESTPASS/${TESTPASS}/" |
     psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
-  psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
-  curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql
-  curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
-  cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql |
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+
+  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins'
+  for i in ${SQL_SCRIPTS}
+  do
+    curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
+    cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
+        | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  done
 
 fi
 

test/support/sql/.gitignore

@@ -0,0 +1 @@
+CDB_*.sql

test/support/sql/CDB_QueryStatements.sql

@@ -1,14 +0,0 @@
--- Return an array of statements found in the given query text
---
--- Regexp curtesy of Hubert Lubaczewski (depesz)
--- Implemented in plpython for performance reasons
---
-CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text) 
-RETURNS SETOF TEXT AS $$
-  import re
-  pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL )
-  for match in pat.findall(query):
-    cleaned = match[0].strip()
-    if ( cleaned ):
-      yield cleaned
-$$ language 'plpythonu' IMMUTABLE STRICT;

test/support/sql/CDB_QueryTables.sql

@@ -1,78 +0,0 @@
--- Return an array of table names scanned by a given query
---
--- Requires PostgreSQL 9.x+
---
-CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
-RETURNS text[]
-AS $$
-DECLARE
-  exp XML;
-  tables text[];
-  rec RECORD;
-  rec2 RECORD;
-BEGIN
-  
-  tables := '{}';
-
-  FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
-
-    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
-        --RAISE WARNING 'Skipping %', rec.q;
-        CONTINUE;
-    END IF;
-
-    BEGIN
-      EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
-    EXCEPTION WHEN others THEN
-      -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
-      --       the affected table ?
-      RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
-      RAISE EXCEPTION '%', SQLERRM;
-      CONTINUE;
-    END;
-
-    -- Now need to extract all values of <Relation-Name>
-
-    -- RAISE DEBUG 'Explain: %', exp;
-
-    FOR rec2 IN WITH
-      inp AS (
-        SELECT
-          xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
-          xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
-      )
-      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
-    LOOP
-      -- RAISE DEBUG 'tab: %', rec2.p;
-      -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
-    END LOOP;
-
-    -- RAISE DEBUG 'Tables: %', tables;
-
-  END LOOP;
-
-  -- RAISE DEBUG 'Tables: %', tables;
-
-  -- Remove duplicates and sort by name
-  IF array_upper(tables, 1) > 0 THEN
-    WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
-       SELECT array_agg(p) from dist into tables;
-  END IF;
-
-  --RAISE DEBUG 'Tables: %', tables;
-
-  return tables;
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-
-
--- Keep CDB_QueryTables with same signature for backwards compatibility.
--- It should probably be removed in the future.
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
-AS $$
-BEGIN
-  RETURN CDB_QueryTablesText(query)::name[];
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

test/support/sql/_CDB_QueryStatements.sql

@@ -0,0 +1,16 @@
+-- DUMMY IMPLEMENTATION
+-- Ref: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_QueryStatements.sql
+-- Originally implemented in plpython for performance reasons
+
+-- Return an array of statements found in the given query text
+--
+-- Regexp curtesy of Hubert Lubaczewski (depesz)
+--
+CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
+RETURNS SETOF TEXT AS $$
+  with matches as (
+    select regexp_matches($1, $regexp$((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)$regexp$, 'g') as m
+  )
+  select btrim(m[1]) from matches
+$$
+LANGUAGE SQL IMMUTABLE STRICT;

test/support/sql/ported/populated_places_simple_reduced.sql

@@ -7385,3 +7385,8 @@ GRANT ALL ON TABLE populated_places_simple_reduced TO :TESTUSER;
 GRANT SELECT ON TABLE populated_places_simple_reduced TO :PUBLICUSER;
 
 VACUUM ANALYZE populated_places_simple_reduced;
+
+create table populated_places_simple_reduced_private AS
+select * from populated_places_simple_reduced;
+
+GRANT ALL ON TABLE populated_places_simple_reduced_private TO :TESTUSER;

test/support/sql/windshaft.test.sql

@@ -238,3 +238,97 @@ CREATE INDEX test_big_poly_the_geom_webmercator_idx ON test_big_poly USING gist
 
 GRANT ALL ON TABLE test_big_poly TO :TESTUSER;
 GRANT SELECT ON TABLE test_big_poly TO :PUBLICUSER;
+
+-- table with overviews
+
+CREATE TABLE test_table_overviews (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+GRANT ALL ON TABLE test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER;
+
+CREATE SEQUENCE test_table_overviews_cartodb_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE test_table_overviews_cartodb_id_seq OWNED BY test_table_overviews.cartodb_id;
+
+SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true);
+
+ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass);
+
+INSERT INTO test_table_overviews VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+
+ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id);
+
+CREATE INDEX test_table_overviews_the_geom_idx ON test_table_overviews USING gist (the_geom);
+CREATE INDEX test_table_overviews_the_geom_webmercator_idx ON test_table_overviews USING gist (the_geom_webmercator);
+
+GRANT ALL ON TABLE test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER;
+
+CREATE TABLE _vovw_1_test_table_overviews (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+GRANT ALL ON TABLE _vovw_1_test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE _vovw_1_test_table_overviews TO :PUBLICUSER;
+
+CREATE TABLE _vovw_2_test_table_overviews (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER;
+
+INSERT INTO _vovw_2_test_table_overviews VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
+
+INSERT INTO _vovw_1_test_table_overviews VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241');

test/support/test-client.js

@@ -0,0 +1,230 @@
+'use strict';
+
+var qs = require('querystring');
+var step = require('step');
+
+var mapnik = require('windshaft').mapnik;
+
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+
+var assert = require('./assert');
+var helper = require('./test_helper');
+
+var CartodbWindshaft = require('../../lib/cartodb/server');
+var serverOptions = require('../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+
+function TestClient(mapConfig, apiKey) {
+    this.mapConfig = mapConfig;
+    this.apiKey = apiKey;
+    this.keysToDelete = {};
+}
+
+module.exports = TestClient;
+
+TestClient.prototype.getWidget = function(widgetName, params, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = params;
+        params = {};
+    }
+
+    var url = '/api/v1/map';
+    if (params && params.filters) {
+        url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
+    }
+
+    var layergroupId;
+    step(
+        function createLayergroup() {
+            var next = this;
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(self.mapConfig)
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+                    var parsedBody = JSON.parse(res.body);
+                    var expectedWidgetURLS = {
+                        http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
+                    };
+                    assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
+                    assert.ok(
+                        parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
+                    );
+                    return next(null, parsedBody.layergroupid);
+                }
+            );
+        },
+        function getWidgetResult(err, _layergroupId) {
+            assert.ifError(err);
+
+            var next = this;
+            layergroupId = _layergroupId;
+
+            var urlParams = {
+                own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
+            };
+            if (params && params.bbox) {
+                urlParams.bbox = params.bbox;
+            }
+            url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
+
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'GET',
+                    headers: {
+                        host: 'localhost'
+                    }
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+
+                    next(null, res);
+                }
+            );
+        },
+        function finish(err, res) {
+            self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+            self.keysToDelete['user:localhost:mapviews:global'] = 5;
+            return callback(err, res);
+        }
+    );
+};
+
+TestClient.prototype.getTile = function(z, x, y, params, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = params;
+        params = {};
+    }
+
+    var url = '/api/v1/map';
+
+    if (this.apiKey) {
+        url += '?' + qs.stringify({api_key: this.apiKey});
+    }
+
+    var layergroupId;
+    step(
+        function createLayergroup() {
+            var next = this;
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(self.mapConfig)
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+                    return next(null, JSON.parse(res.body).layergroupid);
+                }
+            );
+        },
+        function getTileResult(err, _layergroupId) {
+            assert.ifError(err);
+
+            var next = this;
+            layergroupId = _layergroupId;
+
+            url = '/api/v1/map/' + layergroupId + '/';
+
+            var layers = params.layers;
+
+            if (layers !== undefined) {
+                layers = Array.isArray(layers) ? layers : [layers];
+                url += layers.join(',') + '/';
+            }
+
+            var format = params.format || 'png';
+
+            url += [z,x,y].join('/');
+            url += '.' + format;
+
+            if (self.apiKey) {
+                url += '?' + qs.stringify({api_key: self.apiKey});
+            }
+
+            var request = {
+                url: url,
+                method: 'GET',
+                headers: {
+                    host: 'localhost'
+                }
+            };
+
+            var expectedResponse = {
+                status: 200,
+                headers: {
+                    'Content-Type': 'application/json; charset=utf-8'
+                }
+            };
+
+            var isPng = format === 'png' || format === 'torque.png';
+
+            if (isPng) {
+                request.encoding = 'binary';
+                expectedResponse.headers['Content-Type'] = 'image/png';
+            }
+
+            assert.response(server, request, expectedResponse, function(res, err) {
+                assert.ifError(err);
+
+                var image;
+
+                if (isPng) {
+                    image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+                }
+
+                next(null, res, image);
+            });
+        },
+        function finish(err, res, image) {
+            self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+            self.keysToDelete['user:localhost:mapviews:global'] = 5;
+            return callback(err, res, image);
+        }
+    );
+};
+
+TestClient.prototype.drain = function(callback) {
+    helper.deleteRedisKeys(this.keysToDelete, callback);
+};

test/support/test_helper.js

@@ -63,9 +63,25 @@ function checkCache(res) {
 
 function checkSurrogateKey(res, expectedKey) {
     assert.ok(res.headers.hasOwnProperty('surrogate-key'));
-    assert.equal(res.headers['surrogate-key'], expectedKey);
+
+    function createSet(keys, key) {
+        keys[key] = true;
+        return keys;
+    }
+    var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {});
+    var expectedKeys = expectedKey.split(' ').reduce(createSet, {});
+
+    assert.deepEqual(keys, expectedKeys);
 }
 
+var redisClient;
+
+beforeEach(function() {
+    if (!redisClient) {
+        redisClient = redis.createClient(global.environment.redis.port);
+    }
+});
+
 //global afterEach to capture test suites that leave keys in redis
 afterEach(function(done) {
 
@@ -102,7 +118,6 @@ afterEach(function(done) {
     }
 
     Object.keys(databasesTasks).forEach(function(db) {
-        var redisClient = redis.createClient(global.environment.redis.port);
         redisClient.select(db, function() {
             // Check that we start with an empty redis db
             redisClient.keys("*", function(err, keys) {
@@ -129,6 +144,7 @@ function deleteRedisKeys(keysToDelete, callback) {
         var redisClient = redis.createClient(global.environment.redis.port);
         redisClient.select(keysToDelete[k], function() {
             redisClient.del(k, function(err, deletedKeysCount) {
+                redisClient.quit();
                 assert.notStrictEqual(deletedKeysCount, 0, 'No KEYS deleted for: [db=' + keysToDelete[k] + ']' + k);
                 taskDone(k);
             });

test/unit/cartodb/overviews_query_rewriter.js

@@ -0,0 +1,399 @@
+require('../../support/test_helper');
+
+var assert           = require('assert');
+var OverviewsQueryRewriter = require('../../../lib/cartodb/utils/overviews_query_rewriter');
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+  zoom_level: 'ZoomLevel()'
+});
+
+function normalize_whitespace(txt) {
+  return txt.replace(/\s+/g, " ").trim();
+}
+
+// compare SQL statements ignoring whitespace
+function assertSameSql(sql1, sql2) {
+    assert.equal(normalize_whitespace(sql1), normalize_whitespace(sql2));
+}
+
+describe('Overviews query rewriter', function() {
+
+  it('does not alter queries if no overviews data is present', function(done){
+    var sql = "SELECT * FROM table1";
+    var overviews_sql = overviewsQueryRewriter.query(sql);
+    assert.equal(overviews_sql, sql);
+    overviews_sql = overviewsQueryRewriter.query(sql, {});
+    assert.equal(overviews_sql, sql);
+    overviews_sql = overviewsQueryRewriter.query(sql, { overviews: {} });
+    assert.equal(overviews_sql, sql);
+    done();
+  });
+
+
+  it('does not alter queries which don\'t use overviews', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table2: {
+              0: { table: 'table2_ov0' },
+              1: { table: 'table2_ov1' },
+              4: { table: 'table2_ov4' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+    done();
+  });
+
+  // jshint multistr:true
+
+  it('generates query with single overview layer for level 0', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query with single overview layer for level >0', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query with multiple overview layers for all levels up to N', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' },
+              1: { table: 'table1_ov1' },
+              2: { table: 'table1_ov2' },
+              3: { table: 'table1_ov3' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM  (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
+            UNION ALL\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
+            UNION ALL\
+            SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query with multiple overview layers for random levels', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' },
+              1: { table: 'table1_ov1' },
+              6: { table: 'table1_ov6' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
+            UNION ALL\
+            SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table with explicit schema', function(done){
+    var sql = "SELECT * FROM public.table1";
+    var data = {
+        overviews: {
+            'public.table1': {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table with explicit schema in the overviews info', function(done){
+    var sql = "SELECT * FROM public.table1";
+    var data = {
+        overviews: {
+            'public.table1': {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table that needs quoting with explicit schema', function(done){
+    var sql = "SELECT * FROM public.\"table 1\"";
+    var data = {
+        overviews: {
+            'public."table 1"': {
+              2: { table: '"table 1_ov2"' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
+          ) AS \"_vovw_table 1\"\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table with explicit schema that needs quoting', function(done){
+    var sql = "SELECT * FROM \"user-1\".table1";
+    var data = {
+        overviews: {
+            '"user-1".table1': {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+
+  it('generates query for a table with explicit schema both needing quoting', function(done){
+    var sql = "SELECT * FROM \"user-1\".\"table 1\"";
+    var data = {
+        overviews: {
+            '"user-1"."table 1"': {
+              2: { table: '"table 1_ov2"' }
+
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
+          ) AS \"_vovw_table 1\"\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+
+  it('generates query using overviews for queries with selected columns', function(done){
+    var sql = "SELECT column1, column2, column3 FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query using overviews for queries with a semicolon', function(done){
+    var sql = "SELECT column1, column2, column3 FROM table1;";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1;\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query using overviews for queries with extra whitespace', function(done){
+    var sql = "  SELECT  column1,column2,  column3 FROM  table1  ";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1,column2, column3 FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('does not alter queries which have not the simple supported form', function(done){
+    var sql = "SELECT * FROM table1 WHERE column1='x'";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT * FROM table1 JOIN table2 ON (table1.col1=table2.col1)";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT a+b AS c FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT f(a) AS b FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT * FROM table1 AS x";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "WITH a AS (1) SELECT * FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT * FROM table1 WHERE a=1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "\
+        SELECT table1.* FROM table1 \
+               JOIN areas ON ST_Intersects(table1.the_geom, areas.the_geom) \
+               WHERE areas.name='A' \
+          ";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT table1.*, column1, column2, column3 FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    done();
+  });
+
+
+});

test/unit/cartodb/table_name_parser.js

@@ -0,0 +1,60 @@
+require('../../support/test_helper');
+
+var assert        = require('assert');
+var TableNameParser     = require('../../../lib/cartodb/utils/table_name_parser');
+
+describe('TableNameParser', function() {
+
+  it('parses table names with scheme and quotes', function(done){
+
+      var test_cases = [
+        ['xyz',             { schema: null,       table: 'xyz' }],
+        ['"xyz"',           { schema: null,       table: 'xyz' }],
+        ['"xy z"',          { schema: null,       table: 'xy z' }],
+        ['"xy.z"',          { schema: null,       table: 'xy.z' }],
+        ['"x.y.z"',         { schema: null,       table: 'x.y.z' }],
+        ['abc.xyz',         { schema: 'abc',      table: 'xyz' }],
+        ['"abc".xyz',       { schema: 'abc',      table: 'xyz' }],
+        ['abc."xyz"',       { schema: 'abc',      table: 'xyz' }],
+        ['"abc"."xyz"',     { schema: 'abc',      table: 'xyz' }],
+        ['"a bc"."x yz"',   { schema: 'a bc',     table: 'x yz' }],
+        ['"a bc".xyz',      { schema: 'a bc',     table: 'xyz' }],
+        ['"a.bc".xyz',      { schema: 'a.bc',     table: 'xyz' }],
+        ['"a.b.c".xyz',     { schema: 'a.b.c',    table: 'xyz' }],
+        ['"a.b.c.".xyz',    { schema: 'a.b.c.',   table: 'xyz' }],
+        ['"a""bc".xyz',     { schema: 'a"bc',     table: 'xyz' }],
+        ['"a""bc"."x""yz"', { schema: 'a"bc',     table: 'x"yz' }],
+      ];
+
+      test_cases.forEach(function(test_case) {
+          var table_name = test_case[0];
+          var expected_result = test_case[1];
+          var result = TableNameParser.parse(table_name);
+          assert.deepEqual(result, expected_result);
+      });
+      done();
+  });
+
+  it('quotes identifiers that need quoting', function(done){
+      assert.equal(TableNameParser.quote('x yz'), '"x yz"');
+      assert.equal(TableNameParser.quote('x-yz'), '"x-yz"');
+      assert.equal(TableNameParser.quote('x.yz'), '"x.yz"');
+      done();
+  });
+
+  it('doubles quotes', function(done){
+      assert.equal(TableNameParser.quote('x"yz'), '"x""yz"');
+      assert.equal(TableNameParser.quote('x"y"z'), '"x""y""z"');
+      assert.equal(TableNameParser.quote('x""y"z'), '"x""""y""z"');
+      assert.equal(TableNameParser.quote('x "yz'), '"x ""yz"');
+      assert.equal(TableNameParser.quote('x"y-y"z'), '"x""y-y""z"');
+      done();
+  });
+
+  it('does not quote identifiers that don\'t need to be quoted', function(done){
+      assert.equal(TableNameParser.quote('xyz'), 'xyz');
+      assert.equal(TableNameParser.quote('x_z123'), 'x_z123');
+      done();
+  });
+
+});

tools/README.md

@@ -1,8 +0,0 @@
-Deprecated tools
-================
-
-All tools and scripts found in this directory are deprecated and no longer maintained.
-
-Use at your own peril.
-
-In future releases they might get removed.

tools/convert_database_styles

@@ -1,81 +0,0 @@
-#!/usr/bin/env node
-
-var path = require('path');
-
-var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
-var mapnik = require('mapnik');
-var redis = require('redis');
-
-function usage(me, exitcode) {
-  console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
-  process.exit(exitcode);
-}
-
-var node_path = process.argv.shift(); 
-var script_path = process.argv.shift(); 
-var me = path.basename(script_path);
-
-var database_name = process.argv.shift()
-var table_name = process.argv.shift()
-var MAPNIK_VERSION = process.argv.shift()
-
-
-if ( ! MAPNIK_VERSION ) {
-  MAPNIK_VERSION = mapnik.versions.mapnik;
-}
-
-if ( ! database_name || ! table_name) {
-  usage(me, 1);
-}
-
-var REDIS_PORT = 6379; // TODO: make a command line parameter
-
-var dbnum = 0;
-
-var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
-
-var failures = [];
-
-var client = redis.createClient(REDIS_PORT, 'localhost');
-client.on('connect', function() {
-  client.select(dbnum);
-  client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
-
-    processNext = function() {
-      if ( ! matches.length ) process.exit(failures.length);
-      var k = matches.shift();
-
-      if ( /map_style\|.*\|.*\|/.test(k) ) {
-        //console.warn("Key " + k + " is EXTENDED, skipping");
-        processNext();
-      } 
-
-      var out = 'map_style|' + database_name + '|' + table_name + ': ';
-
-      var mml_builder = mml_store.mml_builder({
-        dbname:database_name, 
-        table:table_name},
-        function(err, payload) {
-          if ( err ) { 
-            console.warn(out + err.message); 
-            failures.push(k); processNext(); 
-          }
-          else {
-            mml_builder.resetStyle(function(err, data) {
-              if ( err ) { 
-                console.warn(out + err.message); 
-                failures.push(k); 
-              }
-              else console.log(out + 'OK');
-              processNext();
-            }, true);
-          }
-      });
-
-    };
-
-    processNext();
-
-  });
-
-});

tools/create_multilayer

@@ -1,40 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
-
-# This is for direct windshaft connection
-#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    tiler_url="$1"
-  fi
-  shift
-done
-
-if test -z "$cfg"; then
-  echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  exit 1
-fi
-
-cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
-echo $tok

tools/create_template

@@ -1,49 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$cfg"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-# Successful response contains no space
-echo "$res" | grep " " && { echo $res && exit 1; }
-
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
-echo $tok
-

tools/delete_template

@@ -1,45 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$tpl"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
-echo $tok

tools/examples/mapconfig_simple.js

@@ -1,11 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"cartodb",
-	  "options":{
-      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
-	    "cartocss":"#style{ marker-width: 12;}",
-	    "cartocss_version":"2.1.1",
-      "Interactivity":"id"
-    }
-  }]
-}

tools/examples/mapconfig_slow.js

@@ -1,11 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"cartodb",
-	  "options":{
-      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
-	    "cartocss":"#style{ marker-width: 12;}",
-	    "cartocss_version":"2.1.1",
-      "Interactivity":"id"
-    }
-  }]
-}

tools/examples/mapconfig_torque.js

@@ -1,10 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"torque",
-	  "options":{
-      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
-	    "cartocss":"Map{ -torque-time-attribute:'id'; -torque-aggregation-function:'count(id)'; -torque-frame-count:2; -torque-resolution:2}",
-      "cartocss_version": "2.1.1"
-    }
-  }]
-}

tools/examples/template_simple.js

@@ -1,17 +0,0 @@
-{
- "version":"0.0.1",
- "name":"simple",
- "placeholders":{},
- "auth":{ "method":"open" },
- "layergroup":{
-   "version":"1.0.1",
-    "layers":[{
-      "type":"cartodb",
-      "options":{
-        "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
-        "cartocss":"#s{ marker-width: 12;}",
-        "cartocss_version":"2.1.1"
-      }
-    }]
-  }
-}

tools/instanciate_template

@@ -1,53 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$tpl"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> [<template_params>]" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-if test -z "$cfg"; then
-  cfg="/dev/null"
-fi
-
-tiler_url="${tiler_url}/${tpl}"
-
-cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
-echo $tok

tools/list_templates

@@ -1,45 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test "$1" = "-h" -o "$1" = "-?"; then
-    echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
-    echo "Default <tiler_url> is ${tiler_url}" >&2
-    echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-    exit 0
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-
-cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-node <<EOF
-var parsed = JSON.parse('$res');
-console.dir(parsed);
-EOF
-

tools/munin/Makefile

@@ -1,19 +0,0 @@
-MUNIN_PLUGINS_DIR=/etc/munin/plugins
-MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
-PWD=$(shell pwd)
-
-all: windshaft.conf
-
-windshaft.conf: windshaft.conf.in
-	sed 's#@PWD@#$(PWD)#' < $< > $@
-
-install-munin-plugin-conf: windshaft.conf 
-	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
-
-install-munin-plugin: windshaft
-	install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
-
-install: install-munin-plugin install-munin-plugin-conf
-
-clean:
-	rm -f windshaft.conf

tools/munin/windshaft

@@ -1,87 +0,0 @@
-#!/bin/sh
-
-envnik=$(basename "${TILER_ENVIRONMENT}" .js)
-
-if test "$1" = "config"; then
-echo "graph_title fd usage (${envnik})"
-cat <<"EOM"
-graph_vlabel number of file descriptors
-graph_category windshaft
-graph_scale no
-procs.label Number of tiler processes
-pgsql.label PostgreSQL connections (max)
-redis.label Redis connections (max)
-http.label Incoming http requests (max)
-caches.label Number of renderer caches (max)
-nfd.label Total file descriptors (max)
-EOM
-exit 0
-fi
-
-if test x"$1" != x; then
-  TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
-fi
-
-if test -z "$TILER_ENVIRONMENT"; then
-  echo "Usage: $0 [<environment>]" >&2
-  echo "   or: [TILER_ENVIRONMENT=<environment>] $0" >&2
-  exit 1
-fi
-
-http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
-pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
-redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
-
-pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
-nworkers=$(echo "${pids}" | wc -l)
-pids=$(echo "${pids}" | paste -sd ' ')
-
-if test -z "${pids}"; then
-  echo "No processes found listening on tcp port '${http_port}'" >&2
-  exit 1
-fi
-
-tmpreport="/tmp/checkfd.$$.txt"
-
-lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
-
-maxdb=0
-maxredis=0
-maxhttp=0
-maxtot=0
-maxcache=0
-
-for pid in ${pids}; do
-
-  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
-  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
-  
-  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
-  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
-
-  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
-  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
-
-  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
-  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
-
-  log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
-  if test -e "${log}"; then
-    kill -USR2 "${pid}"
-    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l)
-    if test $cnt -gt $maxcache; then maxcache=$cnt; fi
-  else
-    # report the error...
-    maxcache=-1
-  fi
-
-done
-
-echo "procs.value ${nworkers}"
-echo "pgsql.value ${maxdb}"
-echo "redis.value ${maxredis}"
-echo "http.value ${maxhttp}"
-echo "caches.value ${maxcache}"
-echo "nfd.value ${maxtot}"
-
-rm -f "${tmpreport}"

tools/munin/windshaft.conf.in

@@ -1,5 +0,0 @@
-# Configuration file for munin plugin
-
-[windshaft]
-user root
-env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js

tools/performance/stresstest_templates.sh

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$tpl"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-basedir=$(cd $(dirname $0); cd ..; pwd)
-export CDB_APIKEY=${apikey}
-max=3000000
-i=0
-while test "$i" -le "$max"; do
-  tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"`
-  tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin`
-  if test $? -ne 0; then
-    echo $tpl_id >&2
-    break
-  fi
-  tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin`
-  if test $? -ne 0; then
-    echo $tpl_id >&2
-    break
-  fi
-  out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}`
-  if test $? -ne 0; then
-    echo $out >&2
-    break
-  fi
-  i=$((i+1))
-  if test `expr $i % 100` -eq 0; then
-    echo -n "."
-  fi
-done

tools/reset_styles

@@ -1,105 +0,0 @@
-#!/usr/bin/env node
-
-/*
-
-This scripts drops all extended map_style keys in redis and regenerates
-the XML caches in all the base ones to target the configured mapnik_version.
-
-Optionally (with --convert) it also re-writes the CartoCSS if needed
-to target the configured mapnik_version.
-
-It is recommended to make a backup of the redis database before using
-this script.
-
-*/
-
-var path = require('path');
-
-// Reset all styles in the store
-var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
-var mapnik = require('mapnik');
-var redis = require('redis');
-
-function usage(me, exitcode) {
-  console.log("Usage: " + me + " [--convert] <environment>");
-  process.exit(exitcode);
-}
-
-var doConvert = false;
-
-var node_path = process.argv.shift(); 
-var script_path = process.argv.shift(); 
-var me = path.basename(script_path);
-var ENV;
-var arg;
-while ( arg = process.argv.shift() ) {
-  if ( arg == '--convert' ) {
-    doConvert = true;
-  }
-  else if ( ! ENV ) {
-    ENV = arg;
-  }
-  else {
-    usage(me, 1);
-  }
-}
-
-if ( ! ENV ) usage(me, 1);
-
-global.environment = require('../config/environments/' + ENV);
-var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
-
-var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
-
-console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
-
-var dbnum = 0;
-
-var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
-
-var failures = [];
-
-var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
-client.on('connect', function() {
-  client.select(dbnum);
-  client.keys('map_style|*', function(err, matches) {
-
-    processNext = function() {
-      if ( ! matches.length ) process.exit(failures.length);
-      var k = matches.shift();
-
-      if ( /map_style\|.*\|.*\|/.test(k) ) {
-        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
-        //console.warn("Key " + k + " is EXTENDED, dropping");
-        client.del(k, function(err) {
-          if ( err ) console.warn("Error dropping key " + k);
-          processNext();
-        });
-        return;
-      } 
-
-      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
-      var db = params[1];
-      var tab = params[2];
-      var out = 'map_style|' + db + '|' + tab + ': ';
-
-      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
-                                              function(err, payload) {
-
-          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
-          else {
-            mml_builder.resetStyle(function(err, data) {
-              if ( err ) { console.warn(out + err.message); failures.push(k); }
-              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
-              processNext();
-            }, doConvert);
-          }
-      });
-
-    };
-
-    processNext();
-
-  });
-
-});

tools/show_style

@@ -1,148 +0,0 @@
-#!/usr/bin/env node
-
-var path = require('path');
-var redis = require('redis');
-var Step = require('step');
-
-
-function usage(me, exitcode) {
-  console.log("Usage: " + me + " [--env <environment>] <username> [<tablename>|~<token>]");
-  process.exit(exitcode);
-}
-
-var node_path = process.argv.shift(); 
-var script_path = process.argv.shift(); 
-var basedir = path.dirname(script_path);
-var me = path.basename(script_path);
-
-var ENV = 'development.js';
-var username, token;
-var arg;
-while ( arg = process.argv.shift() ) {
-  if ( arg == '--env' ) {
-    ENV = process.argv.shift();
-  }
-  else if ( ! username ) {
-    username = arg;
-  }
-  else if ( ! token ) {
-    token = arg;
-  }
-  else {
-    console.warn("Unused parameter " + arg);
-  }
-}
-
-if ( ! username ) usage(me, 1);
-
-console.log("Using environment " + ENV);
-
-global.environment = require('../config/environments/' + ENV);
-// _after_ setting global.environment
-var serverOptions = require('../lib/cartodb/server_options')();
-
-var client;
-var dbname;
-Step(
-  function getClient() {
-    client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
-    client.on('connect', this);
-  },
-  function getUserMeta(err) {
-    if ( err ) throw err;
-    client.select(5);
-    client.hgetall('rails:users:' + username, this);
-  },
-  function readDB(err, data) {
-    if ( err ) throw err;
-    if ( ! data )
-      throw new Error('Username ' + username + ' unknown by redis on port '
-        + serverOptions.redis.port + ' (try CARTODB/script/restore_redis?)');
-    //console.log("Data:"); console.dir(data);
-    dbname = data['database_name'];
-    console.log("Database name for user " + username + ": " + dbname);
-    client.select(0);
-    return null;
-  },
-  function showTokens(err) {
-    if ( err ) throw err;
-    if ( token ) return null;
-    var next = this;
-    Step(
-      function getTokens() {
-        client.keys('map_style|' + dbname + '|*', this);
-      },
-      function showTokens(err, data) {
-        if (err) throw err;
-        if ( data ) console.log(data.join('\n'));
-        return null;
-      },
-      function showTokensFinish(err) {
-        next(err);
-      }
-    );
-  },
-  function showStyle(err) {
-    if ( err ) throw err;
-    if ( ! token ) return null;
-    var next = this;
-    Step(
-      function getStyle() {
-        client.get('map_style|' + dbname + '|' + token, this);
-      },
-      function showStyle(err, data) {
-        if ( err ) throw err;
-        if ( ! data ) {
-          throw new Error(token + ': no such map style known by redis on port '
-            + serverOptions.redis.port);
-        }
-//console.log("data: " + data);
-        var x=JSON.parse(data);
-        printMapnikStyle(x, this);
-      },
-      function showStyleFinish(err) {
-        next(err);
-      }
-    );
-  },
-  function finish(err) {
-    if ( err ) {
-      console.error(err.message)
-      process.exit(1);
-    }
-    process.exit(0);
-  }
-);
-
-function printMapnikStyle(x, callback) {
-  console.log('style: ' + x.style);
-  console.log('version: ' + x.version);
-  var grainstore = require(basedir + '/../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
-  var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
-  var builderconfig = {dbname:dbname};
-  if ( token.match(/^~/) ) {
-    builderconfig.token = token.substring(1);
-  } else {
-    builderconfig.table = token;
-  }
-  var mml_builder;
-  Step(
-    function getBuilder() {
-      mml_builder = mml_store.mml_builder(builderconfig, this);
-    },
-    function getXML(err, builder) {
-      if ( err ) throw err;
-      mml_builder.toXML(this);
-    },
-    function showXML(err, xml) {
-      if ( err ) throw err;
-      console.log('- XML - ');
-      console.log(xml);
-      return null;
-    },
-    function finish(err) {
-      callback(err);
-    }
-  );
-}
-

tools/update_template

@@ -1,47 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$cfg"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> <template_config>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
-echo $tok