Release 2.26.3

Optimize overviews queries

.travis.yml

@@ -1,16 +1,12 @@
-sudo: false
+sudo: true
 
 addons:
-  postgresql: "9.4"
-  apt:
-    packages:
-    - postgresql-plpython-9.4
-    - pkg-config
-    - libcairo2-dev
-    - libjpeg8-dev
-    - libgif-dev
+  postgresql: "9.3"
 
 before_install:
+  - sudo apt-get update
+  - sudo apt-get install -y postgresql-plpython-9.3 pkg-config libcairo2-dev libjpeg8-dev libgif-dev
+  - npm install -g npm@2
   - createdb template_postgis
   - psql -c "CREATE EXTENSION postgis" template_postgis
 

INSTALL.md

@@ -16,10 +16,6 @@ Make sure that you have the requirements needed. These are
   - CartoDB 0.9.5+ (for `CDB_QueryTables`)
   - Varnish (http://www.varnish-cache.org)
 
-- For running the testsuite
-  - ImageMagick (http://www.imagemagick.org)
-
-
 On Ubuntu 14.04 the dependencies can be installed with
 
 ```shell

NEWS.md

@@ -1,5 +1,101 @@
 # Changelog
 
+## 2.26.3
+
+Released 2016-03-03
+
+Improvements:
+ - Optimize overviews queries for efficient spatial filtering in PostgreSQL
+
+
+## 2.26.2
+
+Released 2016-02-25
+
+Announcements:
+ - Upgrades windshaft to [1.13.2](https://github.com/CartoDB/Windshaft/releases/tag/1.13.2)
+
+
+## 2.26.1
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.1](https://github.com/CartoDB/Windshaft/releases/tag/1.13.1)
+
+
+## 2.26.0
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.0](https://github.com/CartoDB/Windshaft/releases/tag/1.13.0)
+
+
+## 2.25.2
+
+Released 2016-02-22
+
+Bug fixes:
+ - Correct URLs for widgets in named maps #381
+
+
+## 2.25.1
+
+Released 2016-02-22
+
+Announcements:
+ - Upgrades windshaft to [1.11.1](https://github.com/CartoDB/Windshaft/releases/tag/1.11.1)
+
+
+## 2.25.0
+
+Released 2016-02-18
+
+Announcements:
+ - Upgrades windshaft to [1.11.0](https://github.com/CartoDB/Windshaft/releases/tag/1.11.0)
+
+
+## 2.24.0
+
+Released 2016-02-15
+
+Announcements:
+ - Upgrades windshaft to [1.10.1](https://github.com/CartoDB/Windshaft/releases/tag/1.10.1)
+
+
+## 2.23.0
+
+Released 2016-02-10
+
+Improvements:
+- Support for overviews
+
+
+## 2.22.0
+
+Released 2016-02-08
+
+Announcements:
+ - Upgrades windshaft to [1.8.3](https://github.com/CartoDB/Windshaft/releases/tag/1.8.3)
+
+
+## 2.21.1
+
+Released 2016-02-05
+
+Bug fixes:
+ - Added default config for geojson renderer
+
+
+## 2.21.0
+
+Released 2016-02-04
+
+Announcements:
+ - Upgrades windshaft to [1.8.2](https://github.com/CartoDB/Windshaft/releases/tag/1.8.2)
+
+
 ## 2.20.0
 
 Released 2016-01-20
@@ -819,7 +915,7 @@ Released 2014-03-10
 
 Enhancements:
 
- - Set statsd prefix for all endpoints 
+ - Set statsd prefix for all endpoints
  - Respond with a permission denied on attempt to access map tiles waiving
    signature of someone who had not left any (#170)
  - Do not log an error on GET / (#177)
@@ -859,7 +955,7 @@ Released 2014-02-27
 Enhancements:
 
  - Upgrades windshaft to 0.19.1 with many performance improvements,
-   See node_modules/windshaft/NEWS 
+   See node_modules/windshaft/NEWS
  - Improve speed of instanciating a map (#147, #159, #165)
  - Give meaningful error on attempts to use map tokens
    with attribute service (#156)
@@ -948,7 +1044,7 @@ Bug fixes:
 
 Released 2014-01-30
 
-Bug fixes: 
+Bug fixes:
 
 * layergroup accept both map_key and api_key (#91)
 * Fix public instanciation of signed template accessing private data (#114)
@@ -1071,7 +1167,7 @@ Released 2013-10-03
   "[ zoom > 3]" CartoCSS snippets (note the space)
 * Fix backward compatibility handling of sqlapi.host configuration (#82)
 * Fix error for invalid text-name in CartoCSS (#81)
-* Do not let anonymous requests use authorized renderer caches 
+* Do not let anonymous requests use authorized renderer caches
 
 ## 1.3.4
 
@@ -1118,7 +1214,7 @@ NOTE: configuration sqlapi.host renamed to sqlapi.domain
 * Multilayer API changes
   * Layers passed by index in grid fetching url
   * Interactivity only specified in layergroup config
-  * Embed cache_buster within token 
+  * Embed cache_buster within token
   * Use ISO format for last_modified timestamp
 * Expected LZMA encoding changed to base64
 
@@ -1172,7 +1268,7 @@ Released DD//MM//YY
 
 Released DD//MM//YY
 
-* Reduce default extent to allow for consistent proj4 round-tripping 
+* Reduce default extent to allow for consistent proj4 round-tripping
 * Enhance reset_styles script to use full configuration (#62)
 * Have reset_styles script also drop extended keys (#58)
 * Fix example postgis parameter for simplifying input geoms (#63)
@@ -1222,7 +1318,7 @@ Released (30/10/12)
  * Autodetect target mapnik version and let config override it
  * Add tools/reset_styles script to batch-reset (and optionally convert) styles
 * Configurable logging format (#4)
-* Detailed error on missing user metadata 
+* Detailed error on missing user metadata
 * Properly handle unauthenticated requests for metadata
 * Accept "api_key" in addition to "map_key",
   both in query_string and POST body (#38)

README.md

@@ -67,3 +67,16 @@ Contributing
 ---
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+### Developing with a custom windshaft version
+
+If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
+to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
+
+**Quick start**:
+
+```shell
+~/windshaft-directory $ npm install
+~/windshaft-directory $ npm link
+~/windshaft-cartodb-directory $ npm link windshaft
+```

config/environments/development.js.example

@@ -13,7 +13,7 @@ var config = {
     // Base URLs for the APIs
     //
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
@@ -33,7 +33,7 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in milliseconds 
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
@@ -147,7 +147,24 @@ var config = {
               // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
               // internal cache.
               cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
           }
+
       },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
@@ -232,7 +249,7 @@ var config = {
         serviceId: 'wadus_service_id'
     }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:true
     // Settings for the health check available at /health
@@ -250,7 +267,10 @@ var config = {
         // whether it should intercept tile render errors an act based on them, enabled by default.
         onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
-        cdbQueryTablesFromPostgres: true
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: true
+
     }
 };
 

config/environments/production.js.example

@@ -13,7 +13,7 @@ var config = {
     // Base URLs for the APIs
     //
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
@@ -34,7 +34,7 @@ var config = {
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
     // idle socket timeout, in milliseconds
-    ,socket_timeout: 600000 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
@@ -141,7 +141,24 @@ var config = {
               // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
               // internal cache.
               cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
           }
+
       },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
@@ -226,7 +243,7 @@ var config = {
         serviceId: 'wadus_service_id'
     }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:false
     ,serverMetadata: {
@@ -250,7 +267,9 @@ var config = {
         // whether it should intercept tile render errors an act based on them, enabled by default.
         onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
-        cdbQueryTablesFromPostgres: true
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: false
     }
 };
 

config/environments/staging.js.example

@@ -13,7 +13,7 @@ var config = {
     // Base URLs for the APIs
     //
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/maps/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
@@ -34,7 +34,7 @@ var config = {
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
     // idle socket timeout, in milliseconds
-    ,socket_timeout: 600000 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
@@ -141,7 +141,24 @@ var config = {
               // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
               // internal cache.
               cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
           }
+
       },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
@@ -226,7 +243,7 @@ var config = {
         serviceId: 'wadus_service_id'
     }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:true
     ,serverMetadata: {
@@ -250,7 +267,9 @@ var config = {
         // whether it should intercept tile render errors an act based on them, enabled by default.
         onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
-        cdbQueryTablesFromPostgres: true
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: true
     }
 };
 

config/environments/test.js.example

@@ -13,7 +13,7 @@ var config = {
     // Base URLs for the APIs
     //
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
@@ -34,7 +34,7 @@ var config = {
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
     // idle socket timeout, in milliseconds
-    ,socket_timeout: 600000 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
@@ -141,6 +141,22 @@ var config = {
               // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
               // internal cache.
               cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false // this requires postgis >=2.2 and geos >=3.5
           }
       },
       http: {
@@ -228,7 +244,7 @@ var config = {
         serviceId: 'wadus_service_id'
     }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:true
     // Settings for the health check available at /health
@@ -246,7 +262,9 @@ var config = {
         // whether it should intercept tile render errors an act based on them, enabled by default.
         onTileErrorStrategy: true,
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
-        cdbQueryTablesFromPostgres: true
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: true
     }
 };
 

docs/Map-API.md

@@ -1,6 +1,6 @@
 # Maps API
 
-The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and you can apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
+The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles, using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
 
 You can create two types of maps with the Maps API:
 

docs/anonymous_maps.md

@@ -28,7 +28,7 @@ POST /api/v1/map
 }
 ```
 
-Should be a [Mapconfig](https://github.com/CartoDB/Windshaft/blob/0.44.1/doc/MapConfig-1.3.0.md).
+See [MapConfig File Formats](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for details.
 
 #### Response
 

docs/named_maps.md

@@ -1,6 +1,6 @@
 # Named Maps
 
-Named maps are essentially the same as anonymous maps except the MapConfig is stored on the server and the map is given a unique name. Two other big differences are: you can create named maps from private data and that users without an API Key can see them even though they are from that private data.
+Named maps are essentially the same as anonymous maps except the MapConfig is stored on the server, and the map is given a unique name. You can create named maps from private data, and users without an API Key can view your Named Map (while keeping your data private). The Named map workflow consists of making a call to your database, referencing a table, inserting your variables into the template where placeholders are defined, and creating custom queries.
 
 The main two differences compared to anonymous maps are:
 
@@ -12,7 +12,7 @@ The main two differences compared to anonymous maps are:
 
 Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see auth section).
 
-**Note:** There is a limit of 4,096 named maps allowed per account. If you need to create more Named Maps, it is recommended to use templates.
+**Note:** There is a limit of 4,096 named maps allowed per account. If you need to create more Named maps, it is recommended to use templates.
 
 ## Create
 
@@ -91,7 +91,7 @@ auth |
 |_ method | `"token"` or `"open"` (the default if no `"method"` is given).
 |_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the named map.
 placeholders | Variables not listed here are not substituted. Variables not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
-layergroup | the layer list definition. This is the MapConfig explained in anonymous maps. See [MapConfig documentation](https://github.com/CartoDB/Windshaft/blob/0.44.1/doc/MapConfig-1.3.0.md) for more info.
+layergroup | the layer list definition. This is the MapConfig explained in anonymous maps. See [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for more info.
 
 view (optional) | extra keys to specify the compelling area for the map. It can be used to have a static preview of a named map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
 --- | ---
@@ -437,7 +437,7 @@ curl -X GET 'https://documentation.cartodb.com/api/v1/map/named/:template_name?a
 }
 ```
 
-## Use with CartoDB.js
+## Use CartoDB.js to Create Named Maps
 Named maps can be used with CartoDB.js by specifying a named map in a layer source as follows. Named maps are treated almost the same as other layer source types in most other ways.
 
 ```js
@@ -461,4 +461,15 @@ cartodb.createLayer('map_dom_id',layerSource)
 [CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js/) has methods for accessing your named maps.
 
 1. [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
+
+    **Note:** The CartoDB.js `layer.setParams()` function is not supported when using Named maps for Torque.
+
 2. [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
+
+### Complete Examples of Named Maps created with CartoDB.js
+
+- [Named map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+
+- [Named map with interactivity and config file used to create it](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+
+- [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)

docs/static_maps_api.md

@@ -127,7 +127,7 @@ By manipulating the `"urlTemplate"` custom basemaps can be used in generating st
 
 **CartoDB**
 
-As described in the [Mapconfig documentation](https://github.com/CartoDB/Windshaft/blob/0.44.1/doc/MapConfig-1.3.0.md), a "cartodb" type layer is now just an alias to a "mapnik" type layer as above, intended for backwards compatibility.
+As described in the [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/), a "cartodb" type layer is now just an alias to a "mapnik" type layer as above, intended for backwards compatibility.
 
 ```javascript
 {

lib/cartodb/api/overviews_metadata_api.js

@@ -0,0 +1,43 @@
+function OverviewsMetadataApi(pgQueryRunner) {
+   this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = OverviewsMetadataApi;
+
+// TODO: share this with QueryTablesApi? ... or maintain independence?
+var affectedTableRegexCache = {
+    bbox: /!bbox!/g,
+    scale_denominator: /!scale_denominator!/g,
+    pixel_width: /!pixel_width!/g,
+    pixel_height: /!pixel_height!/g
+};
+
+function prepareSql(sql) {
+    return sql
+        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+        .replace(affectedTableRegexCache.scale_denominator, '0')
+        .replace(affectedTableRegexCache.pixel_width, '1')
+        .replace(affectedTableRegexCache.pixel_height, '1')
+    ;
+}
+
+OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) {
+    var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
+    this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
+        if (err){
+            callback(err);
+            return;
+        }
+        var metadata = {};
+        rows.forEach(function(row) {
+            var table = row.base_table;
+            var table_metadata = metadata[table];
+            if ( !table_metadata ) {
+              table_metadata = metadata[table] = {};
+            }
+            table_metadata[row.z] = { table: row.overview_table };
+        });
+        return callback(null, metadata);
+    });
+
+};

lib/cartodb/backends/template_maps.js

@@ -21,7 +21,7 @@ var util = require('util');
 // @param opts TemplateMap options. Supported elements:
 //    'max_user_templates' limit on the number of per-user
 //
-// 
+//
 function TemplateMaps(redis_pool, opts) {
   if (!(this instanceof TemplateMaps)) {
       return new TemplateMaps();
@@ -41,7 +41,7 @@ function TemplateMaps(redis_pool, opts) {
   //
   // Map templates are owned by a user that specifies access permissions
   // for their instances.
-  // 
+  //
   // We have the following datastores:
   //
   //  1. User templates: set of per-user map templates
@@ -197,7 +197,7 @@ function templateDefaults(template) {
 // @param template layergroup template, see
 //        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
 //
-// @param callback function(err, tpl_id) 
+// @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
 o.addTemplate = function(owner, template, callback) {
@@ -296,7 +296,7 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
 //
 // @param callback function(err)
-//        
+//
 o.updTemplate = function(owner, tpl_id, template, callback) {
     var self = this;
 
@@ -355,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 //
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
-// 
+//
 o.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
@@ -422,7 +422,7 @@ o.isAuthorized = function(template, authTokens) {
 //        Only the ones found in the template's placeholders object
 //        will be used, with missing ones taking default values.
 //
-// @returns a layergroup configuration 
+// @returns a layergroup configuration
 //
 // @throws Error on malformed template or parameter
 //
@@ -431,7 +431,7 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
     _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
 
 function _replaceVars (str, params) {
-  //return _.template(str, params); // lazy way, possibly dangerous 
+  //return _.template(str, params); // lazy way, possibly dangerous
   // Construct regular expressions for each param
     Object.keys(params).forEach(function(k) {
         str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);

lib/cartodb/controllers/base.js

@@ -239,6 +239,7 @@ module.exports.findStatusCode = findStatusCode;
 
 function statusFromErrorMessage(errMsg) {
     // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
     var statusCode = 400;
     if ( -1 !== errMsg.indexOf('permission denied') ) {
         statusCode = 403;
@@ -252,6 +253,8 @@ function statusFromErrorMessage(errMsg) {
     else if ( -1 !== errMsg.indexOf('does not exist') ) {
         if ( -1 !== errMsg.indexOf(' role ') ) {
             statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
         } else {
             statusCode = 404;
         }

lib/cartodb/controllers/map.js

@@ -18,6 +18,7 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
 var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
 var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
 var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
+var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter');
 
 /**
  * @param {AuthApi} authApi
@@ -26,12 +27,14 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye
  * @param {MapBackend} mapBackend
  * @param metadataBackend
  * @param {QueryTablesApi} queryTablesApi
+ * @param {OverviewsMetadataApi} overviewsMetadataApi
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
-function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi,
+function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
+                       queryTablesApi, overviewsMetadataApi,
                        surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
 
     BaseController.call(this, authApi, pgConnection);
@@ -41,11 +44,13 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.mapBackend = mapBackend;
     this.metadataBackend = metadataBackend;
     this.queryTablesApi = queryTablesApi;
+    this.overviewsMetadataApi = overviewsMetadataApi;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi);
 }
 
 util.inherits(MapController, BaseController);
@@ -148,6 +153,22 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
                 }
             );
         },
+        function addOverviewsInformation(err, requestMapConfig, datasource) {
+          assert.ifError(err);
+          var next = this;
+          self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
+              function(err, layers) {
+                  if (err) {
+                      return next(err);
+                  }
+
+                  if (layers) {
+                      requestMapConfig.layers = layers;
+                  }
+                  return next(null, requestMapConfig, datasource);
+              }
+          );
+        },
         function createLayergroup(err, requestMapConfig, datasource) {
             assert.ifError(err);
             mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource());
@@ -165,6 +186,8 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
             if (err) {
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
+                addWidgetsUrl(req.context.user, layergroup);
+
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -203,7 +226,23 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
             );
             mapConfigProvider.getMapConfig(this);
         },
-        function createLayergroup(err, mapConfig_, rendererParams/*, context*/) {
+        function addOverviewsInformation(err, requestMapConfig, rendererParams/*, context*/) {
+          assert.ifError(err);
+          var next = this;
+          self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
+              function(err, layers) {
+                  if (err) {
+                      return next(err);
+                  }
+
+                  if (layers) {
+                      requestMapConfig.layers = layers;
+                  }
+                  return next(null, requestMapConfig, rendererParams);
+              }
+          );
+        },
+        function createLayergroup(err, mapConfig_, rendererParams) {
             assert.ifError(err);
             mapConfig = mapConfig_;
             self.mapBackend.createLayergroup(
@@ -223,6 +262,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
+                addWidgetsUrl(req.context.user, layergroup);
+
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
 
@@ -306,9 +347,6 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
             layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
             layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
 
-            // TODO this should take into account several URL patterns
-            addWidgetsUrl(username, layergroup);
-
             if (req.method === 'GET') {
                 var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
                 var ttl = global.environment.varnish.layergroupTtl || 86400;

lib/cartodb/models/mapconfig_overviews_adapter.js

@@ -0,0 +1,53 @@
+var queue = require('queue-async');
+var _ = require('underscore');
+
+function MapConfigOverviewsAdapter(overviewsMetadataApi) {
+    this.overviewsMetadataApi = overviewsMetadataApi;
+}
+
+module.exports = MapConfigOverviewsAdapter;
+
+MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
+  var self = this;
+
+  if (!layers || layers.length === 0) {
+      return callback(null, layers);
+  }
+
+  var augmentLayersQueue = queue(layers.length);
+
+  function augmentLayer(layer, done) {
+      if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
+          return done(null, layer);
+      }
+      self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){
+           if (err) {
+               done(err, layer);
+           } else {
+               if ( !_.isEmpty(metadata) ) {
+                   layer = _.extend({}, layer);
+                   layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
+               }
+               done(null, layer);
+           }
+      });
+  }
+
+  function layersAugmentQueueFinish(err, layers) {
+    if (err) {
+        return callback(err);
+    }
+
+    if (!layers || layers.length === 0) {
+        return callback(new Error('Missing layers array from layergroup config'));
+    }
+
+    return callback(null, layers);
+  }
+
+  layers.forEach(function(layer) {
+      augmentLayersQueue.defer(augmentLayer, layer);
+  });
+  augmentLayersQueue.awaitAll(layersAugmentQueueFinish);
+
+};

lib/cartodb/server.js

@@ -20,6 +20,7 @@ var mapnik = windshaft.mapnik;
 
 var TemplateMaps = require('./backends/template_maps.js');
 var QueryTablesApi = require('./api/query_tables_api');
+var OverviewsMetadataApi = require('./api/overviews_metadata_api');
 var UserLimitsApi = require('./api/user_limits_api');
 var AuthApi = require('./api/auth_api');
 var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
@@ -52,6 +53,7 @@ module.exports = function(serverOptions) {
     var pgConnection = new PgConnection(metadataBackend);
     var pgQueryRunner = new PgQueryRunner(pgConnection);
     var queryTablesApi = new QueryTablesApi(pgQueryRunner);
+    var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
     var userLimitsApi = new UserLimitsApi(metadataBackend, {
         limits: {
             cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
@@ -175,6 +177,7 @@ module.exports = function(serverOptions) {
         mapBackend,
         metadataBackend,
         queryTablesApi,
+        overviewsMetadataApi,
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache

lib/cartodb/server_options.js

@@ -1,5 +1,10 @@
 var os = require('os');
 var _ = require('underscore');
+var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter');
+
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+  zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
+});
 
 var rendererConfig = _.defaults(global.environment.renderer || {}, {
     cache_ttl: 60000, // milliseconds
@@ -15,6 +20,8 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, {
     http: {}
 });
 
+rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter;
+
 // Perform keyword substitution in statsd
 // See https://github.com/CartoDB/Windshaft-cartodb/issues/153
 if ( global.environment.statsd ) {
@@ -61,7 +68,16 @@ module.exports = {
             statsInterval: rendererConfig.statsInterval
         },
         renderer: {
-            mapnik: rendererConfig.mapnik,
+            mapnik: _.defaults(rendererConfig.mapnik, {
+                geojson: {
+                    dbPoolParams: {
+                          size: 16,
+                          idleTimeout: 3000,
+                          reapInterval: 1000
+                    },
+                    clipByBox2d: false,
+                }
+            }),
             torque: rendererConfig.torque,
             http: rendererConfig.http
         },

lib/cartodb/utils/overviews_query_rewriter.js

@@ -0,0 +1,178 @@
+var TableNameParser = require('./table_name_parser');
+
+function OverviewsQueryRewriter(options) {
+
+    this.options = options;
+}
+
+module.exports = OverviewsQueryRewriter;
+
+// TODO: some names are introudced in the queries, and the
+// '_vovw_' (for vector overviews) is used in them, but no check
+// is performed for conflicts with existing identifiers in the query.
+
+// Build UNION expression to replace table, using overviews metadata
+// overviews metadata: { 1: 'table_ov1', ... }
+// assume table and overview names include schema if necessary and are quoted as needed
+function overviews_view_for_table(table, overviews_metadata, indent) {
+    var condition, i, len, ov_table, overview_layers, selects, z_hi, z_lo;
+    var parsed_table = TableNameParser.parse(table);
+
+    var sorted_overviews = []; // [[1, 'table_ov1'], ...]
+
+    indent = indent || '    ';
+    for (var z in overviews_metadata) {
+        if (overviews_metadata.hasOwnProperty(z)) {
+            sorted_overviews.push([z, overviews_metadata[z].table]);
+        }
+    }
+    sorted_overviews.sort(function(a, b){ return a[0]-b[0]; });
+
+    overview_layers = [];
+    z_lo = null;
+    for (i = 0, len = sorted_overviews.length; i < len; i++) {
+        z_hi = parseInt(sorted_overviews[i][0]);
+        ov_table = sorted_overviews[i][1];
+        overview_layers.push([overview_z_condition(z_lo, z_hi), ov_table]);
+        z_lo = z_hi;
+    }
+    overview_layers.push(["_vovw_z > " + z_lo, table]);
+
+    selects = overview_layers.map(function(condition_table) {
+      condition = condition_table[0];
+      ov_table = TableNameParser.parse(condition_table[1]);
+      ov_table.schema = ov_table.schema || parsed_table.schema;
+      var ov_identifier = TableNameParser.table_identifier(ov_table);
+      return indent + "SELECT * FROM " + ov_identifier + ", _vovw_scale WHERE " + condition;
+    });
+
+    return selects.join("\n"+indent+"UNION ALL\n");
+}
+
+function overview_z_condition(z_lo, z_hi) {
+    if (z_lo !== null) {
+        if (z_lo === z_hi - 1) {
+            return "_vovw_z = " + z_hi;
+        } else {
+            return "_vovw_z > " + z_lo + " AND _vovw_z <= " + z_hi;
+        }
+    } else {
+        if (z_hi === 0) {
+            return "_vovw_z = " + z_hi;
+        } else {
+            return "_vovw_z <= " + z_hi;
+        }
+    }
+}
+
+// name to be used for the view of the table using overviews
+function overviews_view_name(table) {
+    var parsed_table = TableNameParser.parse(table);
+    parsed_table.table = '_vovw_' + parsed_table.table;
+    parsed_table.schema = null;
+    return TableNameParser.table_identifier(parsed_table);
+}
+
+// replace a table name in a query by anoter name
+function replace_table_in_query(sql, old_table_name, replacement) {
+    var old_table = TableNameParser.parse(old_table_name);
+    var old_table_ident = TableNameParser.table_identifier(old_table);
+
+    // regular expression prefix (beginning) to match a table name
+    function pattern_prefix(schema, identifier) {
+        if ( schema ) {
+            // to match a table name including schema prefix
+            // name should not be part of another name, so we require
+            // to start a at a word boundary
+            if ( identifier[0] !== '"' ) {
+                return '\\b';
+            } else {
+                return '';
+            }
+        } else {
+           // to match a table name without schema
+           // name should not begin right after a dot (i.e. have a explicit schema)
+           // nor be part of another name
+           // since the pattern matches the first character of the table
+           // it must be put back in the replacement text
+           replacement = '$01'+replacement;
+           return '([^\.a-z0-9_]|^)';
+        }
+    }
+
+    // regular expression suffix (ending) to match a table name
+    function pattern_suffix(identifier) {
+        // name shouldn't be the prefix of a longer name
+        if ( identifier[identifier.length-1] !== '"' ) {
+            return '\\b';
+        } else {
+            return '';
+        }
+    }
+
+    // regular expression to match a table name
+    var regexp = pattern_prefix(old_table.schema, old_table_ident) +
+                 old_table_ident +
+                 pattern_suffix(old_table_ident);
+
+    // replace all occurrences of the table pattern
+    return sql.replace(new RegExp(regexp, 'g'), replacement);
+}
+
+function overviews_query(query, overviews, zoom_level_expression) {
+    var replaced_query = query;
+    var sql = "WITH\n  _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
+    var replacement;
+    for ( var table in overviews ) {
+        if (overviews.hasOwnProperty(table)) {
+            var table_overviews = overviews[table];
+            var table_view = overviews_view_name(table);
+            replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
+            replaced_query = replace_table_in_query(replaced_query, table, replacement);
+        }
+    }
+    if ( replaced_query !== query ) {
+        sql += "\n";
+        sql += replaced_query;
+    } else {
+        sql = query;
+    }
+    return sql;
+}
+
+// Transform an SQL query so that it uses overviews.
+// overviews contains metadata about the overviews to be used:
+//     { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
+//
+// For a given query `SELECT * FROM table`,  if any of tables in it
+// has overviews as defined by the provided metadat, the query will
+// be transform into something similar to this:
+//
+//     WITH _vovw_scale AS ( ... ), -- define scale level
+//     WITH _vovw_table AS ( ... ), -- define union of overviews and base table
+//     SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
+//
+// This transformation can in principle be applied to arbitrary queries
+// (except for the case of queries that include the name of tables with
+// overviews inside text literals: at the current table name substitution
+// doesnn't prevent substitution inside literals).
+// But the transformation will currently only be applied to simple queries
+// of the form detected by the overviews_supported_query function.
+OverviewsQueryRewriter.prototype.query = function(query, data) {
+    var overviews = this.overviews_metadata(data);
+    if ( !overviews || !this.is_supported_query(query)) {
+        return query;
+    }
+    var zoom_level_expression = this.options.zoom_level || '0';
+    return overviews_query(query, overviews, zoom_level_expression);
+};
+
+OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
+  return !!sql.match(
+    /^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
+  );
+};
+
+OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
+  return data && data.overviews;
+};

lib/cartodb/utils/table_name_parser.js

@@ -0,0 +1,106 @@
+// Quote an PostgreSQL identifier if ncecessary
+function quote_identifier_if_needed(txt) {
+    if ( txt && !txt.match(/^[a-z_][a-z_0-9]*$/)) {
+        return '"' + txt.replace(/\"/g, '""') + '"';
+    } else {
+        return txt;
+    }
+}
+
+// Parse PostgreSQL table name (possibly quoted and with optional schema).+
+// Returns { schema: 'schema_name', table: 'table_name' }
+function parse_table_name(table) {
+
+    function split_as_quoted_parts(table_name) {
+      // parse table into 'parts' that may be quoted, each part
+      // in the parts array being an object { part: 'text', quoted: false/true }
+     var parts = [];
+     var splitted = table_name.split(/\"/);
+     for (var i=0; i<splitted.length; i++ ) {
+       if ( splitted[i] === '' ) {
+         if ( parts.length > 0 && i < splitted.length-1 ) {
+             i++;
+             parts[parts.length - 1].part += '"' + splitted[i];
+         }
+       }
+       else {
+         var is_quoted = (i > 0 && splitted[i-1] === '') ||
+                         (i < splitted.length - 1 && splitted[i+1] === '');
+             parts.push({ part: splitted[i], quoted: is_quoted });
+       }
+     }
+     return parts;
+    }
+
+    var parts = split_as_quoted_parts(table);
+
+    function split_single_part(part) {
+        var schema_part = null;
+        var table_part = null;
+        if ( part.quoted ) {
+            table_part = part.part;
+        } else {
+            var parts = part.part.split('.');
+            if ( parts.length === 1 ) {
+                schema_part = null;
+                table_part = parts[0];
+            } else if ( parts.length === 2 ) {
+                schema_part = parts[0];
+                table_part = parts[1];
+            } // else invalid table name
+        }
+        return {
+            schema: schema_part,
+            table:  table_part
+        };
+    }
+
+    function split_two_parts(part1, part2) {
+        var schema_part = null;
+        var table_part = null;
+        if ( part1.quoted && !part2.quoted ) {
+            if ( part2.part[0] === '.' ) {
+                schema_part = part1.part;
+                table_part = part2.part.slice(1);
+            } // else invalid table name (missing dot)
+        } else if ( !part1.quoted && part2.quoted ) {
+            if ( part1.part[part1.part.length - 1] === '.' ) {
+                schema_part = part1.part.slice(0, -1);
+                table_part = part2.part;
+            } // else invalid table name (missing dot)
+        } // else invalid table name (missing dot)
+        return {
+            schema: schema_part,
+            table:  table_part
+        };
+    }
+
+    if ( parts.length === 1 ) {
+        return split_single_part(parts[0]);
+    } else if ( parts.length === 2 ) {
+        return split_two_parts(parts[0], parts[1]);
+    } else if ( parts.length === 3 && parts[1].part === '.' ) {
+        return {
+            schema: parts[0].part,
+            table:  parts[2].part
+        };
+    } // else invalid table name
+}
+
+function table_identifier(parsed_name) {
+    if ( parsed_name && parsed_name.table ) {
+        if ( parsed_name.schema ) {
+            return quote_identifier_if_needed(parsed_name.schema) + '.' + quote_identifier_if_needed(parsed_name.table);
+        } else {
+            return quote_identifier_if_needed(parsed_name.table);
+        }
+    } else {
+        return null;
+    }
+}
+
+module.exports = {
+  parse: parse_table_name,
+  quote: quote_identifier_if_needed,
+  table_identifier: table_identifier
+};

package.json

@@ -1,16 +1,13 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.20.0",
+    "version": "2.26.3",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
     ],
     "url": "https://github.com/CartoDB/Windshaft-cartodb",
-    "licenses": [{
-        "type": "BSD",
-        "url": "https://github.com/CartoDB/Windshaft-cartodb/blob/master/LICENCE"
-    }],
+    "license": "BSD-3-Clause",
     "repository": {
         "type": "git",
         "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
@@ -29,7 +26,7 @@
         "node-statsd": "~0.0.7",
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "1.7.0",
+        "windshaft": "1.13.2",
         "step": "~0.0.6",
         "queue-async": "~1.0.7",
         "request": "~2.62.0",
@@ -56,6 +53,6 @@
     },
     "engines": {
         "node": ">=0.8 <0.11",
-        "npm": ">=1.2.1"
+        "npm": ">=2.14.16"
     }
 }

test/acceptance/health_check.js

@@ -111,6 +111,6 @@ describe('health checks', function () {
 
             done();
         });
-    }); 
+    });
 
 });

test/acceptance/limits.js

@@ -292,7 +292,8 @@ describe('render limits', function() {
                                 if (err) {
                                     done(err);
                                 }
-                                assert.imageEqualsFile(res.body, './test/fixtures/render-timeout-fallback.png', 25,
+                                var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
+                                assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
                                     function(imgErr/*, similarity*/) {
                                         done(imgErr);
                                     }

test/acceptance/multilayer.js

@@ -52,14 +52,14 @@ describe(suiteName, function() {
            { options: {
                sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
                    ' from test_table limit 2',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
                cartocss_version: '2.0.1',
                interactivity: 'cartodb_id'
              } },
            { options: {
                sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator' +
                    ' from test_table limit 2 offset 2',
-               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                cartocss_version: '2.0.2',
                interactivity: 'cartodb_id'
              } }
@@ -104,7 +104,7 @@ describe(suiteName, function() {
 
               // Check X-Cache-Channel
               cc = res.headers['x-cache-channel'];
-              assert.ok(cc); 
+              assert.ok(cc);
               var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
@@ -122,7 +122,7 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png',
+              assert.imageBufferIsSimilarToFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png',
                   IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
                       next(err);
                   }
@@ -190,38 +190,50 @@ describe(suiteName, function() {
     });
 
 
-    it("should include serverMedata in the response", function(done) {
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
-      var layergroup =  {
-        version: '1.0.0',
-        layers: [
-           { options: {
-               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
-                   ' from test_table limit 2',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
-               cartocss_version: '2.0.1'
-             } }
-        ]
-      };
+    describe('server-metadata', function() {
+        var serverMetadata;
+        beforeEach(function() {
+            serverMetadata = global.environment.serverMetadata;
+            global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
+        });
+
+        afterEach(function() {
+            global.environment.serverMetadata = serverMetadata;
+        });
+
+        it("should include serverMedata in the response", function(done) {
+          var layergroup =  {
+            version: '1.0.0',
+            layers: [
+               { options: {
+                   sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
+                       ' from test_table limit 2',
+                   cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+                   cartocss_version: '2.0.1'
+                 } }
+            ]
+          };
+
+          step(
+            function do_create_get()
+            {
+              var next = this;
+              assert.response(server, {
+                  url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+                  method: 'GET',
+                  headers: {host: 'localhost'}
+              }, {}, function(res, err) { next(err, res); });
+            },
+            function do_check_create(err, res) {
+              var parsed = JSON.parse(res.body);
+              keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
+              keysToDelete['user:localhost:mapviews:global'] = 5;
+              assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+              done();
+            }
+          );
+        });
 
-      step(
-        function do_create_get()
-        {
-          var next = this;
-          assert.response(server, {
-              url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
-              method: 'GET',
-              headers: {host: 'localhost'} 
-          }, {}, function(res, err) { next(err, res); });
-        },
-        function do_check_create(err, res) {
-          var parsed = JSON.parse(res.body);
-          keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-          keysToDelete['user:localhost:mapviews:global'] = 5;
-          assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-          done();
-        }
-      );
     });
 
 
@@ -245,7 +257,7 @@ describe(suiteName, function() {
             ]
         };
 
-      var expected_token; 
+      var expected_token;
       step(
         function do_create_get()
         {
@@ -253,7 +265,7 @@ describe(suiteName, function() {
           assert.response(server, {
               url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
               method: 'GET',
-              headers: {host: 'localhost'} 
+              headers: {host: 'localhost'}
           }, {}, function(res, err) { next(err, res); });
         },
         function do_check_create(err, res) {
@@ -329,7 +341,7 @@ describe(suiteName, function() {
            { options: {
                sql: 'select 1 as cartodb_id, ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!))' +
                    ' as the_geom_webmercator from test_table limit 1',
-               cartocss: '#layer { polygon-fill:red; }', 
+               cartocss: '#layer { polygon-fill:red; }',
                cartocss_version: '2.0.1',
                interactivity: 'cartodb_id'
              } }
@@ -392,7 +404,8 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = 'test/fixtures/test_multilayer_bbox.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err/*, similarity*/) {
                   next(err);
               });
@@ -413,7 +426,7 @@ describe(suiteName, function() {
 
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
-              assert.ok(cc); 
+              assert.ok(cc);
               var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API
@@ -431,7 +444,8 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = 'test/fixtures/test_multilayer_bbox.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err/*, similarity*/) {
                   next(err);
               });
@@ -485,8 +499,8 @@ describe(suiteName, function() {
            { options: {
                sql: 'select 1 as cartodb_id, !pixel_height! as h,' +
                    ' ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
-               cartocss: '#layer { polygon-fill:red; }', 
-               cartocss_version: '2.0.1' 
+               cartocss: '#layer { polygon-fill:red; }',
+               cartocss_version: '2.0.1'
              } }
         ]
       };
@@ -576,10 +590,10 @@ describe(suiteName, function() {
         version: '1.0.0',
         layers: [
            { options: {
-               sql: 'select 1 as cartodb_id, !pixel_height! as h' +
+               sql: 'select 1 as cartodb_id, !pixel_height! as h,' +
                    'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
-               cartocss: '#layer { polygon-fit:red; }', 
-               cartocss_version: '2.0.1' 
+               cartocss: '#layer { polygon-fit:red; }',
+               cartocss_version: '2.0.1'
              } }
         ]
       };
@@ -606,8 +620,8 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: 'select bogus(0,0) as the_geom_webmercator',
-               cartocss: '#layer { polygon-fill:red; }', 
-               cartocss_version: '2.0.1' 
+               cartocss: '#layer { polygon-fill:red; }',
+               cartocss_version: '2.0.1'
              } }
         ]
       };
@@ -617,7 +631,7 @@ describe(suiteName, function() {
           headers: {host: 'localhost', 'Content-Type': 'application/json' },
           data: JSON.stringify(layergroup)
       }, {}, function(res) {
-          assert.equal(res.statusCode, 404, res.statusCode + ": " + res.body);
+          assert.equal(res.statusCode, 400, res.statusCode + ": " + res.body);
           var parsed = JSON.parse(res.body);
           var msg = parsed.errors[0];
           assert.ok(msg.match(/bogus.*exist/), msg);
@@ -633,13 +647,13 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: 'select * from test_table_private_1 where cartodb_id=1',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
                cartocss_version: '2.1.0',
                interactivity: 'cartodb_id'
              } },
            { options: {
                sql: 'select * from test_table_private_1 where cartodb_id=2',
-               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                cartocss_version: '2.1.0',
                interactivity: 'cartodb_id'
              } }
@@ -684,7 +698,7 @@ describe(suiteName, function() {
 
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
-              assert.ok(cc); 
+              assert.ok(cc);
               var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               next(err);
@@ -780,7 +794,7 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: 'select * from test_table where cartodb_id=1',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
                cartocss_version: '2.1.0',
                interactivity: 'cartodb_id'
              } }
@@ -830,7 +844,7 @@ describe(suiteName, function() {
 
           // Check X-Cache-Channel
           var cc = res.headers['x-cache-channel'];
-          assert.ok(cc, "Missing X-Cache-Channel"); 
+          assert.ok(cc, "Missing X-Cache-Channel");
           var dbname = test_database;
           assert.equal(cc.substring(0, dbname.length), dbname);
           return null;
@@ -965,7 +979,7 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: "select 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
-               cartocss: '#layer { point-transform:"scale(20)"; }', 
+               cartocss: '#layer { point-transform:"scale(20)"; }',
                cartocss_version: '2.0.1'
              } }
         ]
@@ -1007,7 +1021,7 @@ describe(suiteName, function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
+              assert.imageBufferIsSimilarToFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
                       next(err);
                   }
@@ -1031,7 +1045,7 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: "select * from test_table_private_1 LIMIT 0",
-               cartocss: '#layer { marker-fill:red; }', 
+               cartocss: '#layer { marker-fill:red; }',
                cartocss_version: '2.0.1'
              } }
         ]
@@ -1105,7 +1119,7 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: sql,
-               cartocss: '#layer { marker-fill:red; }', 
+               cartocss: '#layer { marker-fill:red; }',
                cartocss_version: '2.0.1'
              } }
         ]
@@ -1194,7 +1208,7 @@ describe(suiteName, function() {
         layers: [
            { options: {
                sql: "select *, 'SQLAPINOANSWER' from test_table",
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
                cartocss_version: '2.1.0'
              } }
         ]
@@ -1318,7 +1332,40 @@ describe(suiteName, function() {
             }
         );
     });
-    
+
+    it('should response to empty layers mapconfig', function(done) {
+        var layergroup =  {
+            layers: []
+        };
+
+        assert.response(
+            server,
+            {
+                url: '/api/v1/map',
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(layergroup)
+            },
+            {
+                status: 200
+            },
+            function(res, err) {
+                assert.ok(!err);
+
+                var parsedBody = JSON.parse(res.body);
+                assert.ok(parsedBody.layergroupid);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                done();
+            }
+        );
+    });
+
 });
 
 });

test/acceptance/multilayer_server.js

@@ -332,13 +332,9 @@ describe('tests from old api translated to multilayer', function() {
 
                 assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
 
-                // TODO when affected tables query makes the request to fail layergroup should be removed
-                keysToDelete['map_cfg|4fb7bd7008322ce66f22d20aebba1ab0'] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
-
                 var parsed = JSON.parse(res.body);
                 assert.deepEqual(parsed, {
-                    errors: ["Error: could not fetch affected tables or last updated time: fake error message"]
+                    errors: ["fake error message"]
                 });
 
                 done();

test/acceptance/overviews_metadata.js

@@ -0,0 +1,110 @@
+var test_helper = require('../support/test_helper');
+
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+
+var RedisPool = require('redis-mpool');
+
+var step = require('step');
+
+var windshaft = require('windshaft');
+
+
+describe('overviews metadata', function() {
+    // configure redis pool instance to use in tests
+    var redisPool = new RedisPool(global.environment.redis);
+
+    var overviews_layer = {
+        type: 'cartodb',
+        options: {
+            sql: 'SELECT * FROM test_table_overviews',
+            cartocss: '#layer { marker-fill: black; }',
+            cartocss_version: '2.3.0'
+        }
+    };
+
+    var non_overviews_layer = {
+        type: 'cartodb',
+        options: {
+            sql: 'SELECT * FROM test_table',
+            cartocss: '#layer { marker-fill: black; }',
+            cartocss_version: '2.3.0'
+        }
+    };
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        test_helper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    it("layers with and without overviews", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [overviews_layer, non_overviews_layer]
+      };
+
+      var layergroup_url = '/api/v1/map';
+
+      var expected_token;
+      step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: layergroup_url,
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
+              expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_mapconfig(err)
+        {
+            assert.ifError(err);
+            var next = this;
+
+            var mapStore  = new windshaft.storage.MapStore({
+                pool: redisPool,
+                expire_time: 500000
+            });
+            mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) {
+              assert.ifError(err);
+              assert.deepEqual(non_overviews_layer, mapConfig._cfg.layers[1]);
+              assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
+              assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
+              var expected_data = {
+                overviews: {
+                  test_table_overviews: {
+                    1: { table: '_vovw_1_test_table_overviews' },
+                    2: { table: '_vovw_2_test_table_overviews' }
+                  }
+                }
+              };
+              assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
+            });
+
+            next(err);
+        },
+        function finish(err) {
+            keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
+            keysToDelete['user:localhost:mapviews:global'] = 5;
+            done(err);
+        }
+      );
+    });
+});

test/acceptance/overviews_queries.js

@@ -0,0 +1,73 @@
+var testHelper = require('../support/test_helper');
+var assert = require('../support/assert');
+
+var cartodbServer = require('../../lib/cartodb/server');
+var ServerOptions = require('./ported/support/ported_server_options');
+var testClient = require('./ported/support/test_client');
+var BaseController = require('../../lib/cartodb/controllers/base');
+
+describe('overviews_queries', function() {
+
+    var server = cartodbServer(ServerOptions);
+    server.setMaxListeners(0);
+
+    var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2;
+
+    var req2paramsFn;
+    before(function() {
+        req2paramsFn = BaseController.prototype.req2params;
+        BaseController.prototype.req2params = ServerOptions.req2params;
+    });
+
+    after(function() {
+        BaseController.prototype.req2params = req2paramsFn;
+
+        testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir);
+    });
+
+    function imageCompareFn(fixture, done) {
+        return function(err, tile) {
+            if (err) {
+                return done(err);
+            }
+            var referenceImagePath = './test/fixtures/' + fixture;
+            assert.imageBufferIsSimilarToFile(tile.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+        };
+    }
+
+    it("should not use overview for tables without overviews", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table'), 1, 0, 0,
+          imageCompareFn('test_table_1_0_0.png', done)
+      );
+    });
+
+    it("should not use overview for tables without overviews at z=2", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table'), 2, 1, 1,
+          imageCompareFn('test_table_2_1_1.png', done)
+      );
+    });
+
+    it("should not use overview for tables without overviews at z=2", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table'), 3, 3, 3,
+          imageCompareFn('test_table_3_3_3.png', done)
+      );
+    });
+
+    it("should use overview for zoom level 1", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 1, 0, 0,
+          imageCompareFn('_vovw_1_test_table_1_0_0.png', done)
+      );
+    });
+
+    it("should use overview for zoom level 1", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 2, 1, 1,
+          imageCompareFn('_vovw_2_test_table_2_1_1.png', done)
+      );
+    });
+
+    it("should not use overview for zoom level 3", function(done){
+      testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 3, 3, 3,
+          imageCompareFn('test_table_3_3_3.png', done)
+      );
+    });
+});

test/acceptance/ported/blend.js

@@ -95,10 +95,12 @@ describe('blend png renderer', function() {
         var zxy = [tileRequest.z, tileRequest.x, tileRequest.y];
         it('tile all/' + zxy.join('/') + '.png', function (done) {
             testClient.getTileLayer(plainTorqueMapConfig(testScenario.plainColor), tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(zxy), IMAGE_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(zxy), IMAGE_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/blend_filtering.js

@@ -156,10 +156,12 @@ describe('blend layer filtering', function() {
 
         it('should filter on ' + layerFilter + '/1/0/0.png', function (done) {
             testClient.getTileLayer(mapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/blend_http_fallback.js

@@ -111,10 +111,12 @@ describe('blend http fallback', function() {
 
         it('should fallback on http error while blending layers ' + layerFilter + '/1/0/0.png', function (done) {
             testClient.getTileLayer(mapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err, err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err, err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/external_resources.js

@@ -57,7 +57,8 @@ describe('external resources', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            var referenceImagePath = './test/fixtures/' + fixture;
+            assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
         };
     }
 

test/acceptance/ported/limits.js

@@ -84,11 +84,13 @@ describe.skip('render limits', function() {
         testClient.withLayergroup(slowQueryMapConfig, options, function(err, requestTile, finish) {
             var tileUrl = '/0/0/0.png';
             requestTile(tileUrl, options, function(err, res) {
-                assert.imageEqualsFile(res.body, fixtureImage, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
-                    finish(function(finishErr) {
-                        done(err || finishErr);
-                    });
-                });
+                assert.imageBufferIsSimilarToFile(res.body, fixtureImage, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                    function(err) {
+                        finish(function(finishErr) {
+                            done(err || finishErr);
+                        });
+                    }
+                );
             });
         });
     });

test/acceptance/ported/multilayer.js

@@ -117,7 +117,8 @@ describe('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
               checkCORSHeaders(res);
-              assert.imageEqualsFile(res.body, './test/fixtures/test_bigpoint_red.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = './test/fixtures/test_bigpoint_red.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                   function(err) {
                   next(err);
               });
@@ -191,7 +192,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -302,7 +304,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -400,12 +403,11 @@ describe('multilayer', function() {
           function jsonp_test(body) {
               assert.ok(body.layergroupid);
               expected_token = LayergroupToken.parse(body.layergroupid).token;
-              assert.deepEqual(body.metadata, {
-                  layers: [
-                      { type: "mapnik", "meta":{} },
-                      { type: "mapnik", "meta":{} }
-                  ]
-              });
+              assert.ok(body.metadata.layers.length === 2);
+              assert.ok(body.metadata.layers[0].type === 'mapnik');
+              assert.ok(body.metadata.layers[0].meta);
+              assert.ok(body.metadata.layers[1].type === 'mapnik');
+              assert.ok(body.metadata.layers[1].meta);
               didRunJsonCallback = true;
           }
           eval(res.body);
@@ -426,7 +428,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -542,7 +545,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -728,7 +732,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer2.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer2.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -762,7 +767,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer3.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer3.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -857,7 +863,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer4.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer4.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -1260,7 +1267,7 @@ describe('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
               checkCORSHeaders(res);
-              assert.imageEqualsFile(res.body, './test/fixtures/test_bigpoint_red.png',
+              assert.imageBufferIsSimilarToFile(res.body, './test/fixtures/test_bigpoint_red.png',
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -1278,4 +1285,3 @@ describe('multilayer', function() {
     });
 
 });
-

test/acceptance/ported/multilayer_interactivity.js

@@ -374,4 +374,3 @@ describe('multilayer interactivity and layers order', function() {
     chaosScenarios.forEach(testInteractivityLayersOrderScenario);
 
 });
-

test/acceptance/ported/raster.js

@@ -87,7 +87,7 @@ describe('raster', function() {
           assert.equal(res.statusCode, 200, res.body);
           assert.deepEqual(res.headers['content-type'], "image/png");
           var next = this;
-          assert.imageEqualsFile(res.body,
+          assert.imageBufferIsSimilarToFile(res.body,
             './test/fixtures/raster_gray_rect.png',
             IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
               try {

test/acceptance/ported/server_gettile.js

@@ -34,7 +34,9 @@ describe('server_gettile', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            assert.imageBufferIsSimilarToFile(
+                res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done
+            );
         };
     }
 
@@ -113,12 +115,13 @@ describe('server_gettile', function() {
                 assert.ok(res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time");
                 assert.ok(res.headers['x-windshaft-cache'] >= 0);
 
-                assert.imageEqualsFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
-
-                    finish(function(finishErr) {
-                        done(err || finishErr);
-                    });
-                });
+                assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                    function(err) {
+                        finish(function(finishErr) {
+                            done(err || finishErr);
+                        });
+                    }
+                );
             });
         });
     });

test/acceptance/ported/server_png8_format.js

@@ -108,7 +108,7 @@ describe('server_png8_format', function() {
                             assert.equal(responsePng8.headers['content-type'], "image/png");
                             bufferPng8 = responsePng8.body;
                             assert.ok(bufferPng8.length < bufferPng32.length);
-                            assert.imageBuffersAreEqual(bufferPng32, bufferPng8, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                            assert.imageBuffersAreSimilar(bufferPng32, bufferPng8, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                                 function(err, imagePaths, similarity) {
 
                                     keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;

test/acceptance/ported/support/ported_server_options.js

@@ -2,6 +2,10 @@ var _ = require('underscore');
 var serverOptions = require('../../../../lib/cartodb/server_options');
 var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token');
 var mapnik = require('windshaft').mapnik;
+var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter');
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+  zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
+});
 
 module.exports = _.extend({}, serverOptions, {
     base_url: '/database/:dbname/table/:table',
@@ -26,7 +30,8 @@ module.exports = _.extend({}, serverOptions, {
             limits: {
                 render: 0,
                 cacheOnTimeout: true
-            }
+            },
+            queryRewriter: overviewsQueryRewriter
         },
         http: {
             timeout: 5000,

test/acceptance/ported/torque.js

@@ -418,7 +418,7 @@ describe('torque', function() {
           assert.response(server, {
               url: '/database/windshaft_test/layergroup',
               method: 'POST',
-              headers: {'Content-Type': 'application/json' }, 
+              headers: {'Content-Type': 'application/json' },
               data: JSON.stringify(layergroup)
           }, {}, function(res) { next(null, res); });
         },

test/acceptance/ported/torque_png.js

@@ -83,10 +83,12 @@ describe('torque png renderer', function() {
         var zxy = [z, x, y];
         it('tile ' + zxy.join('/') + '.torque.png', function (done) {
             testClient.getTileLayer(torquePngPointsMapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, torquePngFixture(zxy), IMAGE_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, torquePngFixture(zxy), IMAGE_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/wrap.js

@@ -107,10 +107,12 @@ describe('wrap x coordinate', function() {
             var fixtureZxy = [testScenario.fixture.z, testScenario.fixture.x, testScenario.fixture.y];
             it('tile all/' + zxy.join('/') + '.png', function (done) {
                 testClient.getTileLayer(plainTorqueMapConfig(testScenario.plainColor), tileRequest, function(err, res) {
-                    assert.imageEqualsFile(res.body, blendPngFixture(fixtureZxy), IMG_TOLERANCE_PER_MIL, function(err) {
-                        assert.ok(!err);
-                        done();
-                    });
+                    assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(fixtureZxy), IMG_TOLERANCE_PER_MIL,
+                        function(err) {
+                            assert.ok(!err);
+                            done();
+                        }
+                    );
                 });
             });
         });

test/acceptance/templates.js

@@ -65,7 +65,7 @@ describe('template_api', function() {
              { options: {
                  sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator' +
                      ' from test_table limit 2 offset 2',
-                 cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                 cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                  cartocss_version: '2.0.2',
                  interactivity: 'cartodb_id'
                } }
@@ -313,51 +313,63 @@ describe('template_api', function() {
       });
     });
 
-    it("instance endpoint should return server metadata", function(done){
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
-      var tmpl = _.clone(template_acceptance1);
-      tmpl.name = "rambotemplate2";
+    describe('server-metadata', function() {
+        var serverMetadata;
+        beforeEach(function() {
+            serverMetadata = global.environment.serverMetadata;
+            global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
+        });
 
-      step(function postTemplate1() {
-          var next = this;
-          var post_request = {
-              url: '/api/v1/map/named?api_key=1234',
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' },
-              data: JSON.stringify(tmpl)
-          };
-          assert.response(server, post_request, {}, function(res) { 
-            next(null, res); 
-          });
-        },
-        function testCORS() {
-          var next = this;
-          assert.response(server, {
-              url: '/api/v1/map/named/' + tmpl.name,
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          },{
-              status: 200
-          }, function(res) { 
-            var parsed = JSON.parse(res.body);
-            keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-            keysToDelete['user:localhost:mapviews:global'] = 5;
-            assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-            next(null); 
-          });
-        },
-        function deleteTemplate(err) {
-            assert.ifError(err);
-          var del_request = {
-              url: '/api/v1/map/named/' + tmpl.name + '?api_key=1234',
-              method: 'DELETE',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          };
-          assert.response(server, del_request, {}, function() {
-              done();
-          });
-        }
-      );
+        afterEach(function() {
+            global.environment.serverMetadata = serverMetadata;
+        });
+
+
+        it("instance endpoint should return server metadata", function(done){
+          var tmpl = _.clone(template_acceptance1);
+          tmpl.name = "rambotemplate2";
+
+          step(function postTemplate1() {
+              var next = this;
+              var post_request = {
+                  url: '/api/v1/map/named?api_key=1234',
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' },
+                  data: JSON.stringify(tmpl)
+              };
+              assert.response(server, post_request, {}, function(res) {
+                next(null, res);
+              });
+            },
+            function testCORS() {
+              var next = this;
+              assert.response(server, {
+                  url: '/api/v1/map/named/' + tmpl.name,
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' }
+              },{
+                  status: 200
+              }, function(res) {
+                var parsed = JSON.parse(res.body);
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+                next(null);
+              });
+            },
+            function deleteTemplate(err) {
+                assert.ifError(err);
+              var del_request = {
+                  url: '/api/v1/map/named/' + tmpl.name + '?api_key=1234',
+                  method: 'DELETE',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' }
+              };
+              assert.response(server, del_request, {}, function() {
+                  done();
+              });
+            }
+          );
+        });
     });
 
 
@@ -720,7 +732,7 @@ describe('template_api', function() {
             layers: [
                { options: {
                    sql: "select * from test_table_private_1 LIMIT 0",
-                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                    cartocss_version: '2.0.2',
                    interactivity: 'cartodb_id'
                  } }
@@ -848,7 +860,7 @@ describe('template_api', function() {
         },
         function checkTile(err, res) {
             assert.ifError(err);
-          assert.equal(res.statusCode, 200, 
+          assert.equal(res.statusCode, 200,
             'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "image/png");
           return null;
@@ -869,7 +881,7 @@ describe('template_api', function() {
         },
         function checkForeignSignerError(err, res) {
             assert.ifError(err);
-          assert.equal(res.statusCode, 403, 
+          assert.equal(res.statusCode, 403,
             'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('errors'),
@@ -1036,7 +1048,7 @@ describe('template_api', function() {
         },
         function checkTile_fetchOnRestart(err, res) {
             assert.ifError(err);
-          assert.equal(res.statusCode, 200, 
+          assert.equal(res.statusCode, 200,
             'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
           var cc = res.headers['x-cache-channel'];
@@ -1056,7 +1068,7 @@ describe('template_api', function() {
         },
         function checkCacheChannel(err, res) {
             assert.ifError(err);
-          assert.equal(res.statusCode, 200, 
+          assert.equal(res.statusCode, 200,
             'Unexpected error for authorized instance: ' + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
           var cc = res.headers['x-cache-channel'];
@@ -1112,7 +1124,7 @@ describe('template_api', function() {
             layers: [
                { options: {
                    sql: "select * from test_table_private_1 where cartodb_id in ( 5,6 )",
-                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                    cartocss_version: '2.0.2',
                    attributes: { id:'cartodb_id', columns: ['name', 'address'] }
                  } }
@@ -1224,7 +1236,7 @@ describe('template_api', function() {
         },
         function checkAttribute(err, res) {
             assert.ifError(err);
-          assert.equal(res.statusCode, 200, 
+          assert.equal(res.statusCode, 200,
             'Unexpected error for authorized getAttributes: ' + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
           return null;
@@ -1277,7 +1289,7 @@ describe('template_api', function() {
             layers: [
                { options: {
                    sql: "select * from test_table_private_1 LIMIT 0",
-                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                    cartocss_version: '2.0.2',
                    interactivity: 'cartodb_id'
                  } }
@@ -1347,7 +1359,7 @@ describe('template_api', function() {
             layers: [
                { options: {
                    sql: "select * from test_table_private_1 LIMIT 0",
-                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                    cartocss_version: '2.0.2',
                    interactivity: 'cartodb_id'
                  } }
@@ -1430,7 +1442,7 @@ describe('template_api', function() {
             layers: [
                { options: {
                    sql: "select * from test_table_private_1 LIMIT 0",
-                   cartocss: '#layer { marker-fill: <%= color %>; marker-allow-overlap:true; }', 
+                   cartocss: '#layer { marker-fill: <%= color %>; marker-allow-overlap:true; }',
                    cartocss_version: '2.0.2',
                    interactivity: 'cartodb_id'
                  } }
@@ -1503,8 +1515,8 @@ describe('template_api', function() {
            { options: {
                sql: 'select 1 as cartodb_id, !pixel_height! as h,' +
                    ' ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
-               cartocss: '#layer { polygon-fill:red; }', 
-               cartocss_version: '2.0.1' 
+               cartocss: '#layer { polygon-fill:red; }',
+               cartocss_version: '2.0.1'
              } }
         ]
       };
@@ -1516,7 +1528,7 @@ describe('template_api', function() {
       };
       var statskey = "user:localhost:mapviews";
       var redis_stats_client = redis.createClient(global.environment.redis.port);
-      var template_id; // will be set on template post 
+      var template_id; // will be set on template post
       var now = strftime("%Y%m%d", new Date());
       var errors = [];
       step(
@@ -1621,7 +1633,7 @@ describe('template_api', function() {
             layers: [
                { options: {
                    sql: "select * from test_table_private_1 LIMIT 0",
-                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
                    cartocss_version: '2.0.2',
                    interactivity: 'cartodb_id'
                  } }

test/acceptance/widgets.js

@@ -1,479 +0,0 @@
-var assert = require('../support/assert');
-var step = require('step');
-var qs = require('querystring');
-
-var helper = require(__dirname + '/../support/test_helper');
-var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
-
-var CartodbWindshaft = require('../../lib/cartodb/server');
-var serverOptions = require('../../lib/cartodb/server_options');
-var server = new CartodbWindshaft(serverOptions);
-server.setMaxListeners(0);
-
-
-describe('widgets', function() {
-
-    var keysToDelete;
-
-    beforeEach(function() {
-        keysToDelete = {};
-    });
-
-    afterEach(function(done) {
-        helper.deleteRedisKeys(keysToDelete, done);
-    });
-
-    function getWidget(mapConfig, widgetName, params, callback) {
-        if (!callback) {
-            callback = params;
-            params = {};
-        }
-
-        var url = '/api/v1/map';
-        if (params && params.filters) {
-            url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
-        }
-
-        var layergroupId;
-        step(
-            function createLayergroup() {
-                var next = this;
-                assert.response(server,
-                    {
-                        url: url,
-                        method: 'POST',
-                        headers: {
-                            host: 'localhost',
-                            'Content-Type': 'application/json'
-                        },
-                        data: JSON.stringify(mapConfig)
-                    },
-                    {
-                        status: 200,
-                        headers: {
-                            'Content-Type': 'application/json; charset=utf-8'
-                        }
-                    },
-                    function(res, err) {
-                        if (err) {
-                            return next(err);
-                        }
-                        var parsedBody = JSON.parse(res.body);
-                        var expectedWidgetURLS = {
-                            http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
-                        };
-                        assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
-                        assert.ok(
-                            parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
-                        );
-                        return next(null, parsedBody.layergroupid);
-                    }
-                );
-            },
-            function getWidgetResult(err, _layergroupId) {
-                assert.ifError(err);
-
-                var next = this;
-                layergroupId = _layergroupId;
-
-                var urlParams = {
-                    own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
-                };
-                if (params && params.bbox) {
-                    urlParams.bbox = params.bbox;
-                }
-                url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
-
-                assert.response(server,
-                    {
-                        url: url,
-                        method: 'GET',
-                        headers: {
-                            host: 'localhost'
-                        }
-                    },
-                    {
-                        status: 200,
-                        headers: {
-                            'Content-Type': 'application/json; charset=utf-8'
-                        }
-                    },
-                    function(res, err) {
-                        if (err) {
-                            return next(err);
-                        }
-
-                        next(null, res);
-                    }
-                );
-            },
-            function finish(err, res) {
-                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
-                return callback(err, res);
-            }
-        );
-    }
-
-
-    it("should expose layer list", function(done) {
-
-        var listWidgetMapConfig =  {
-            version: '1.5.0',
-            layers: [
-                {
-                    type: 'mapnik',
-                    options: {
-                        sql: 'select * from test_table',
-                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                        cartocss_version: '2.3.0',
-                        widgets: {
-                            names: {
-                                type: 'list',
-                                options: {
-                                    columns: ['name']
-                                }
-                            }
-                        }
-                    }
-                }
-            ]
-        };
-
-        getWidget(listWidgetMapConfig, 'names', function(err, res) {
-            if (err) {
-                return done(err);
-            }
-
-            var expectedList = [
-                {name:"Hawai"},
-                {name:"El Estocolmo"},
-                {name:"El Rey del Tallarín"},
-                {name:"El Lacón"},
-                {name:"El Pico"}
-            ];
-            assert.deepEqual(JSON.parse(res.body).rows, expectedList);
-
-            done();
-        });
-    });
-
-    it("should expose layer histogram", function(done) {
-        var histogramMapConfig =  {
-            version: '1.5.0',
-            layers: [
-                {
-                    type: 'mapnik',
-                    options: {
-                        sql: 'select * from populated_places_simple_reduced',
-                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                        cartocss_version: '2.3.0',
-                        widgets: {
-                            pop_max: {
-                                type: 'histogram',
-                                options: {
-                                    column: 'pop_max'
-                                }
-                            }
-                        }
-                    }
-                }
-            ]
-        };
-        getWidget(histogramMapConfig, 'pop_max', function(err, res) {
-            if (err) {
-                return done(err);
-            }
-
-            var histogram = JSON.parse(res.body);
-            assert.ok(histogram.bins.length);
-
-            done();
-        });
-    });
-
-    describe('filters', function() {
-
-        describe('category', function() {
-            var aggregationMapConfig =  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        type: 'mapnik',
-                        options: {
-                            sql: 'select * from populated_places_simple_reduced',
-                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                            cartocss_version: '2.3.0',
-                            widgets: {
-                                country_places_count: {
-                                    type: 'aggregation',
-                                    options: {
-                                        column: 'adm0_a3',
-                                        aggregation: 'count'
-                                    }
-                                }
-                            }
-                        }
-                    }
-                ]
-            };
-
-            it("should expose an aggregation", function(done) {
-                getWidget(aggregationMapConfig, 'country_places_count', { own_filter: 0 }, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-                    assert.equal(aggregation.categories.length, 6);
-                    assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false });
-
-                    done();
-                });
-            });
-
-            it("should expose a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {country_places_count: {accept: ['CAN']}}
-                        ]
-                    }
-                };
-                getWidget(aggregationMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-                    assert.equal(aggregation.categories.length, 1);
-                    assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false });
-
-                    done();
-                });
-            });
-        });
-
-        describe('range', function() {
-            var histogramMapConfig =  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        type: 'mapnik',
-                        options: {
-                            sql: 'select * from populated_places_simple_reduced',
-                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                            cartocss_version: '2.3.0',
-                            widgets: {
-                                country_places_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'pop_max'
-                                    }
-                                }
-                            }
-                        }
-                    }
-                ]
-            };
-
-            it("should expose an histogram", function(done) {
-                getWidget(histogramMapConfig, 'country_places_histogram', { own_filter: 0 }, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var histogram = JSON.parse(res.body);
-                    // notice min value
-                    assert.deepEqual(
-                        histogram.bins[0],
-                        { bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 }
-                    );
-
-                    done();
-                });
-            });
-
-            it("should expose a filtered histogram", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_histogram: { min: 4000000 }
-                            }
-                        ]
-                    }
-                };
-                getWidget(histogramMapConfig, 'country_places_histogram', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var histogram = JSON.parse(res.body);
-                    // notice min value
-                    assert.deepEqual(histogram.bins[0], {
-                        bin: 0,
-                        freq: 62,
-                        min: 4000000,
-                        max: 9276403,
-                        avg: 5815009.596774193
-                    });
-
-                    done();
-                });
-            });
-        });
-
-        describe('combine widget filters', function() {
-            var combinedWidgetsMapConfig =  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        type: 'mapnik',
-                        options: {
-                            sql: 'select * from populated_places_simple_reduced',
-                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
-                            cartocss_version: '2.3.0',
-                            widgets: {
-                                country_places_count: {
-                                    type: 'aggregation',
-                                    options: {
-                                        column: 'adm0_a3',
-                                        aggregation: 'count'
-                                    }
-                                },
-                                country_places_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'pop_max'
-                                    }
-                                }
-                            }
-                        }
-                    }
-                ]
-            };
-
-            it("should expose a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_count: { reject: ['CHN'] }
-                            }
-                        ]
-                    }
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-
-            it("should expose a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_count: { reject: ['CHN'] },
-                                country_places_histogram: { min: 7000000 }
-                            }
-                        ]
-                    }
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-
-            it("should allow to filter by bounding box a filtered aggregation", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_histogram: { min: 50000 }
-                            }
-                        ]
-                    },
-                    bbox: '-20,0,45,60'
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-
-            it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) {
-                var params = {
-                    filters: {
-                        layers: [
-                            {
-                                country_places_count: { reject: ['RUS'] },
-                                country_places_histogram: { min: 50000 }
-                            }
-                        ]
-                    },
-                    bbox: '-20,0,45,60'
-                };
-                getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
-                    if (err) {
-                        return done(err);
-                    }
-
-                    var aggregation = JSON.parse(res.body);
-
-                    // first one would be CHN if reject filter wasn't applied
-                    assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false });
-
-                    // confirm 'CHN' was filtered out (reject)
-                    assert.equal(aggregation.categories.reduce(function(sum, row) {
-                        return sum + (row.category === 'CHN' ? 1 : 0);
-                    }, 0), 0);
-
-                    done();
-                });
-            });
-        });
-    });
-
-});

test/acceptance/widgets/aggregation.js

@@ -0,0 +1,74 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('aggregation widgets', function() {
+
+    var aggregationMapConfig =  {
+        version: '1.5.0',
+        layers: [
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                    cartocss_version: '2.3.0',
+                    widgets: {
+                        country_places_count: {
+                            type: 'aggregation',
+                            options: {
+                                column: 'adm0_a3',
+                                aggregation: 'count'
+                            }
+                        }
+                    }
+                }
+            }
+        ]
+    };
+
+    it("should expose an aggregation", function(done) {
+        var testClient = new TestClient(aggregationMapConfig);
+        testClient.getWidget('country_places_count', { own_filter: 0 }, function(err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            var aggregation = JSON.parse(res.body);
+            assert.equal(aggregation.categories.length, 6);
+            assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false });
+
+            testClient.drain(done);
+        });
+    });
+
+    describe('filters', function() {
+
+        describe('category', function () {
+
+            it("should expose a filtered aggregation", function (done) {
+                var params = {
+                    filters: {
+                        layers: [
+                            {country_places_count: {accept: ['CAN']}}
+                        ]
+                    }
+                };
+                var testClient = new TestClient(aggregationMapConfig);
+                testClient.getWidget('country_places_count', params, function (err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var aggregation = JSON.parse(res.body);
+                    assert.equal(aggregation.categories.length, 1);
+                    assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false });
+
+                    testClient.drain(done);
+                });
+            });
+        });
+
+    });
+});

test/acceptance/widgets/histogram.js

@@ -0,0 +1,120 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('histogram widgets', function() {
+
+    it("should expose layer histogram", function(done) {
+        var histogramMapConfig =  {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            pop_max: {
+                                type: 'histogram',
+                                options: {
+                                    column: 'pop_max'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        var testClient = new TestClient(histogramMapConfig);
+
+        testClient.getWidget('pop_max', function(err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            var histogram = JSON.parse(res.body);
+            assert.ok(histogram.bins.length);
+
+            testClient.drain(done);
+        });
+    });
+
+    describe('filters', function() {
+
+        describe('range', function() {
+            var histogramMapConfig =  {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                            cartocss_version: '2.3.0',
+                            widgets: {
+                                country_places_histogram: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            };
+
+            it("should expose an histogram", function(done) {
+                var testClient = new TestClient(histogramMapConfig);
+                testClient.getWidget('country_places_histogram', { own_filter: 0 }, function(err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var histogram = JSON.parse(res.body);
+                    // notice min value
+                    assert.deepEqual(
+                        histogram.bins[0],
+                        { bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 }
+                    );
+
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered histogram", function(done) {
+                var params = {
+                    filters: {
+                        layers: [
+                            {
+                                country_places_histogram: { min: 4000000 }
+                            }
+                        ]
+                    }
+                };
+                var testClient = new TestClient(histogramMapConfig);
+                testClient.getWidget('country_places_histogram', params, function(err, res) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var histogram = JSON.parse(res.body);
+                    // notice min value
+                    assert.deepEqual(histogram.bins[0], {
+                        bin: 0,
+                        freq: 62,
+                        min: 4000000,
+                        max: 9276403,
+                        avg: 5815009.596774193
+                    });
+
+                    testClient.drain(done);
+                });
+            });
+        });
+    });
+
+});

test/acceptance/widgets/list.js

@@ -0,0 +1,51 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('list widgets', function() {
+
+    it("should expose layer list", function(done) {
+
+        var listWidgetMapConfig =  {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from test_table',
+                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            names: {
+                                type: 'list',
+                                options: {
+                                    columns: ['name']
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        var testClient = new TestClient(listWidgetMapConfig);
+
+        testClient.getWidget('names', function(err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            var expectedList = [
+                {name:"Hawai"},
+                {name:"El Estocolmo"},
+                {name:"El Rey del Tallarín"},
+                {name:"El Lacón"},
+                {name:"El Pico"}
+            ];
+            assert.deepEqual(JSON.parse(res.body).rows, expectedList);
+
+            testClient.drain(done);
+        });
+    });
+});

test/acceptance/widgets/named-maps.js

@@ -0,0 +1,250 @@
+var assert = require('../../support/assert');
+var step = require('step');
+
+var url = require('url');
+var queue = require('queue-async');
+
+var helper = require('../../support/test_helper');
+
+var CartodbWindshaft = require('../../../lib/cartodb/server');
+var serverOptions = require('../../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+
+describe('named-maps widgets', function() {
+
+    var username = 'localhost';
+    var widgetsTemplateName = 'widgets-template';
+
+    var layergroupid;
+    var layergroup;
+    var keysToDelete;
+
+    beforeEach(function(done) {
+        keysToDelete = {};
+
+        var widgetsTemplate =  {
+            version: '0.0.1',
+            name: widgetsTemplateName,
+            layergroup:  {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: "select * from populated_places_simple_reduced_private",
+                            cartocss: '#layer { marker-fill: blue; }',
+                            cartocss_version: '2.3.0',
+                            widgets: {
+                                pop_max_formula_sum: {
+                                    type: 'formula',
+                                    options: {
+                                        column: 'pop_max',
+                                        operation: 'sum'
+                                    }
+                                },
+                                country_places_count: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm0_a3',
+                                        aggregation: 'count'
+                                    }
+                                },
+                                pop_max: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        };
+
+        var template_params = {};
+
+        step(
+            function createTemplate()
+            {
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named?api_key=1234',
+                        method: 'POST',
+                        headers: {
+                            host: username,
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(widgetsTemplate)
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res, err) {
+                        next(err, res);
+                    }
+                );
+            },
+            function instantiateTemplate(err, res) {
+                assert.ifError(err);
+
+                assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName });
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + widgetsTemplateName,
+                        method: 'POST',
+                        headers: {
+                            host: username,
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(template_params)
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        next(null, res);
+                    }
+                );
+            },
+            function fetchTile(err, res) {
+                assert.ifError(err);
+
+                layergroup = JSON.parse(res.body);
+                assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body);
+                layergroupid = layergroup.layergroupid;
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                return done();
+            }
+        );
+
+    });
+
+    afterEach(function(done) {
+        step(
+            function deleteTemplate(err) {
+                assert.ifError(err);
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234',
+                        method: 'DELETE',
+                        headers: {
+                            host: username
+                        }
+                    },
+                    {
+                        status: 204
+                    },
+                    function(res, err) {
+                        next(err, res);
+                    }
+                );
+            },
+            function deleteRedisKeys(err) {
+                assert.ifError(err);
+                helper.deleteRedisKeys(keysToDelete, done);
+            }
+        );
+    });
+
+    function getWidget(widgetName, callback) {
+        assert.response(
+            server,
+            {
+                url: '/api/v1/map/' + layergroupid + '/0/widget/' + widgetName,
+                method: 'GET',
+                headers: {
+                    host: username
+                }
+            },
+            {
+                status: 200
+            },
+            function(res, err) {
+                if (err) {
+                    return callback(err);
+                }
+                var parsedBody = JSON.parse(res.body);
+                return callback(err, res, parsedBody);
+            }
+        );
+    }
+
+    it('should be able to retrieve widgets from all URLs', function(done) {
+        var widgetsPaths = layergroup.metadata.layers.reduce(function(paths, layer) {
+            var widgets = layer.widgets || {};
+            Object.keys(widgets).forEach(function(widget) {
+                paths.push(url.parse(widgets[widget].url.http).path);
+            });
+
+            return paths;
+        }, []);
+
+        var widgetsQueue = queue(widgetsPaths.length);
+
+        widgetsPaths.forEach(function(path) {
+            widgetsQueue.defer(function(path, done) {
+                assert.response(
+                    server,
+                    {
+                        url: path,
+                        method: 'GET',
+                        headers: {
+                            host: username
+                        }
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res, err) {
+                        if (err) {
+                            return done(err);
+                        }
+                        var parsedBody = JSON.parse(res.body);
+                        return done(null, parsedBody);
+                    }
+                );
+            }, path);
+        });
+
+        widgetsQueue.awaitAll(function(err, results) {
+            assert.equal(results.length, 3);
+            done(err);
+        });
+    });
+
+
+    it("should retrieve aggregation", function(done) {
+        getWidget('country_places_count', function(err, response, aggregation) {
+            assert.ok(!err, err);
+
+            assert.equal(aggregation.type, 'aggregation');
+            assert.equal(aggregation.max, 769);
+
+            return done();
+        });
+    });
+
+    it("should retrieve histogram", function(done) {
+        getWidget('pop_max', function(err, response, histogram) {
+            assert.ok(!err, err);
+
+            assert.equal(histogram.type, 'histogram');
+            assert.equal(histogram.bin_width, 743250);
+
+            return done();
+        });
+    });
+
+});

test/acceptance/widgets/widgets.js

@@ -0,0 +1,162 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('widget filters', function() {
+
+    describe('combine widget filters', function() {
+        var combinedWidgetsMapConfig =  {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            country_places_count: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0_a3',
+                                    aggregation: 'count'
+                                }
+                            },
+                            country_places_histogram: {
+                                type: 'histogram',
+                                options: {
+                                    column: 'pop_max'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        it("should expose a filtered aggregation", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_count: { reject: ['CHN'] }
+                        }
+                    ]
+                }
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+
+        it("should expose a filtered aggregation", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_count: { reject: ['CHN'] },
+                            country_places_histogram: { min: 7000000 }
+                        }
+                    ]
+                }
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+
+        it("should allow to filter by bounding box a filtered aggregation", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_histogram: { min: 50000 }
+                        }
+                    ]
+                },
+                bbox: '-20,0,45,60'
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+
+        it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) {
+            var params = {
+                filters: {
+                    layers: [
+                        {
+                            country_places_count: { reject: ['RUS'] },
+                            country_places_histogram: { min: 50000 }
+                        }
+                    ]
+                },
+                bbox: '-20,0,45,60'
+            };
+            var testClient = new TestClient(combinedWidgetsMapConfig);
+            testClient.getWidget('country_places_count', params, function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var aggregation = JSON.parse(res.body);
+
+                // first one would be CHN if reject filter wasn't applied
+                assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false });
+
+                // confirm 'CHN' was filtered out (reject)
+                assert.equal(aggregation.categories.reduce(function(sum, row) {
+                    return sum + (row.category === 'CHN' ? 1 : 0);
+                }, 0), 0);
+
+                testClient.drain(done);
+            });
+        });
+    });
+});

test/integration/mapconfig_overviews_adapter.js

@@ -0,0 +1,87 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
+var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
+var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
+
+// configure redis pool instance to use in tests
+var redisPool = new RedisPool(global.environment.redis);
+var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool }));
+
+var redisPool = new RedisPool(global.environment.redis);
+var metadataBackend = cartodbRedis({pool: redisPool});
+var pgConnection = new PgConnection(metadataBackend);
+var pgQueryRunner = new PgQueryRunner(pgConnection);
+var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+
+
+var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
+
+describe('MapConfigOverviewsAdapter', function() {
+
+    it('should not modify layers for which no overviews are available', function(done) {
+        var sql = 'SELECT * FROM test_table';
+        var cartocss = '#layer { marker-fill: black; }';
+        var cartocss_version = '2.3.0';
+        var layer_without_overviews = {
+            type: 'cartodb',
+            options: {
+                sql: sql,
+                cartocss: cartocss,
+                cartocss_version: cartocss_version
+            }
+        };
+
+        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
+            assert.ok(!err);
+            assert.equal(layers.length, 1);
+            assert.equal(layers[0].type, 'cartodb');
+            assert.equal(layers[0].options.sql, sql);
+            assert.equal(layers[0].options.cartocss, cartocss);
+            assert.equal(layers[0].options.cartocss_version, cartocss_version);
+            assert.equal(layers[0].options.overviews, undefined);
+            done();
+        });
+    });
+});
+
+describe('MapConfigOverviewsAdapter', function() {
+
+    it('should add overviews metadata for layers using tables with overviews', function(done) {
+        var sql = 'SELECT * FROM test_table_overviews';
+        var cartocss = '#layer { marker-fill: black; }';
+        var cartocss_version = '2.3.0';
+        var layer_without_overviews = {
+            type: 'cartodb',
+            options: {
+                sql: sql,
+                cartocss: cartocss,
+                cartocss_version: cartocss_version
+            }
+        };
+
+        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
+            assert.ok(!err);
+            assert.equal(layers.length, 1);
+            assert.equal(layers[0].type, 'cartodb');
+            assert.equal(layers[0].options.sql, sql);
+            assert.equal(layers[0].options.cartocss, cartocss);
+            assert.equal(layers[0].options.cartocss_version, cartocss_version);
+            assert.ok(layers[0].options.query_rewrite_data);
+            var expected_data = {
+              overviews: {
+                test_table_overviews: {
+                  1: { table: '_vovw_1_test_table_overviews' },
+                  2: { table: '_vovw_2_test_table_overviews' }
+                }
+              }
+            };
+            assert.deepEqual(layers[0].options.query_rewrite_data, expected_data);
+            done();
+        });
+    });
+});

test/integration/overviews-metadata-api.js

@@ -0,0 +1,54 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var PgConnection = require('../../lib/cartodb/backends/pg_connection');
+var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
+var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
+
+
+describe('OverviewsMetadataApi', function() {
+
+    var queryTablesApi, overviewsMetadataApi;
+
+    before(function() {
+        var redisPool = new RedisPool(global.environment.redis);
+        var metadataBackend = cartodbRedis({pool: redisPool});
+        var pgConnection = new PgConnection(metadataBackend);
+        var pgQueryRunner = new PgQueryRunner(pgConnection);
+        queryTablesApi = new QueryTablesApi(pgQueryRunner);
+        overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+    });
+
+    it('should return an empty relation for tables that have no overviews', function(done) {
+        var query = 'select * from test_table';
+        overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.deepEqual(result, {});
+
+            done();
+        });
+    });
+
+    it('should return overviews metadata', function(done) {
+        var query = 'select * from test_table_overviews';
+        overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.deepEqual(result, {
+                'test_table_overviews': {
+                  1: { table: '_vovw_1_test_table_overviews' },
+                  2: { table: '_vovw_2_test_table_overviews' }
+                }
+            });
+
+            done();
+        });
+    });
+
+});

test/support/assert.js

@@ -1,7 +1,6 @@
 // Cribbed from the ever prolific Konstantin Kaefer
 // https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js
 
-var exec = require('child_process').exec;
 var fs = require('fs');
 var path = require('path');
 var util = require('util');
@@ -13,7 +12,7 @@ var request = require('request');
 var assert = module.exports = exports = require('assert');
 
 /**
- * Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
+ * Takes an image data as an input and an image path and compare them using mapnik.Image.compare mechanism, in case the
  * similarity is not within the tolerance limit it will callback with an error.
  *
  * @param buffer The image data to compare from
@@ -22,70 +21,39 @@ var assert = module.exports = exports = require('assert');
  * @param {function} callback Will call to home with null in case there is no error, otherwise with the error itself
  * @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
  */
-assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
+assert.imageBufferIsSimilarToFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
     callback = callback || function(err) { assert.ifError(err); };
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
-        testImageFilePath = createImageFromBuffer(buffer, 'test');
 
-    imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, function(err) {
-        fs.unlinkSync(testImageFilePath);
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
+    var referenceImageBuffer = fs.readFileSync(referenceImageFilePath, { encoding: null });
+
+    assert.imageBuffersAreSimilar(buffer, referenceImageBuffer, tolerance, callback);
+};
+
+assert.imageBuffersAreSimilar = function(bufferA, bufferB, tolerance, callback) {
+    var testImage = mapnik.Image.fromBytes(Buffer.isBuffer(bufferA) ? bufferA : new Buffer(bufferA, 'binary'));
+    var referenceImage = mapnik.Image.fromBytes(Buffer.isBuffer(bufferB) ? bufferB : new Buffer(bufferB, 'binary'));
+
+    imagesAreSimilar(testImage, referenceImage, tolerance, callback);
+};
+
+assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
+    callback = callback || function(err) { assert.ifError(err); };
+
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
+
+    var referenceImage = mapnik.Image.fromBytes(fs.readFileSync(referenceImageFilePath,  { encoding: null }));
+
+    imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
+        if (err) {
+            var testImageFilePath = randomImagePath();
+            testImage.save(testImageFilePath);
+        }
         callback(err);
     });
 };
 
-assert.imageBuffersAreEqual = function(bufferA, bufferB, tolerance, callback) {
-    var randStr = (Math.random() * 1e16).toString().substring(0, 8);
-    var imageFilePathA = createImageFromBuffer(bufferA, randStr + '-a'),
-        imageFilePathB = createImageFromBuffer(bufferB, randStr + '-b');
-
-    imageFilesAreEqual(imageFilePathA, imageFilePathB, tolerance, function(err, similarity) {
-        callback(err, [imageFilePathA, imageFilePathB], similarity);
-    });
-};
-
-function createImageFromBuffer(buffer, nameHint) {
-    var imageFilePath = path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
-    var err = fs.writeFileSync(imageFilePath, buffer, 'binary');
-    assert.ifError(err);
-    return imageFilePath;
-}
-
-function imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, callback) {
-    var resultFilePath = path.resolve(util.format('/tmp/windshaft-result-%s-diff.png', Date.now()));
-    var imageMagickCmd = util.format(
-        'compare -metric fuzz "%s" "%s" "%s"',
-        testImageFilePath, referenceImageFilePath, resultFilePath
-    );
-
-    exec(imageMagickCmd, function(err, stdout, stderr) {
-        if (err) {
-            fs.unlinkSync(testImageFilePath);
-            callback(err);
-        } else {
-            stderr = stderr.trim();
-            var metrics = stderr.match(/([0-9]*) \((.*)\)/);
-            if ( ! metrics ) {
-                callback(new Error("No match for " + stderr));
-                return;
-            }
-            var similarity = parseFloat(metrics[2]),
-                tolerancePerMil = (tolerance / 1000);
-            if (similarity > tolerancePerMil) {
-                err = new Error(util.format(
-                        'Images %s and %s are not equal (got %d similarity, expected %d). Result %s',
-                        testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil, resultFilePath)
-                );
-                err.similarity = similarity;
-                callback(err, similarity);
-            } else {
-                fs.unlinkSync(resultFilePath);
-                callback(null, similarity);
-            }
-        }
-    });
-}
-
-assert.imagesAreSimilar = function(testImage, referenceImage, tolerance, callback) {
+function imagesAreSimilar(testImage, referenceImage, tolerance, callback) {
     if (testImage.width() !== referenceImage.width() || testImage.height() !== referenceImage.height()) {
         return callback(new Error('Images are not the same size'));
     }
@@ -103,27 +71,10 @@ assert.imagesAreSimilar = function(testImage, referenceImage, tolerance, callbac
     } else {
         callback(null, similarity);
     }
-};
+}
 
-assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
-    callback = callback || function(err) { assert.ifError(err); };
-
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
-
-    var referenceImage = mapnik.Image.fromBytes(fs.readFileSync(referenceImageFilePath,  { encoding: null }));
-
-    assert.imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
-        if (err) {
-            var testImageFilePath = randomImagePath();
-            testImage.save(testImageFilePath);
-        }
-        callback(err);
-    });
-};
-
-function randomImagePath(nameHint) {
-    nameHint = nameHint || 'test';
-    return path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
+function randomImagePath() {
+    return path.resolve('test/results/png/image-test-' + Date.now() + '.png');
 }
 
 // jshint maxcomplexity:9

test/support/config.js

@@ -7,8 +7,8 @@ module.exports = function(opts) {
 
     var config = {
         redis_pool: {
-            max: 10, 
-            idleTimeoutMillis: 1, 
+            max: 10,
+            idleTimeoutMillis: 1,
             reapIntervalMillis: 1,
             port: global.environment.redis.port
         }

test/support/prepare_db.sh

@@ -72,16 +72,16 @@ if test x"$PREPARE_PGSQL" = xyes; then
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
   cat sql/windshaft.test.sql sql/gadm4.sql |
-    sed "s/:PUBLICUSER/${PUBLICUSER}/" | 
-    sed "s/:PUBLICPASS/${PUBLICPASS}/" | 
-    sed "s/:TESTUSER/${TESTUSER}/" | 
-    sed "s/:TESTPASS/${TESTPASS}/" | 
+    sed "s/:PUBLICUSER/${PUBLICUSER}/" |
+    sed "s/:PUBLICPASS/${PUBLICPASS}/" |
+    sed "s/:TESTUSER/${TESTUSER}/" |
+    sed "s/:TESTPASS/${TESTPASS}/" |
     psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
   psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
   curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql
   curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
-  cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql |
+  cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql sql/CDB_Overviews.sql |
     psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
 fi
@@ -98,7 +98,7 @@ HMSET rails:users:localhost id ${TESTUSERID} \
 SADD rails:users:localhost:map_key 1235
 EOF
 
-  # A user configured as with cartodb-2.5.0+ 
+  # A user configured as with cartodb-2.5.0+
   cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
 HMSET rails:users:cartodb250user id ${TESTUSERID} \
                                  database_name "${TEST_DB}" \

test/support/sql/.gitignore

@@ -0,0 +1 @@
+CDB_*.sql

test/support/sql/CDB_Overviews.sql

@@ -0,0 +1,46 @@
+-- Mockup for CDB_Overviews
+CREATE OR REPLACE FUNCTION CDB_Overviews(table_names regclass[])
+RETURNS TABLE(base_table regclass, z integer, overview_table regclass)
+AS $$
+  BEGIN
+    IF (SELECT 'test_table_overviews'::regclass = ANY (table_names)) THEN
+      RETURN QUERY
+        SELECT 'test_table_overviews'::regclass AS base_table, 1 AS z, '_vovw_1_test_table_overviews'::regclass AS overview_table
+        UNION ALL
+        SELECT 'test_table_overviews'::regclass AS base_table, 2 AS z, '_vovw_2_test_table_overviews'::regclass AS overview_table;
+    ELSE
+      RETURN;
+    END IF;
+  END
+$$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
+BEGIN
+  CASE
+    WHEN scaleDenominator > 500000000 THEN RETURN 0;
+    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
+    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
+    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
+    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
+    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
+    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
+    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
+    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
+    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
+    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
+    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
+    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
+    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
+    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
+    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
+    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
+    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
+    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
+    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
+    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
+    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
+    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
+    WHEN scaleDenominator <= 100 THEN RETURN 23;
+  END CASE;
+END
+$$ LANGUAGE plpgsql IMMUTABLE;

test/support/sql/ported/populated_places_simple_reduced.sql

@@ -7385,3 +7385,8 @@ GRANT ALL ON TABLE populated_places_simple_reduced TO :TESTUSER;
 GRANT SELECT ON TABLE populated_places_simple_reduced TO :PUBLICUSER;
 
 VACUUM ANALYZE populated_places_simple_reduced;
+
+create table populated_places_simple_reduced_private AS
+select * from populated_places_simple_reduced;
+
+GRANT ALL ON TABLE populated_places_simple_reduced_private TO :TESTUSER;

test/support/sql/windshaft.test.sql

@@ -1,6 +1,6 @@
 --
 -- Windshaft test database
--- 
+--
 -- To use run ../prepare_db.sh
 -- NOTE: requires a postgis template called template_postgis
 --
@@ -238,3 +238,97 @@ CREATE INDEX test_big_poly_the_geom_webmercator_idx ON test_big_poly USING gist
 
 GRANT ALL ON TABLE test_big_poly TO :TESTUSER;
 GRANT SELECT ON TABLE test_big_poly TO :PUBLICUSER;
+
+-- table with overviews
+
+CREATE TABLE test_table_overviews (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+GRANT ALL ON TABLE test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER;
+
+CREATE SEQUENCE test_table_overviews_cartodb_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE test_table_overviews_cartodb_id_seq OWNED BY test_table_overviews.cartodb_id;
+
+SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true);
+
+ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass);
+
+INSERT INTO test_table_overviews VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+
+ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id);
+
+CREATE INDEX test_table_overviews_the_geom_idx ON test_table_overviews USING gist (the_geom);
+CREATE INDEX test_table_overviews_the_geom_webmercator_idx ON test_table_overviews USING gist (the_geom_webmercator);
+
+GRANT ALL ON TABLE test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER;
+
+CREATE TABLE _vovw_1_test_table_overviews (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+GRANT ALL ON TABLE _vovw_1_test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE _vovw_1_test_table_overviews TO :PUBLICUSER;
+
+CREATE TABLE _vovw_2_test_table_overviews (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER;
+GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER;
+
+INSERT INTO _vovw_2_test_table_overviews VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
+
+INSERT INTO _vovw_1_test_table_overviews VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241');

test/support/test-client.js

@@ -0,0 +1,119 @@
+'use strict';
+
+var qs = require('querystring');
+var step = require('step');
+
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+
+var assert = require('./assert');
+var helper = require('./test_helper');
+
+var CartodbWindshaft = require('../../lib/cartodb/server');
+var serverOptions = require('../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+
+function TestClient(mapConfig) {
+    this.mapConfig = mapConfig;
+    this.keysToDelete = {};
+}
+
+module.exports = TestClient;
+
+TestClient.prototype.getWidget = function(widgetName, params, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = params;
+        params = {};
+    }
+
+    var url = '/api/v1/map';
+    if (params && params.filters) {
+        url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
+    }
+
+    var layergroupId;
+    step(
+        function createLayergroup() {
+            var next = this;
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(self.mapConfig)
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+                    var parsedBody = JSON.parse(res.body);
+                    var expectedWidgetURLS = {
+                        http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
+                    };
+                    assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
+                    assert.ok(
+                        parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
+                    );
+                    return next(null, parsedBody.layergroupid);
+                }
+            );
+        },
+        function getWidgetResult(err, _layergroupId) {
+            assert.ifError(err);
+
+            var next = this;
+            layergroupId = _layergroupId;
+
+            var urlParams = {
+                own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
+            };
+            if (params && params.bbox) {
+                urlParams.bbox = params.bbox;
+            }
+            url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
+
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'GET',
+                    headers: {
+                        host: 'localhost'
+                    }
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+
+                    next(null, res);
+                }
+            );
+        },
+        function finish(err, res) {
+            self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+            self.keysToDelete['user:localhost:mapviews:global'] = 5;
+            return callback(err, res);
+        }
+    );
+};
+
+TestClient.prototype.drain = function(callback) {
+    helper.deleteRedisKeys(this.keysToDelete, callback);
+};

test/support/test_helper.js

@@ -66,6 +66,14 @@ function checkSurrogateKey(res, expectedKey) {
     assert.equal(res.headers['surrogate-key'], expectedKey);
 }
 
+var redisClient;
+
+beforeEach(function() {
+    if (!redisClient) {
+        redisClient = redis.createClient(global.environment.redis.port);
+    }
+});
+
 //global afterEach to capture test suites that leave keys in redis
 afterEach(function(done) {
 
@@ -102,7 +110,6 @@ afterEach(function(done) {
     }
 
     Object.keys(databasesTasks).forEach(function(db) {
-        var redisClient = redis.createClient(global.environment.redis.port);
         redisClient.select(db, function() {
             // Check that we start with an empty redis db
             redisClient.keys("*", function(err, keys) {
@@ -129,6 +136,7 @@ function deleteRedisKeys(keysToDelete, callback) {
         var redisClient = redis.createClient(global.environment.redis.port);
         redisClient.select(keysToDelete[k], function() {
             redisClient.del(k, function(err, deletedKeysCount) {
+                redisClient.quit();
                 assert.notStrictEqual(deletedKeysCount, 0, 'No KEYS deleted for: [db=' + keysToDelete[k] + ']' + k);
                 taskDone(k);
             });

test/unit/cartodb/overviews_query_rewriter.js

@@ -0,0 +1,399 @@
+require('../../support/test_helper');
+
+var assert           = require('assert');
+var OverviewsQueryRewriter = require('../../../lib/cartodb/utils/overviews_query_rewriter');
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+  zoom_level: 'ZoomLevel()'
+});
+
+function normalize_whitespace(txt) {
+  return txt.replace(/\s+/g, " ").trim();
+}
+
+// compare SQL statements ignoring whitespace
+function assertSameSql(sql1, sql2) {
+    assert.equal(normalize_whitespace(sql1), normalize_whitespace(sql2));
+}
+
+describe('Overviews query rewriter', function() {
+
+  it('does not alter queries if no overviews data is present', function(done){
+    var sql = "SELECT * FROM table1";
+    var overviews_sql = overviewsQueryRewriter.query(sql);
+    assert.equal(overviews_sql, sql);
+    overviews_sql = overviewsQueryRewriter.query(sql, {});
+    assert.equal(overviews_sql, sql);
+    overviews_sql = overviewsQueryRewriter.query(sql, { overviews: {} });
+    assert.equal(overviews_sql, sql);
+    done();
+  });
+
+
+  it('does not alter queries which don\'t use overviews', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table2: {
+              0: { table: 'table2_ov0' },
+              1: { table: 'table2_ov1' },
+              4: { table: 'table2_ov4' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+    done();
+  });
+
+  // jshint multistr:true
+
+  it('generates query with single overview layer for level 0', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query with single overview layer for level >0', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query with multiple overview layers for all levels up to N', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' },
+              1: { table: 'table1_ov1' },
+              2: { table: 'table1_ov2' },
+              3: { table: 'table1_ov3' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM  (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
+            UNION ALL\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
+            UNION ALL\
+            SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query with multiple overview layers for random levels', function(done){
+    var sql = "SELECT * FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' },
+              1: { table: 'table1_ov1' },
+              6: { table: 'table1_ov6' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
+            UNION ALL\
+            SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table with explicit schema', function(done){
+    var sql = "SELECT * FROM public.table1";
+    var data = {
+        overviews: {
+            'public.table1': {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table with explicit schema in the overviews info', function(done){
+    var sql = "SELECT * FROM public.table1";
+    var data = {
+        overviews: {
+            'public.table1': {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table that needs quoting with explicit schema', function(done){
+    var sql = "SELECT * FROM public.\"table 1\"";
+    var data = {
+        overviews: {
+            'public."table 1"': {
+              2: { table: '"table 1_ov2"' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
+          ) AS \"_vovw_table 1\"\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query for a table with explicit schema that needs quoting', function(done){
+    var sql = "SELECT * FROM \"user-1\".table1";
+    var data = {
+        overviews: {
+            '"user-1".table1': {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+
+  it('generates query for a table with explicit schema both needing quoting', function(done){
+    var sql = "SELECT * FROM \"user-1\".\"table 1\"";
+    var data = {
+        overviews: {
+            '"user-1"."table 1"': {
+              2: { table: '"table 1_ov2"' }
+
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
+            SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
+          ) AS \"_vovw_table 1\"\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+
+  it('generates query using overviews for queries with selected columns', function(done){
+    var sql = "SELECT column1, column2, column3 FROM table1";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query using overviews for queries with a semicolon', function(done){
+    var sql = "SELECT column1, column2, column3 FROM table1;";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1;\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('generates query using overviews for queries with extra whitespace', function(done){
+    var sql = "  SELECT  column1,column2,  column3 FROM  table1  ";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1,column2, column3 FROM (\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
+
+  it('does not alter queries which have not the simple supported form', function(done){
+    var sql = "SELECT * FROM table1 WHERE column1='x'";
+    var data = {
+        overviews: {
+            table1: {
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT * FROM table1 JOIN table2 ON (table1.col1=table2.col1)";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT a+b AS c FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT f(a) AS b FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT * FROM table1 AS x";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "WITH a AS (1) SELECT * FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT * FROM table1 WHERE a=1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "\
+        SELECT table1.* FROM table1 \
+               JOIN areas ON ST_Intersects(table1.the_geom, areas.the_geom) \
+               WHERE areas.name='A' \
+          ";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    sql = "SELECT table1.*, column1, column2, column3 FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
+    done();
+  });
+
+
+});

test/unit/cartodb/req2params.test.js

@@ -30,7 +30,7 @@ describe('req2params', function() {
         baseController = new BaseController(authApi, pgConnection);
     });
 
-    
+
     it('can be found in server_options', function(){
       assert.ok(_.isFunction(baseController.req2params));
     });

test/unit/cartodb/table_name_parser.js

@@ -0,0 +1,60 @@
+require('../../support/test_helper');
+
+var assert        = require('assert');
+var TableNameParser     = require('../../../lib/cartodb/utils/table_name_parser');
+
+describe('TableNameParser', function() {
+
+  it('parses table names with scheme and quotes', function(done){
+
+      var test_cases = [
+        ['xyz',             { schema: null,       table: 'xyz' }],
+        ['"xyz"',           { schema: null,       table: 'xyz' }],
+        ['"xy z"',          { schema: null,       table: 'xy z' }],
+        ['"xy.z"',          { schema: null,       table: 'xy.z' }],
+        ['"x.y.z"',         { schema: null,       table: 'x.y.z' }],
+        ['abc.xyz',         { schema: 'abc',      table: 'xyz' }],
+        ['"abc".xyz',       { schema: 'abc',      table: 'xyz' }],
+        ['abc."xyz"',       { schema: 'abc',      table: 'xyz' }],
+        ['"abc"."xyz"',     { schema: 'abc',      table: 'xyz' }],
+        ['"a bc"."x yz"',   { schema: 'a bc',     table: 'x yz' }],
+        ['"a bc".xyz',      { schema: 'a bc',     table: 'xyz' }],
+        ['"a.bc".xyz',      { schema: 'a.bc',     table: 'xyz' }],
+        ['"a.b.c".xyz',     { schema: 'a.b.c',    table: 'xyz' }],
+        ['"a.b.c.".xyz',    { schema: 'a.b.c.',   table: 'xyz' }],
+        ['"a""bc".xyz',     { schema: 'a"bc',     table: 'xyz' }],
+        ['"a""bc"."x""yz"', { schema: 'a"bc',     table: 'x"yz' }],
+      ];
+
+      test_cases.forEach(function(test_case) {
+          var table_name = test_case[0];
+          var expected_result = test_case[1];
+          var result = TableNameParser.parse(table_name);
+          assert.deepEqual(result, expected_result);
+      });
+      done();
+  });
+
+  it('quotes identifiers that need quoting', function(done){
+      assert.equal(TableNameParser.quote('x yz'), '"x yz"');
+      assert.equal(TableNameParser.quote('x-yz'), '"x-yz"');
+      assert.equal(TableNameParser.quote('x.yz'), '"x.yz"');
+      done();
+  });
+
+  it('doubles quotes', function(done){
+      assert.equal(TableNameParser.quote('x"yz'), '"x""yz"');
+      assert.equal(TableNameParser.quote('x"y"z'), '"x""y""z"');
+      assert.equal(TableNameParser.quote('x""y"z'), '"x""""y""z"');
+      assert.equal(TableNameParser.quote('x "yz'), '"x ""yz"');
+      assert.equal(TableNameParser.quote('x"y-y"z'), '"x""y-y""z"');
+      done();
+  });
+
+  it('does not quote identifiers that don\'t need to be quoted', function(done){
+      assert.equal(TableNameParser.quote('xyz'), 'xyz');
+      assert.equal(TableNameParser.quote('x_z123'), 'x_z123');
+      done();
+  });
+
+});