Release 2.32.0

Overrided cartocss in the instantiation of named maps

.travis.yml

@@ -1,18 +1,18 @@
 sudo: false
 
 addons:
-  postgresql: "9.4"
+  postgresql: "9.3"
   apt:
     packages:
-    - postgresql-plpython-9.4
-    - pkg-config
-    - libcairo2-dev
-    - libjpeg8-dev
-    - libgif-dev
+      - pkg-config
+      - libcairo2-dev
+      - libjpeg8-dev
+      - libgif-dev
 
 before_install:
   - npm install -g npm@2
   - createdb template_postgis
+  - createuser publicuser
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
 env:

INSTALL.md

@@ -16,10 +16,6 @@ Make sure that you have the requirements needed. These are
   - CartoDB 0.9.5+ (for `CDB_QueryTables`)
   - Varnish (http://www.varnish-cache.org)
 
-- For running the testsuite
-  - ImageMagick (http://www.imagemagick.org)
-
-
 On Ubuntu 14.04 the dependencies can be installed with
 
 ```shell

NEWS.md

@@ -1,11 +1,141 @@
 # Changelog
 
+## 2.32.0
+
+Released 2016-04-06
+
+New features:
+ - Added support for dynamic styling for widgets in named maps
+
+Announcements:
+ - Upgrades windshaft to [1.17.0](https://github.com/CartoDB/Windshaft/releases/tag/1.17.0)
+
+
+## 2.31.2
+
+Released 2016-04-04
+
+Bug fixes:
+ - Overviews integration for named layers #400
+ - Support wrapped queries in named layers #405
+
+
+## 2.31.1
+
+Released 2016-03-23
+
+Announcements:
+ - Upgrades windshaft to [1.16.1](https://github.com/CartoDB/Windshaft/releases/tag/1.16.1)
+
+
+## 2.31.0
+
+Released 2016-03-16
+
+Announcements:
+ - Upgrades windshaft to [1.16.0](https://github.com/CartoDB/Windshaft/releases/tag/1.16.0)
+
+
+## 2.30.0
+
+Released 2016-03-15
+
+Announcements:
+ - Upgrades windshaft to [1.15.0](https://github.com/CartoDB/Windshaft/releases/tag/1.15.0)
+
+
+## 2.29.0
+
+Released 2016-03-14
+
+Announcements:
+ - Upgrades windshaft to [1.14.0](https://github.com/CartoDB/Windshaft/releases/tag/1.14.0)
+
+
+## 2.28.0
+
+Released 2016-03-14
+
+New features:
+ - Added [turbo-cartocss](https://github.com/CartoDB/turbo-cartocss) to preprocess CartoCSS.
+
+
+## 2.27.0
+
+Released 2016-03-09
+
+New features:
+ - Add [Surrogate-Key](https://github.com/CartoDB/cartodb/wiki/CartoDB-Surrogate-Keys) headers to responses
+
+Enhancements:
+ - Use new `node-cartodb-query-tables` library to obtain affected tables in queries
+
+Announcements:
+ - Remove deprecated tools directory
+
+
+## 2.26.3
+
+Released 2016-03-03
+
+Improvements:
+ - Optimize overviews queries for efficient spatial filtering in PostgreSQL
+
+
+## 2.26.2
+
+Released 2016-02-25
+
+Announcements:
+ - Upgrades windshaft to [1.13.2](https://github.com/CartoDB/Windshaft/releases/tag/1.13.2)
+
+
+## 2.26.1
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.1](https://github.com/CartoDB/Windshaft/releases/tag/1.13.1)
+
+
+## 2.26.0
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.0](https://github.com/CartoDB/Windshaft/releases/tag/1.13.0)
+
+
+## 2.25.2
+
+Released 2016-02-22
+
+Bug fixes:
+ - Correct URLs for widgets in named maps #381
+
+
+## 2.25.1
+
+Released 2016-02-22
+
+Announcements:
+ - Upgrades windshaft to [1.11.1](https://github.com/CartoDB/Windshaft/releases/tag/1.11.1)
+
+
+## 2.25.0
+
+Released 2016-02-18
+
+Announcements:
+ - Upgrades windshaft to [1.11.0](https://github.com/CartoDB/Windshaft/releases/tag/1.11.0)
+
+
 ## 2.24.0
 
 Released 2016-02-15
 
 Announcements:
- - Upgrades windshaft to [1.10.0](https://github.com/CartoDB/Windshaft/releases/tag/1.10.1)
+ - Upgrades windshaft to [1.10.1](https://github.com/CartoDB/Windshaft/releases/tag/1.10.1)
 
 
 ## 2.23.0

README.md

@@ -67,3 +67,16 @@ Contributing
 ---
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+### Developing with a custom windshaft version
+
+If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
+to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
+
+**Quick start**:
+
+```shell
+~/windshaft-directory $ npm install
+~/windshaft-directory $ npm link
+~/windshaft-cartodb-directory $ npm link windshaft
+```

docs/Map-API.md

@@ -4,10 +4,10 @@ The CartoDB Maps API allows you to generate maps based on data hosted in your Ca
 
 You can create two types of maps with the Maps API:
 
-- **Anonymous maps**  
+- **Anonymous Maps**  
   You can create maps using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example](/cartodb-platform/cartodb-js/getting-started/).
 
-- **Named maps**  
+- **Named Maps**  
   There are also maps that have access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls.
 
 ## Documentation

docs/MapConfig-NamedMaps-extension.md

@@ -6,7 +6,7 @@ This specification describes an extension for
 
 # 2. Changes over specification
 
-This extension introduces a new layer type so it's possible to use a named map by its name as a layer.
+This extension introduces a new layer type so it's possible to use a Named Map by its name as a layer.
 
 ## 2.1 Named layers definition
 
@@ -21,18 +21,18 @@ This extension introduces a new layer type so it's possible to use a named map b
     options: {
 
         // REQUIRED
-        // string, the name for the named map to use
+        // string, the name for the Named Map to use
         name: "world_borders",
 
         // OPTIONAL
-        // object, the replacement values for the named map's template placeholders
+        // object, the replacement values for the Named Map's template placeholders
         // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#instantiate-1 for more details
         config: {
             "color": "#000"
         },
 
         // OPTIONAL
-        // string array, the authorized tokens in case the named map has auth method set to `token`
+        // string array, the authorized tokens in case the Named Map has auth method set to `token`
         // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#named-maps-1 for more details
         auth_tokens: [
             "token1",

docs/anonymous_maps.md

@@ -1,6 +1,6 @@
 # Anonymous Maps
 
-Anonymous maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
 
 
 ## Instantiate
@@ -36,7 +36,7 @@ The response includes:
 
 Attributes | Description
 --- | ---
-layergroupid | The ID for that map, used to compose the URL for the tiles. The final URL is: `https://{account}.cartodb.com/api/v1/map/:layergroupid/{z}/{x}/{y}.png`
+layergroupid | The ID for that map, used to compose the URL for the tiles. The final URL is: `https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png`
 updated_at | The ISO date of the last time the data involved in the query was updated.
 metadata | Includes information about the layers.
 cdn_url | URLs to fetch the data using the best CDN for your zone.
@@ -46,7 +46,7 @@ cdn_url | URLs to fetch the data using the best CDN for your zone.
 #### Call
 
 ```bash
-curl 'https://documentation.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
+curl 'https://{username}.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
 ```
 
 #### Response
@@ -72,12 +72,14 @@ curl 'https://documentation.cartodb.com/api/v1/map' -H 'Content-Type: applicatio
 
 ### Retrieve resources from the layergroup
 
-#### Mapnik tiles can be accessed using
+When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection.
 
-These tiles will get just the mapnik layers. To get individual layers see next section.
+#### Mapnik tiles
+
+These tiles will get just the Mapnik layers. To get individual layers, see the following section.
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 ```
 
 #### Individual layers
@@ -87,21 +89,21 @@ The MapConfig specification holds the layers definition in a 0-based index. Laye
 Individual layers can be accessed using that 0-based index. For UTF grid tiles:
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/{z}/{x}/{y}.grid.json
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer}/{z}/{x}/{y}.grid.json
 ```
 
-In this case, `:layer` as 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example MapConfig.
+In this case, `layer` as 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example MapConfig.
 
 If the MapConfig had a Torque layer at index 1 it could be possible to request it with:
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/1/{z}/{x}/{y}.torque.json
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json
 ```
 
 #### Attributes defined in `attributes` section
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/attributes/:feature_id
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id}
 ```
 
 Which returns JSON with the attributes defined, like:
@@ -113,19 +115,19 @@ Which returns JSON with the attributes defined, like:
 #### Blending and layer selection
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer_filter/{z}/{x}/{y}.png
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png
 ```
 
 Note: currently format is limited to `png`.
 
-`:layer_filter` can be used to select some layers to be rendered together. `:layer_filter` supports two formats:
+`layer_filter` can be used to select some layers to be rendered together. `layer_filter` supports two formats:
 
 - `all` alias
 
-Using `all` as `:layer_filter` will blend all layers in the layergroup
+Using `all` as `layer_filter` will blend all layers in the layergroup
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/all/{z}/{x}/{y}.png
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/all/{z}/{x}/{y}.png
 ```
 
 - Filter by layer index
@@ -133,14 +135,14 @@ https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/
 A list of comma separated layer indexes can be used to just render a subset of layers. For example `0,3,4` will filter and blend layers with indexes 0, 3, and 4.
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/0,3,4/{z}/{x}/{y}.png
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
 ```
 
 Some notes about filtering:
 
   - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors.
-  - Once a mapnik layer is selected, all mapnik layers will get blended. As this may change in the future **it is
-  recommended** to always select all mapnik layers if you want to select at least one so you will get a consistent
+  - Once a Mapnik layer is selected, all Mapnik layers will get blended. As this may change in the future **it is
+  recommended** to always select all Mapnik layers if you want to select at least one so you will get a consistent
   behavior in the future.
   - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this
   may change in the future **it is recommended** to always select the layers in ascending order so you will get a
@@ -161,7 +163,7 @@ GET /api/v1/map?callback=method
 
 Param | Description
 --- | ---
-config | Encoded JSON with the params for creating named maps (the variables defined in the template).
+config | Encoded JSON with the params for creating Named Maps (the variables defined in the template).
 lmza | This attribute contains the same as config but LZMA compressed. It cannot be used at the same time as `config`.
 callback | JSON callback name.
 
@@ -170,7 +172,7 @@ callback | JSON callback name.
 #### Call
 
 ```bash
-curl "https://documentation.cartodb.com/api/v1/map?callback=callback&config=%7B%22version%22%3A%221.0.1%22%2C%22layers%22%3A%5B%7B%22type%22%3A%22cartodb%22%2C%22options%22%3A%7B%22sql%22%3A%22select+%2A+from+european_countries_e%22%2C%22cartocss%22%3A%22%23european_countries_e%7B+polygon-fill%3A+%23FF6600%3B+%7D%22%2C%22cartocss_version%22%3A%222.3.0%22%2C%22interactivity%22%3A%5B%22cartodb_id%22%5D%7D%7D%5D%7D"
+curl "https://{username}.cartodb.com/api/v1/map?callback=callback&config=%7B%22version%22%3A%221.0.1%22%2C%22layers%22%3A%5B%7B%22type%22%3A%22cartodb%22%2C%22options%22%3A%7B%22sql%22%3A%22select+%2A+from+european_countries_e%22%2C%22cartocss%22%3A%22%23european_countries_e%7B+polygon-fill%3A+%23FF6600%3B+%7D%22%2C%22cartocss_version%22%3A%222.3.0%22%2C%22interactivity%22%3A%5B%22cartodb_id%22%5D%7D%7D%5D%7D"
 ```
 
 #### Response
@@ -189,4 +191,4 @@ callback({
 
 ## Remove
 
-Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
+Anonymous Maps cannot be removed by an API call. They will expire after about five minutes, or sometimes longer. If an Anonymous Map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.

docs/general_concepts.md

@@ -4,7 +4,7 @@ The following concepts are the same for every endpoint in the API except when it
 
 ## Auth
 
-By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a named map).
+By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a Named Map).
 
 To execute an authorized request, `api_key=YOURAPIKEY` should be added to the request URL. The param can be also passed as POST param. Using HTTPS is mandatory when you are performing requests that include your `api_key`.
 

docs/named_maps.md

@@ -1,18 +1,24 @@
 # Named Maps
 
-Named maps are essentially the same as anonymous maps except the MapConfig is stored on the server, and the map is given a unique name. You can create named maps from private data, and users without an API Key can view your Named Map (while keeping your data private). The Named map workflow consists of making a call to your database, referencing a table, inserting your variables into the template where placeholders are defined, and creating custom queries.
+Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private). 
 
-The main two differences compared to anonymous maps are:
+The Named Map workflow consists of uploading a MapConfig file to CartoDB servers, to select data from your CartoDB user database by using SQL, and specifying the CartoCSS for your map. 
 
-- **auth layer**  
-  This allows you to control who is able to see the map based on a token auth
+The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps. 
 
-- **templates**  
-  Since the MapConfig is static it can contain some variables so the client can modify the map's appearance using those variables.
+**Tip:** You can also use a Named Map that you created (which is defined by its `name`), to create a map using CartoDB.js. This is achieved by adding the [`namedmap` type](http://docs.cartodb.com/cartodb-platform/cartodb-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
 
-Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see auth section).
+The main differences, compared to Anonymous Maps, is that Named Maps include:
 
-**Note:** There is a limit of 4,096 named maps allowed per account. If you need to create more Named maps, it is recommended to use templates.
+- **auth token**  
+  This allows you to control who is able to see the map based on an auth token, and create a secure Named Map with password-protection.
+
+- **template map**  
+  The template map is static and may contain placeholders, enabling you to modify your maps appearance by using variables. Templates maps are persistent with no preset expiration. They can only be created, or deleted, by a CartoDB user with a valid API KEY (See [auth argument](#arguments)).
+
+  Uploading a MapConfig creates a Named Map. MapConfigs are uploaded to the server by sending the server a "template".json file, which contain the [MapConfig specifications](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/).
+
+**Note:** There is a limit of 4,096 Named Maps allowed per account. If you need to create more Named Maps, it is recommended to use a single Named Map and change the variables using [placeholders](#placeholder-format), instead of uploading multiple [Named Map MapConfigs](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#named-map-layer-options).
 
 ## Create
 
@@ -27,9 +33,12 @@ POST /api/v1/map/named
 Params | Description
 --- | ---
 api_key | is required
+MapConfig | a [Named Map MapConfig](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#named-map-layer-options) is required to create a Named Map
 
 #### template.json
 
+The `name` argument defines how to name this "template_name".json. Note that there are some requirements for how to name a Named Map template. See the [`name`](#arguments) argument description for details.
+
 ```javascript
 {
   "version": "0.0.1",
@@ -84,16 +93,15 @@ api_key | is required
 
 Params | Description
 --- | ---
-name | There can be at most _one_ template with the same name for any user. Valid names start with a letter or a number, and only contain letters, numbers, dashes (-) or underscores (_).
+name | There can only be _one_ template with the same name for any user. Valid names start with a letter or a number, and only contain letters, numbers, dashes (-), or underscores (_). _This is specific to the name of your Named Map that is specified in the `name` property of the template file_.
 
 auth | 
 --- | ---
-|_ method | `"token"` or `"open"` (the default if no `"method"` is given).
-|_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the named map.
-placeholders | Variables not listed here are not substituted. Variables not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
-layergroup | the layer list definition. This is the MapConfig explained in anonymous maps. See [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for more info.
-
-view (optional) | extra keys to specify the compelling area for the map. It can be used to have a static preview of a named map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+|_ method | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
+|_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.cartodb.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
+placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
+layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for more information.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
 --- | ---
 |_ zoom | The zoom level to use
 
@@ -109,11 +117,12 @@ view (optional) | extra keys to specify the compelling area for the map. It can
 |_ |_ east | UpperCorner longitude for the bounding box, in decimal degrees (aka most eastern)
 |_ |_ north | UpperCorner latitude for the bounding box, in decimal degrees (aka most northern)
 
-### Template Format
 
-A templated `layergroup` allows the use of placeholders in the "cartocss" and "sql" elements of the "option" object in any "layer" of a `layergroup` configuration
+### Placeholder Format
 
-Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced.
+Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergoup configurations](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#layergroup-configurations).
+
+Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced inside the Named Maps API.
 
 #### Example
 
@@ -121,7 +130,7 @@ Valid placeholder names start with a letter and can only contain letters, number
 <%= my_color %>
 ```
 
-The set of supported placeholders for a template will need to be explicitly defined with a specific type and default value for each.
+The set of supported placeholders for a template need to be explicitly defined with a specific type, and default value, for each placeholder.
 
 ### Placeholder Types
 
@@ -134,60 +143,64 @@ sql_ident | internal double-quotes will be sql-escaped
 number | can only contain numerical representation
 css_color | can only contain color names or hex-values
 
-Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with new options provided.
+Placeholder default values will be used whenever new values are not provided as options, at the time of creation on the client. They can also be used to test the template by creating a default version with new options provided.
 
-When using templates, be very careful about your selections as they can give broad access to your data if they are defined losely.
+When using templates, be very careful about your selections as they can give broad access to your data if they are defined loosely.
 
 #### Call
 
-```html
+This is the call for creating the Named Map. It is sending the template.json file to the service, and the server responds with the template id.
+
+```bash
 curl -X POST \
    -H 'Content-Type: application/json' \
    -d @template.json \
-   'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
+   'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}'
 ```
 
 #### Response
 
+The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL. 
+
 ```javascript
 {
-  "template_id":"name",
+  "template_id":"name"
 }
 ```
 
 ## Instantiate
 
-Instantiating a map allows you to get the information needed to fetch tiles. That temporal map is an anonymous map.
+Instantiating a Named Map allows you to fetch the map tiles. You can use the Maps API to instantiate, or use the CartoDB.js `createLayer()` function. The result is an Anonymous Map.
 
 #### Definition
 
 ```html
-POST /api/v1/map/named/:template_name
+POST /api/v1/map/named/{template_name}
 ```
 
 #### Param
 
 Param | Description
 --- | ---
-auth_token | optional, but required when `"method"` is set to `"token"`
+auth_token | `"token"` or `"open"` (`"open"` is the default if not specified. Use `"token"` to password-protect your map)
 
 ```javascript
-// params.json
+// params.json, this is required if the Named Map allows variables (if placeholders were defined in the template.json by the user)
 {
  "color": "#ff0000",
  "cartodb_id": 3
 }
 ```
 
-The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400)
+The fields you pass as `params.json` depend on the variables allowed by the Named Map. If there are variables missing, it will raise an error (HTTP 400).
 
-- **auth_token** *optional* if the named map needs auth
+**Note:** It is required that you include a `params.json` file to instantiate a Named Map that contains variables, even if you have no fields to pass and the JSON is empty. (This is specific to when a Named Map allows variables (if placeholders were defined in the template.json by the user).
 
-### Example
+#### Example
 
-You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/:template_name`.
+You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/{template_name}`.
 
-Valid credentials will be needed if required by the template.
+Valid auth token will be needed, if required by the template.
 
 
 #### Call
@@ -196,7 +209,7 @@ Valid credentials will be needed if required by the template.
 curl -X POST \
   -H 'Content-Type: application/json' \
   -d @params.json \
-  'https://documentation.cartodb.com/api/v1/map/named/@template_name?auth_token=AUTH_TOKEN'
+  'https://{username}.cartodb.com/api/v1/map/named/{template_name}?auth_token={auth_token}'
 ```
 
 #### Response
@@ -216,68 +229,14 @@ curl -X POST \
 }
 ```
 
-You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section). However you'll need to show the `auth_token`, if required by the template.
-
-## Using JSONP
-
-There is also a special endpoint to be able to initialize a map using JSONP (for old browsers).
-
-#### Definition
-
-```bash
-GET /api/v1/map/named/:template_name/jsonp
-```
-
-#### Params
-
-Params | Description
---- | ---
-auth_token | optional, but required when `"method"` is set to `"token"`
-config | Encoded JSON with the params for creating named maps (the variables defined in the template)
-lmza | This attribute contains the same as config but LZMA compressed. It cannot be used at the same time than `config`.
-callback | JSON callback name
-
-#### Call
-
-```bash
-curl 'https://documentation.cartodb.com/api/v1/map/named/:template_name/jsonp?auth_token=AUTH_TOKEN&callback=callback&config=template_params_json'
-```
-
-#### Response
-
-```javascript
-callback({
-  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
-  "last_updated":"1970-01-01T00:00:00.000Z"
-  "cdn_url": {
-    "http": "http://cdb.com",
-    "https": "https://cdb.com"
-  }
-})
-```
-
-This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables).
-
-```javascript
-url += "config=" + encodeURIComponent(
-JSON.stringify({ color: 'red' });
-```
-
-The response is in this format:
-
-```javascript
-callback({
-  layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
-  last_updated: "2014-01-27T17:41:03.021Z"
-})
-```
+You can then use the `layergroupid` for fetching tiles and grids as you would normally (see [Anonymous Maps](http://docs.cartodb.com/cartodb-platform/maps-api/anonymous-maps/)).
 
 ## Update
 
 #### Definition
 
 ```bash
-PUT /api/v1/map/named/:template_name
+PUT /api/v1/map/named/{template_name}
 ```
 
 #### Params
@@ -290,9 +249,9 @@ api_key | is required
 
 Same as updating a map.
 
-### Other Info
+### Other Information
 
-Updating a named map removes all the named map instances so they need to be initialized again.
+Updating a Named Map removes all the Named Map instances, so they need to be initialized again.
 
 ### Example
 
@@ -302,7 +261,7 @@ Updating a named map removes all the named map instances so they need to be init
 curl -X PUT \
   -H 'Content-Type: application/json' \
   -d @template.json \
-  'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
+  'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
 ```
 
 #### Response
@@ -325,12 +284,12 @@ If a template with the same name does NOT exist, a 400 HTTP response is generate
 
 ## Delete
 
-Delete the specified template map from the server and it disables any previously initialized versions of the map.
+Deletes the specified template map from the server, and disables any previously initialized versions of the map.
 
 #### Definition
 
 ```bash
-DELETE /api/v1/map/named/:template_name
+DELETE /api/v1/map/named/{template_name}
 ```
 
 #### Params
@@ -344,7 +303,7 @@ api_key | is required
 #### Call
 
 ```bash
-curl -X DELETE 'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
+curl -X DELETE 'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
 ```
 
 #### Response
@@ -378,7 +337,7 @@ api_key | is required
 #### Call
 
 ```bash
-curl -X GET 'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
+curl -X GET 'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}'
 ```
 
 #### Response
@@ -397,14 +356,14 @@ curl -X GET 'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
 }
 ```
 
-## Getting a Specific Template
+## Get Template Definition
 
-This gets the definition of a template.
+This gets the definition of a requested template.
 
 #### Definition
 
 ```bash
-GET /api/v1/map/named/:template_name
+GET /api/v1/map/named/{template_name}
 ```
 
 #### Params
@@ -418,14 +377,14 @@ api_key | is required
 #### Call
 
 ```bash
-curl -X GET 'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
+curl -X GET 'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
 ```
 
 #### Response
 
 ```javascript
 {
-  "template": {...} // see template.json above
+  "template": {...} // see [template.json](#templatejson)
 }
 ```
 
@@ -437,36 +396,136 @@ curl -X GET 'https://documentation.cartodb.com/api/v1/map/named/:template_name?a
 }
 ```
 
-## Use CartoDB.js to Create Named Maps
-Named maps can be used with CartoDB.js by specifying a named map in a layer source as follows. Named maps are treated almost the same as other layer source types in most other ways.
+## JSONP for Named Maps
 
-```js
-var layerSource = {
-  user_name: '{your_user_name}',
-  type: 'namedmap',
-  named_map: {
-    name: '{template_name}',
-	layers: [{
-	  layer_name: "layer1",
-      interactivity: "column1, column2, ..."
-	}]
-  }
-}
+If using a [JSONP](https://en.wikipedia.org/wiki/JSONP) (for old browsers) request, there is a special endpoint used to initialize and create a Named Map.
 
-cartodb.createLayer('map_dom_id',layerSource)
-  .addTo(map_object);
+#### Definition
 
+```bash
+GET /api/v1/map/named/{template_name}/jsonp
 ```
 
-[CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js/) has methods for accessing your named maps.
+#### Params
+
+Params | Description
+--- | ---
+auth_token | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
+params | Encoded JSON with the params (variables) needed for the Named Map
+lmza | You can use an LZMA compressed file instead of a params JSON file
+callback | JSON callback name
+
+#### Call
+
+```bash
+curl 'https://{username}.cartodb.com/api/v1/map/named/{template_name}/jsonp?auth_token={auth_token}&callback=callback&config=template_params_json'
+```
+
+#### Response
+
+```javascript
+callback({
+  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
+  "last_updated":"1970-01-01T00:00:00.000Z"
+  "cdn_url": {
+    "http": "http://cdb.com",
+    "https": "https://cdb.com"
+  }
+})
+```
+
+This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables).
+
+```javascript
+url += "config=" + encodeURIComponent(
+JSON.stringify({ color: 'red' });
+```
+
+The response is:
+
+```javascript
+callback({
+  layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
+  last_updated: "2014-01-27T17:41:03.021Z"
+})
+```
+
+## CartoDB.js for Named Maps
+You can use a Named Map that you created (which is defined by its `name`), to create a map using CartoDB.js. This is achieved by adding the [`namedmap` type](http://docs.cartodb.com/cartodb-platform/cartodb-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
+
+```javascript
+{
+  user_name: '{username}', // Required
+  type: 'namedmap', // Required
+  named_map: {
+    name: '{name_of_map}', // Required, the 'name' of the Named Map that you have created
+    // Optional
+    layers: [{
+      layer_name: "sublayer0", // Optional
+      interactivity: "column1, column2, ..." // Optional
+    },
+    {
+      layer_name: "sublayer1",
+      interactivity: "column1, column2, ..."
+    },
+      ...
+    ],
+    // Optional
+    params: {
+      color: "hex_value",
+      num: 2
+    }
+  }
+}
+```
+**Note:** Instantiating a Named Map over a `createLayer` does not require an API Key and by default, does not include auth tokens. _If_ you defined auth tokens for the Named Map configuration, then you will have to include them.
+
+[CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js/) has methods for accessing your Named Maps.
 
 1. [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
+
+    **Note:** The CartoDB.js `layer.setParams()` function is not supported when using Named Maps for Torque.
+
 2. [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
 
-### Complete Examples of Named Maps created with CartoDB.js
+#### Examples of Named Maps created with CartoDB.js
 
-- [Named map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+- [Named Map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
 
-- [Named map with interactivity and config file used to create it](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+- [Named Map with interactivity](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
 
 - [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)
+
+## Fetching XYZ Tiles for Named Maps
+
+Optionally, authenticated users can fetch projected tiles (XYZ tiles or Mapnik Retina tiles) for your Named Map.
+
+### Fetch XYZ Tiles Directly with a URL
+
+Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map. 
+
+To call a template_id in a URL:
+
+`/{template_id}/{layer}/{z}/{x}/{y}.{format}`
+
+For example, a complete URL might appear as:
+
+"https://{username}.cartodb.com/api/v1/map/named/{template_id}/{layer}/{z}/{x}/{y}.png"
+
+The placeholders indicate the following:
+
+- [`template_id`](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#response) is the response of your Named Map.
+- layers can be a number (referring to the # layer of your map), all layers of your map, or a list of layers.
+  - To show just the basemap layer, enter the number value `0` in the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/0/{z}/{x}/{y}.png"
+  - To show the first layer, enter the number value `1` in the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/1/{z}/{x}/{y}.png"
+  - To show all layers, enter the value `all` for the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/all/{z}/{x}/{y}.png"
+  - To show a [list of layers](http://docs.cartodb.com/cartodb-platform/maps-api/anonymous-maps/#blending-and-layer-selection), enter the comma separated layer value as 0,1,2 in the layer placeholder. For example, to show the basemap and the first layer, "https://{username}.cartodb.com/api/v1/map/named/{template_id}/0,1/{z}/{x}/{y}.png"
+
+
+### Get Mapnik Retina Tiles
+
+Mapnik Retina tiles are not directly supported for Named Maps, so you cannot use the Named Map template_id. To fetch Mapnik Retina tiles, get the [layergroupid](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#response-1) to initialize the map.
+
+Instantiate the map by using your `layergroupid` in the token placeholder:
+
+ `{token}/{z}/{x}/{y}@{scale_factor}?{x}.{format}`

docs/quickstart.md

@@ -1,8 +1,8 @@
 # Quickstart
 
-## Anonymous maps
+## Anonymous Maps
 
-Here is an example of how to create an anonymous map with JavaScript:
+Here is an example of how to create an Anonymous Map with JavaScript:
 
 ```javascript
 var mapconfig = {
@@ -22,18 +22,18 @@ $.ajax({
   type: 'POST',
   dataType: 'json',
   contentType: 'application/json',
-  url: 'https://documentation.cartodb.com/api/v1/map',
+  url: 'https://{username}.cartodb.com/api/v1/map',
   data: JSON.stringify(mapconfig),
   success: function(data) {
-    var templateUrl = 'https://documentation.cartodb.com/api/v1/map/' + data.layergroupid + '/{z}/{x}/{y}.png'
+    var templateUrl = 'https://{username}.cartodb.com/api/v1/map/' + data.layergroupid + '/{z}/{x}/{y}.png'
     console.log(templateUrl);
   }
 })
 ```
 
-## Named maps
+## Named Maps
 
-Let's create a named map using some private tables in a CartoDB account.
+Let's create a Named Map using some private tables in a CartoDB account.
 The following map config sets up a map of European countries that have a white fill color:
 
 ```javascript
@@ -56,12 +56,12 @@ The following map config sets up a map of European countries that have a white f
 }
 ```
 
-The map config needs to be sent to CartoDB's Map API using an authenticated call. Here we will use a command line tool called `curl`. For more info about this tool, see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl), or type `man curl` in bash. Using `curl`, and storing the config from above in a file `mapconfig.json`, the call would look like:
+The MapConfig needs to be sent to CartoDB's Map API using an authenticated call. Here we will use a command line tool called `curl`. For more info about this tool, see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl), or type `man curl` in bash. Using `curl`, and storing the config from above in a file `MapConfig.json`, the call would look like:
 
 #### Call
 
 ```bash
-curl 'https://{account}.cartodb.com/api/v1/map/named?api_key=APIKEY' -H 'Content-Type: application/json' -d @mapconfig.json
+curl 'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}' -H 'Content-Type: application/json' -d @mapconfig.json
 ```
 
 To get the `URL` to fetch the tiles you need to instantiate the map, where `template_id` is the template name from the previous response.
@@ -69,7 +69,7 @@ To get the `URL` to fetch the tiles you need to instantiate the map, where `temp
 #### Call
 
 ```bash
-curl -X POST 'https://{account}.cartodb.com/api/v1/map/named/:template_id' -H 'Content-Type: application/json'
+curl -X POST 'https://{username}.cartodb.com/api/v1/map/named/{template_id}' -H 'Content-Type: application/json'
 ```
 
 The response will return JSON with properties for the `layergroupid`, the timestamp (`last_updated`) of the last data modification and some key/value pairs with `metadata` for the `layers`.
@@ -96,5 +96,5 @@ Note: all `layers` in `metadata` will always have a `type` string and a `meta` d
 You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
 
 ```bash
-https://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 ```

docs/static_maps_api.md

@@ -1,28 +1,28 @@
 # Static Maps API
 
-The Static Maps API can be initiated using both named and anonymous maps using the 'layergroupid' token. The API can be used to create static images of parts of maps and thumbnails for use in web design, graphic design, print, field work, and many other applications that require standard image formats.
+The Static Maps API can be initiated using both Named and Anonymous Maps using the 'layergroupid' token. The API can be used to create static images of parts of maps and thumbnails for use in web design, graphic design, print, field work, and many other applications that require standard image formats.
 
 ## Maps API endpoints
 
-Begin by instantiating either a named or anonymous map using the `layergroupid token` as demonstrated in the Maps API documentation above. The `layergroupid` token calls to the map and allows for parameters in the definition to generate static images.
+Begin by instantiating either a Named or Anonymous Map using the `layergroupid token` as demonstrated in the Maps API documentation above. The `layergroupid` token calls to the map and allows for parameters in the definition to generate static images.
 
 ### Zoom + center
 
 #### Definition
 
 ```bash
-GET /api/v1/map/static/center/:token/:z/:lat/:lng/:width/:height.:format
+GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format}
 ```
 
 #### Params
 
 Param | Description
 --- | ---
-:token | the layergroupid token from the map instantiation
-:z | the zoom level of the map
-:lat | the latitude for the center of the map
+token | the layergroupid token from the map instantiation
+z | the zoom level of the map
+lat | the latitude for the center of the map
 
-:format | the format for the image, supported types: `png`, `jpg`
+format | the format for the image, supported types: `png`, `jpg`
 --- | ---
 |_ jpg | will have a default quality of 85.
 
@@ -31,57 +31,56 @@ Param | Description
 #### Definition
 
 ```bash
-GET /api/v1/map/static/bbox/:token/:bbox/:width/:height.:format`
+GET /api/v1/map/static/bbox/{token}/{bbox}/{width}/{height}.{format}`
 ```
 
 #### Params
 
 Param | Description
 --- | ---
-:token | the layergroupid token from the map instantiation
+token | the layergroupid token from the map instantiation
 
-:bbox | the bounding box in WGS 84 (EPSG:4326), comma separated values for:
+bbox | the bounding box in WGS 84 (EPSG:4326), comma separated values for:
 --- | ---
  | LowerCorner longitude, in decimal degrees (aka most western)
  | LowerCorner latitude, in decimal degrees (aka most southern)
  | UpperCorner longitude, in decimal degrees (aka most eastern)
  | UpperCorner latitude, in decimal degrees (aka most northern)
-:width | the width in pixels for the output image
-:height | the height in pixels for the output image
-:format | the format for the image, supported types: `png`, `jpg`
+width | the width in pixels for the output image
+height | the height in pixels for the output image
+format | the format for the image, supported types: `png`, `jpg`
 
-:format | the bounding box in WGS 84 (EPSG:4326), comma separated values for:
+format | the bounding box in WGS 84 (EPSG:4326), comma separated values for:
 --- | ---
 |_ jpg | will have a default quality of 85.
 
 Note: you can see this endpoint as
 
 ```bash
-GET /api/v1/map/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`
+GET /api/v1/map/static/bbox/{token}/{west},{south},{east},{north}/{width}/{height}.{format}`
 ```
 
-### Named map
+### Named Map
 
 #### Definition
 
 ```bash
-GET /api/v1/map/static/named/:name/:width/:height.:format
+GET /api/v1/map/static/named/{name}/{width}/{height}.{format}
 ```
 
 #### Params
 
 Param | Description
 --- | ---
-:name | the name of the named map
-:width | the width in pixels for the output image
-:height | the height in pixels for the output image
-:height | the height in pixels for the output image
+name | the name of the Named Map
+width | the width in pixels for the output image
+height | the height in pixels for the output image
 
-:format | the format for the image, supported types: `png`, `jpg`
+format | the format for the image, supported types: `png`, `jpg`
 --- | ---
 |_ jpg | will have a default quality of 85.
 
-A named maps static image will get its constraints from the [view in the template](#Arguments), if `view` is not present it will estimate the extent based on the involved tables otherwise it fallback to `"zoom": 1`, `"lng": 0` and `"lat": 0`.
+A Named Maps static image will get its constraints from the [`view` argument of the Create Named Map function](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#arguments). If `view` is not defined, it will estimate the extent based on the involved tables, otherwise it fallbacks to `"zoom": 1`, `"lng": 0` and `"lat": 0`.
 
 #### Layers
 
@@ -161,7 +160,7 @@ After instantiating a map from a CartoDB account:
 #### Call
 
 ```bash
- GET /api/v1/map/static/center/4b615ff367e498e770e7d05e99181873:1420231989550.8699/14/40.71502926732618/-73.96039009094238/600/400.png
+ GET /api/v1/map/static/center/{layergroupid}/{z}/{x}/{y}/{width}/{height}.png
 ```
 
 #### Response

lib/cartodb/api/overviews_metadata_api.js

@@ -25,8 +25,7 @@ OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, c
     var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
     this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
         if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not get overviews metadata: ' + msg));
+            callback(err);
             return;
         }
         var metadata = {};

lib/cartodb/api/query_tables_api.js

@@ -1,96 +0,0 @@
-function QueryTablesApi(pgQueryRunner) {
-    this.pgQueryRunner = pgQueryRunner;
-}
-
-var affectedTableRegexCache = {
-    bbox: /!bbox!/g,
-    scale_denominator: /!scale_denominator!/g,
-    pixel_width: /!pixel_width!/g,
-    pixel_height: /!pixel_height!/g
-};
-
-module.exports = QueryTablesApi;
-
-
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
-    var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
-
-    this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
-        if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch source tables: ' + msg));
-            return;
-        }
-
-        // This is an Array, so no need to split into parts
-        var tableNames = rows[0].cdb_querytablestext;
-        return callback(null, tableNames);
-    });
-};
-
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
-    var query = [
-        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
-        ')',
-        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m',
-        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
-        if (err || rows.length === 0) {
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-            return;
-        }
-
-        var result = rows[0];
-
-        // This is an Array, so no need to split into parts
-        var tableNames = result.tablenames;
-
-        var lastUpdatedTime = result.max || 0;
-
-        callback(null, {
-            affectedTables: tableNames,
-            lastUpdatedTime: lastUpdatedTime * 1000
-        });
-    });
-};
-
-QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
-    if (!Array.isArray(tableNames) || tableNames.length === 0) {
-        return callback(null, 0);
-    }
-
-    var query = [
-        'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
-            tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
-        '])'
-    ].join(' ');
-
-    this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
-        if (err) {
-            var msg = err.message ? err.message : err;
-            return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
-        }
-        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
-        var lastUpdated = 0;
-        if (rows.length !== 0) {
-            lastUpdated = rows[0].max || 0;
-        }
-
-        return callback(null, lastUpdated*1000);
-    });
-};
-
-function prepareSql(sql) {
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
-}

lib/cartodb/api/tables_extent_api.js

@@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
  *  `table_name` format as valid input
  * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
  */
-TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) {
-    var estimatedExtentSQLs = tableNames.map(function(tableName) {
-        var schemaTable = tableName.split('.');
-        if (schemaTable.length > 1) {
-            return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
-        }
-        return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
+TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
+    var estimatedExtentSQLs = tables.map(function(table) {
+        return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
     });
 
     var query = [

lib/cartodb/backends/pg_connection.js

@@ -1,5 +1,6 @@
 var assert = require('assert');
 var step = require('step');
+var PSQL = require('cartodb-psql');
 var _ = require('underscore');
 
 function PgConnection(metadataBackend) {
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
         }
     );
 };
+
+
+/**
+ * Returns a `cartodb-psql` object for a given username.
+ * @param {String} username
+ * @param {Function} callback function({Error}, {PSQL})
+ */
+
+PgConnection.prototype.getConnection = function(username, callback) {
+    var self = this;
+
+    var params = {};
+
+    require('debug')('cachechan')("getConn1");
+    step(
+        function setAuth() {
+            self.setDBAuth(username, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.setDBConn(username, params, this);
+        },
+        function openConnection(err) {
+            assert.ifError(err);
+            return callback(err, new PSQL({
+                user: params.dbuser,
+                pass: params.dbpass,
+                host: params.dbhost,
+                port: params.dbport,
+                dbname: params.dbname
+            }));
+        }
+    );
+};

lib/cartodb/backends/template_maps.js

@@ -477,7 +477,11 @@ o.instance = function(template, params) {
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
-    if ( lyropt.cartocss ) {
+
+    if ( params.styles && params.styles[i] ) {
+        // dynamic styling for this layer
+        lyropt.cartocss = params.styles[i];
+    } else if ( lyropt.cartocss ) {
         lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
     }
     if ( lyropt.sql) {

lib/cartodb/cache/model/database_tables_entry.js

@@ -1,24 +0,0 @@
-var crypto = require('crypto');
-
-function DatabaseTables(dbName, tableNames) {
-    this.namespace = 't';
-    this.dbName = dbName;
-    this.tableNames = tableNames;
-}
-
-module.exports = DatabaseTables;
-
-
-DatabaseTables.prototype.key = function() {
-    return this.tableNames.map(function(tableName) {
-        return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
-    }.bind(this));
-};
-
-DatabaseTables.prototype.getCacheChannel = function() {
-    return this.dbName + ':' + this.tableNames.join(',');
-};
-
-function shortHashKey(target) {
-    return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
-}

lib/cartodb/cache/named_map_provider_cache.js

@@ -7,13 +7,14 @@ var queue = require('queue-async');
 
 var LruCache = require("lru-cache");
 
-function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
+function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, overviewsAdapter, turboCartocssAdapter) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.overviewsAdapter = overviewsAdapter;
+    this.turboCartocssAdapter = turboCartocssAdapter;
 
     this.providerCache = new LruCache({ max: 2000 });
 }
@@ -30,8 +31,9 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
             this.templateMaps,
             this.pgConnection,
             this.userLimitsApi,
-            this.queryTablesApi,
             this.namedLayersAdapter,
+            this.overviewsAdapter,
+            this.turboCartocssAdapter,
             user,
             templateId,
             config,

lib/cartodb/controllers/layergroup.js

@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
 var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
+
+var QueryTables = require('cartodb-query-tables');
 
 /**
  * @param {AuthApi} authApi
@@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
  * @param {WidgetBackend} widgetBackend
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
- * @param {QueryTablesApi} queryTablesApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @constructor
  */
 function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
-                          widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
+                          widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
     BaseController.call(this, authApi, pgConnection);
 
+    this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
@@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
     this.widgetBackend = widgetBackend;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
 }
 
@@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 global.logger.warn('ERROR generating cache channel: ' + err);
             }
             if (!!affectedTables) {
-                var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
-                self.surrogateKeysCache.tag(res, tablesCacheEntry);
+                res.set('X-Cache-Channel', affectedTables.getCacheChannel());
+                self.surrogateKeysCache.tag(res, affectedTables);
             }
             self.send(req, res, body, status, headers);
         }
@@ -366,17 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 throw new Error("this request doesn't need an X-Cache-Channel generated");
             }
 
-            self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(user, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
-        function buildCacheChannel(err, tableNames) {
+        function buildCacheChannel(err, tables) {
             assert.ifError(err);
+            self.layergroupAffectedTables.set(dbName, layergroupId, tables);
 
-            self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
-
-            return tableNames;
+            return tables;
         },
-        function finish(err, affectedTables) {
-            callback(err, affectedTables);
-        }
+        callback
     );
 };

lib/cartodb/controllers/map.js

@@ -2,6 +2,7 @@ var _ = require('underscore');
 var assert = require('assert');
 var step = require('step');
 var windshaft = require('windshaft');
+var QueryTables = require('cartodb-query-tables');
 
 var util = require('util');
 var BaseController = require('./base');
@@ -13,12 +14,10 @@ var MapConfig = windshaft.model.MapConfig;
 var Datasource = windshaft.model.Datasource;
 
 var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
 
 var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
 var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
 var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
-var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter');
 
 /**
  * @param {AuthApi} authApi
@@ -26,7 +25,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
  * @param {TemplateMaps} templateMaps
  * @param {MapBackend} mapBackend
  * @param metadataBackend
- * @param {QueryTablesApi} queryTablesApi
  * @param {OverviewsMetadataApi} overviewsMetadataApi
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
@@ -34,8 +32,8 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
  * @constructor
  */
 function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
-                       queryTablesApi, overviewsMetadataApi,
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables,
+                       overviewsAdapter, turboCartoCssAdapter) {
 
     BaseController.call(this, authApi, pgConnection);
 
@@ -43,14 +41,13 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.templateMaps = templateMaps;
     this.mapBackend = mapBackend;
     this.metadataBackend = metadataBackend;
-    this.queryTablesApi = queryTablesApi;
-    this.overviewsMetadataApi = overviewsMetadataApi;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
+    this.turboCartoCssAdapter = turboCartoCssAdapter;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
-    this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi);
+    this.overviewsAdapter = overviewsAdapter;
 }
 
 util.inherits(MapController, BaseController);
@@ -154,20 +151,35 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
             );
         },
         function addOverviewsInformation(err, requestMapConfig, datasource) {
-          assert.ifError(err);
-          var next = this;
-          self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
-              function(err, layers) {
-                  if (err) {
-                      return next(err);
-                  }
+            assert.ifError(err);
+            var next = this;
+            self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
+                if (err) {
+                    return next(err);
+                }
 
-                  if (layers) {
-                      requestMapConfig.layers = layers;
-                  }
-                  return next(null, requestMapConfig, datasource);
-              }
-          );
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
+        },
+        function parseTurboCartoCss(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+
+            var next = this;
+            self.turboCartoCssAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
         },
         function createLayergroup(err, requestMapConfig, datasource) {
             assert.ifError(err);
@@ -186,6 +198,8 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
             if (err) {
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
+                addWidgetsUrl(req.context.user, layergroup);
+
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -214,8 +228,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 self.templateMaps,
                 self.pgConnection,
                 self.userLimitsApi,
-                self.queryTablesApi,
                 self.namedLayersAdapter,
+                self.overviewsAdapter,
+                self.turboCartoCssAdapter,
                 cdbuser,
                 req.params.template_id,
                 templateParams,
@@ -224,22 +239,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
             );
             mapConfigProvider.getMapConfig(this);
         },
-        function addOverviewsInformation(err, requestMapConfig, rendererParams/*, context*/) {
-          assert.ifError(err);
-          var next = this;
-          self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers,
-              function(err, layers) {
-                  if (err) {
-                      return next(err);
-                  }
-
-                  if (layers) {
-                      requestMapConfig.layers = layers;
-                  }
-                  return next(null, requestMapConfig, rendererParams);
-              }
-          );
-        },
         function createLayergroup(err, mapConfig_, rendererParams) {
             assert.ifError(err);
             mapConfig = mapConfig_;
@@ -260,6 +259,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
+                addWidgetsUrl(req.context.user, layergroup);
+
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
 
@@ -314,46 +315,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
     var layergroupId = layergroup.layergroupid;
 
     step(
-        function checkCachedAffectedTables() {
-            return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
+        function getPgConnection() {
+            self.pgConnection.getConnection(username, this);
         },
-        function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
+        function getAffectedTablesAndLastUpdatedTime(err, connection) {
             assert.ifError(err);
-            if (hasCache) {
-                var next = this;
-                var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
-                self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
-                    if (err) {
-                        return next(err);
-                    }
-                    return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
-                });
-            } else {
-                self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
-            }
+            QueryTables.getAffectedTablesFromQuery(connection, sql, this);
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
             if (req.profiler) {
                 req.profiler.done('queryTablesAndLastUpdated');
             }
             assert.ifError(err);
-            self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
+            // feed affected tables cache so it can be reused from, for instance, layergroup controller
+            self.layergroupAffectedTables.set(dbName, layergroupId, result);
 
             // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
-            layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
+            layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
+            layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
 
             // TODO this should take into account several URL patterns
             addWidgetsUrl(username, layergroup);
-
             if (req.method === 'GET') {
-                var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
                 var ttl = global.environment.varnish.layergroupTtl || 86400;
                 res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
                 res.set('Last-Modified', (new Date()).toUTCString());
-                res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
-                if (result.affectedTables && result.affectedTables.length > 0) {
-                    self.surrogateKeysCache.tag(res, tableCacheEntry);
+                res.set('X-Cache-Channel', result.getCacheChannel());
+                if (result.tables && result.tables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
                 }
             }
 

lib/cartodb/controllers/named_maps.js

@@ -9,8 +9,6 @@ var BaseController = require('./base');
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-var TablesCacheEntry = require('../cache/model/database_tables_entry');
-
 function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
     BaseController.call(this, authApi, pgConnection);
@@ -44,7 +42,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
 
     var self = this;
 
-    var dbName = req.params.dbname;
     step(
         function getAffectedTablesAndLastUpdatedTime() {
             namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
@@ -54,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
             if (err) {
                 global.logger.log('ERROR generating cache channel: ' + err);
             }
-            if (!result || !!result.affectedTables) {
+            if (!result || !!result.tables) {
                 // we increase cache control as we can invalidate it
                 res.set('Cache-Control', 'public,max-age=31536000');
 
                 var lastModifiedDate;
                 if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.lastUpdatedTime);
+                    lastModifiedDate = new Date(result.getLastUpdatedAt());
                 } else {
                     lastModifiedDate = new Date();
                 }
                 res.set('Last-Modified', lastModifiedDate.toUTCString());
 
-                var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
-                res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
-                if (result.affectedTables.length > 0) {
-                    self.surrogateKeysCache.tag(res, tablesCacheEntry);
+                res.set('X-Cache-Channel', result.getCacheChannel());
+                if (result.tables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
                 }
             }
             self.send(req, res, resource, 200);
@@ -231,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
                     return next(null);
                 }
 
-                var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
+                var affectedTables = affectedTablesAndLastUpdate.tables || [];
 
                 if (affectedTables.length === 0) {
                     return next(null);

lib/cartodb/controllers/named_maps_admin.js

@@ -65,6 +65,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
     var cdbuser = req.context.user;
     var template;
     var tpl_id;
+
     step(
         function checkPerms(){
             self.authApi.authorizedByAPIKey(cdbuser, req, this);

lib/cartodb/models/mapconfig/named_map_provider.js

@@ -5,18 +5,21 @@ var dot = require('dot');
 var step = require('step');
 var MapConfig = require('windshaft').model.MapConfig;
 var templateName = require('../../backends/template_maps').templateName;
+var QueryTables = require('cartodb-query-tables');
 
 /**
  * @constructor
  * @type {NamedMapMapConfigProvider}
  */
-function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
+function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi,
+                                   namedLayersAdapter, overviewsAdapter, turboCartoCssAdapter,
                                    owner, templateId, config, authToken, params) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.userLimitsApi = userLimitsApi;
-    this.queryTablesApi = queryTablesApi;
     this.namedLayersAdapter = namedLayersAdapter;
+    this.turboCartoCssAdapter = turboCartoCssAdapter;
+    this.overviewsAdapter = overviewsAdapter;
 
     this.owner = owner;
     this.templateName = templateName(templateId);
@@ -91,6 +94,38 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
                 }
             );
         },
+        function addOverviewsInformation(err, _mapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+
+            self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, function(err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    _mapConfig.layers = layers;
+                }
+
+                return next(null, _mapConfig, datasource);
+            });
+        },
+        function parseTurboCartoCss(err, _mapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+
+            self.turboCartoCssAdapter.getLayers(self.owner, _mapConfig.layers, function (err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    _mapConfig.layers = layers;
+                }
+
+                return next(null, _mapConfig, datasource);
+            });
+        },
         function beforeLayergroupCreate(err, _mapConfig, _datasource) {
             assert.ifError(err);
             mapConfig = _mapConfig;
@@ -256,7 +291,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
         },
         function getAffectedTables(err, sql) {
             assert.ifError(err);
-            self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(self.owner, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
         },
         function finish(err, result) {
             self.affectedTablesAndLastUpdate = result;

lib/cartodb/server.js

@@ -19,7 +19,6 @@ var windshaft = require('windshaft');
 var mapnik = windshaft.mapnik;
 
 var TemplateMaps = require('./backends/template_maps.js');
-var QueryTablesApi = require('./api/query_tables_api');
 var OverviewsMetadataApi = require('./api/overviews_metadata_api');
 var UserLimitsApi = require('./api/user_limits_api');
 var AuthApi = require('./api/auth_api');
@@ -31,6 +30,10 @@ var PgConnection = require('./backends/pg_connection');
 var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
+var MapConfigOverviewsAdapter = require('./models/mapconfig_overviews_adapter');
+
+var TurboCartocssParser = require('./utils/style/turbo-cartocss-parser');
+var TurboCartocssAdapter = require('./utils/style/turbo-cartocss-adapter');
 
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
@@ -52,7 +55,6 @@ module.exports = function(serverOptions) {
     var metadataBackend = cartodbRedis({pool: redisPool});
     var pgConnection = new PgConnection(metadataBackend);
     var pgQueryRunner = new PgQueryRunner(pgConnection);
-    var queryTablesApi = new QueryTablesApi(pgQueryRunner);
     var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
     var userLimitsApi = new UserLimitsApi(metadataBackend, {
         limits: {
@@ -142,7 +144,19 @@ module.exports = function(serverOptions) {
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
+    var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
+
+    var turboCartoCssParser = new TurboCartocssParser(pgQueryRunner);
+    var turboCartocssAdapter = new TurboCartocssAdapter(turboCartoCssParser);
+
+    var namedMapProviderCache = new NamedMapProviderCache(
+        templateMaps,
+        pgConnection,
+        userLimitsApi,
+        overviewsAdapter,
+        turboCartocssAdapter
+    );
+
     ['update', 'delete'].forEach(function(eventType) {
         templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
     });
@@ -166,7 +180,6 @@ module.exports = function(serverOptions) {
         new windshaft.backend.Widget(),
         surrogateKeysCache,
         userLimitsApi,
-        queryTablesApi,
         layergroupAffectedTablesCache
     ).register(app);
 
@@ -176,11 +189,11 @@ module.exports = function(serverOptions) {
         templateMaps,
         mapBackend,
         metadataBackend,
-        queryTablesApi,
-        overviewsMetadataApi,
         surrogateKeysCache,
         userLimitsApi,
-        layergroupAffectedTablesCache
+        layergroupAffectedTablesCache,
+        overviewsAdapter,
+        turboCartocssAdapter
     ).register(app);
 
     new controller.NamedMaps(

lib/cartodb/utils/overviews_query_rewriter.js

@@ -74,14 +74,9 @@ function overviews_view_name(table) {
 }
 
 // replace a table name in a query by anoter name
-function replace_table_in_query(sql, old_table_name, new_table_name) {
+function replace_table_in_query(sql, old_table_name, replacement) {
     var old_table = TableNameParser.parse(old_table_name);
-    var new_table = TableNameParser.parse(new_table_name);
     var old_table_ident = TableNameParser.table_identifier(old_table);
-    var new_table_ident = TableNameParser.table_identifier(new_table);
-
-    // text that will be substituted by the table name pattern
-    var replacement = new_table_ident;
 
     // regular expression prefix (beginning) to match a table name
     function pattern_prefix(schema, identifier) {
@@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
 function overviews_query(query, overviews, zoom_level_expression) {
     var replaced_query = query;
     var sql = "WITH\n  _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
+    var replacement;
     for ( var table in overviews ) {
         if (overviews.hasOwnProperty(table)) {
             var table_overviews = overviews[table];
             var table_view = overviews_view_name(table);
-            replaced_query = replace_table_in_query(replaced_query, table, table_view);
-            sql += ",\n  " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n  )";
+            replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
+            replaced_query = replace_table_in_query(replaced_query, table, replacement);
         }
     }
     if ( replaced_query !== query ) {
@@ -172,9 +168,16 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
 };
 
 OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
-  return !!sql.match(
-    /^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
+  var basic_query = /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
+  var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i');
+  // queries for named maps are wrapped like this:
+  var wrapped_query = new RegExp(
+    "^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" +
+    basic_query.source +
+    "\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$",
+    'i'
   );
+  return !!(sql.match(unwrapped_query) || sql.match(wrapped_query));
 };
 
 OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {

lib/cartodb/utils/style/postgres-datasource.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function createTemplate(method) {
+    return dot.template([
+        'SELECT',
+        method,
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
+    ].join('\n'));
+}
+
+var methods = {
+    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
+    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+};
+
+var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {
+    methodTemplates[methodName] = createTemplate(methods[methodName]);
+    return methodTemplates;
+}, {});
+
+function PostgresDatasource (pgQueryRunner, username, query) {
+    this.pgQueryRunner = pgQueryRunner;
+    this.username = username;
+    this.query = query;
+}
+
+PostgresDatasource.prototype.getName = function () {
+    return 'PostgresDatasource';
+};
+
+PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) {
+    var methodName = methods.hasOwnProperty(method) ? method : 'quantiles';
+    var template = methodTemplates[methodName];
+
+    var query = template({ _column: column, _sql: this.query, _buckets: buckets });
+
+    this.pgQueryRunner.run(this.username, query, function (err, result) {
+        if (err) {
+            return callback(err);
+        }
+
+        var ramp = result[0][methodName].sort(function(a, b) {
+            return a - b;
+        });
+
+        return callback(null, ramp);
+    });
+};
+
+module.exports = PostgresDatasource;

lib/cartodb/utils/style/turbo-cartocss-adapter.js

@@ -0,0 +1,56 @@
+'use strict';
+
+var queue = require('queue-async');
+
+function TurboCartocssAdapter(turboCartocssParser) {
+    this.turboCartocssParser = turboCartocssParser;
+}
+
+module.exports = TurboCartocssAdapter;
+
+TurboCartocssAdapter.prototype.getLayers = function (username, layers, callback) {
+    var self = this;
+
+    if (!layers || layers.length === 0) {
+        return callback(null, layers);
+    }
+
+    var parseCartoCssQueue = queue(layers.length);
+
+    layers.forEach(function(layer) {
+        parseCartoCssQueue.defer(self._parseCartoCss.bind(self), username, layer);
+    });
+
+    parseCartoCssQueue.awaitAll(function (err, layers) {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, layers);
+    });
+};
+
+TurboCartocssAdapter.prototype._parseCartoCss = function (username, layer, callback) {
+    if (isNotLayerToParseCartocss(layer)) {
+        return process.nextTick(function () {
+            callback(null, layer);
+        });
+    }
+
+    this.turboCartocssParser.process(username, layer.options.cartocss, layer.options.sql, function (err, cartocss) {
+        // Ignore turbo-cartocss errors and continue
+        if (!err && cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+
+        callback(null, layer);
+    });
+};
+
+function isNotLayerToParseCartocss(layer) {
+    if (!layer || !layer.options || !layer.options.cartocss || !layer.options.sql) {
+        return true;
+    }
+
+    return false;
+}

lib/cartodb/utils/style/turbo-cartocss-parser.js

@@ -0,0 +1,15 @@
+'use strict';
+
+var turboCartoCss = require('turbo-cartocss');
+var PostgresDatasource = require('./postgres-datasource');
+
+function TurboCartocssParser (pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = TurboCartocssParser;
+
+TurboCartocssParser.prototype.process = function (username, cartocss, sql, callback) {
+    var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql);
+    turboCartoCss(cartocss, datasource, callback);
+};

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.24.0",
+    "version": "2.32.0",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -26,7 +26,7 @@
         "node-statsd": "~0.0.7",
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "1.10.1",
+        "windshaft": "1.17.0",
         "step": "~0.0.6",
         "queue-async": "~1.0.7",
         "request": "~2.62.0",
@@ -36,7 +36,9 @@
         "redis-mpool": "~0.4.0",
         "lru-cache": "2.6.5",
         "lzma": "~1.3.7",
-        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
+        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
+        "cartodb-query-tables": "~0.1.0",
+        "turbo-cartocss": "0.4.0"
     },
     "devDependencies": {
         "istanbul": "~0.3.6",

test/acceptance/dynamic-styling-named-maps.js

@@ -0,0 +1,169 @@
+var assert = require('../support/assert');
+var step = require('step');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var testHelper = require(__dirname + '/../support/test_helper');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+describe('dynamic styling for named maps', function() {
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        testHelper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    var templateId = 'dynamic-styling-template-1';
+
+    var template = {
+        version: '0.0.1',
+        name: templateId,
+        auth: { method: 'open' },
+        placeholders: {
+            color: {
+                type: "css_color",
+                default: "Reds"
+            }
+        },
+        layergroup:  {
+            version: '1.0.0',
+            layers: [{
+                options: {
+                    sql: 'SELECT * FROM test_table',
+                    cartocss: [
+                        '#layer {',
+                        '  marker-fill: #000;',
+                        '}'
+                    ].join('\n'),
+                    cartocss_version: '2.0.2'
+                }
+            }, {
+                options: {
+                    sql: 'SELECT * FROM test_table',
+                    cartocss: [
+                        '#layer {',
+                        '  marker-fill: #000;',
+                        '}'
+                    ].join('\n'),
+                    cartocss_version: '2.0.2'
+                }
+            }, {
+                options: {
+                    sql: 'SELECT * FROM test_table',
+                    cartocss: [
+                        '#layer {',
+                        '  marker-fill: #000;',
+                        '}'
+                    ].join('\n'),
+                    cartocss_version: '2.0.2'
+                }
+            }]
+        }
+    };
+
+    var templateParams = {
+        styles: {
+            0: [
+                '#layer {',
+                '  marker-fill: #fabada;',
+                '}'
+            ].join('\n'),
+            2: [
+                '#layer {',
+                '  marker-fill: #cebada;',
+                '}'
+            ].join('\n')
+        }
+    };
+
+    it('should instantiate a template applying cartocss dynamicly', function (done) {
+        step(
+            function postTemplate() {
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/named?api_key=1234',
+                    method: 'POST',
+                    headers: { host: 'localhost', 'Content-Type': 'application/json' },
+                    data: JSON.stringify(template)
+                }, {},
+                function (res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTemplate(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 200);
+                assert.deepEqual(JSON.parse(res.body), {
+                    template_id: templateId
+                });
+
+                return null;
+            },
+            function instantiateTemplate(err) {
+                assert.ifError(err);
+
+                var next = this;
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(templateParams)
+                }, {},
+                function (res, err) {
+                    return next(err, res);
+                });
+            },
+            function checkInstanciation(err, res) {
+                assert.ifError(err);
+
+                assert.equal(res.statusCode, 200);
+
+                var parsedBody = JSON.parse(res.body);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                assert.equal(parsedBody.metadata.layers[0].meta.cartocss, templateParams.styles['0']);
+                assert.equal(
+                    parsedBody.metadata.layers[1].meta.cartocss,
+                    template.layergroup.layers[1].options.cartocss
+                );
+                assert.equal(parsedBody.metadata.layers[2].meta.cartocss, templateParams.styles['2']);
+
+                return parsedBody.layergroupid;
+            },
+            function deleteTemplate(err) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId + '?api_key=1234',
+                    method: 'DELETE',
+                    headers: { host: 'localhost' }
+                }, {}, function (res, err) {
+                    next(err, res);
+                });
+            },
+            function checkDeleteTemplate(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 204);
+                assert.ok(!res.body);
+
+                return null;
+            },
+            function finish(err) {
+                done(err);
+            }
+        );
+    });
+});

test/acceptance/limits.js

@@ -292,7 +292,8 @@ describe('render limits', function() {
                                 if (err) {
                                     done(err);
                                 }
-                                assert.imageEqualsFile(res.body, './test/fixtures/render-timeout-fallback.png', 25,
+                                var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
+                                assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
                                     function(imgErr/*, similarity*/) {
                                         done(imgErr);
                                     }

test/acceptance/multilayer.js

@@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
+var QueryTables = require('cartodb-query-tables');
 
 ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
 
@@ -122,7 +122,7 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png',
+              assert.imageBufferIsSimilarToFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png',
                   IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
                       next(err);
                   }
@@ -190,38 +190,50 @@ describe(suiteName, function() {
     });
 
 
-    it("should include serverMedata in the response", function(done) {
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
-      var layergroup =  {
-        version: '1.0.0',
-        layers: [
-           { options: {
-               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
-                   ' from test_table limit 2',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
-               cartocss_version: '2.0.1'
-             } }
-        ]
-      };
+    describe('server-metadata', function() {
+        var serverMetadata;
+        beforeEach(function() {
+            serverMetadata = global.environment.serverMetadata;
+            global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
+        });
+
+        afterEach(function() {
+            global.environment.serverMetadata = serverMetadata;
+        });
+
+        it("should include serverMedata in the response", function(done) {
+          var layergroup =  {
+            version: '1.0.0',
+            layers: [
+               { options: {
+                   sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator' +
+                       ' from test_table limit 2',
+                   cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+                   cartocss_version: '2.0.1'
+                 } }
+            ]
+          };
+
+          step(
+            function do_create_get()
+            {
+              var next = this;
+              assert.response(server, {
+                  url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+                  method: 'GET',
+                  headers: {host: 'localhost'}
+              }, {}, function(res, err) { next(err, res); });
+            },
+            function do_check_create(err, res) {
+              var parsed = JSON.parse(res.body);
+              keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
+              keysToDelete['user:localhost:mapviews:global'] = 5;
+              assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+              done();
+            }
+          );
+        });
 
-      step(
-        function do_create_get()
-        {
-          var next = this;
-          assert.response(server, {
-              url: layergroup_url + '?config=' + encodeURIComponent(JSON.stringify(layergroup)),
-              method: 'GET',
-              headers: {host: 'localhost'}
-          }, {}, function(res, err) { next(err, res); });
-        },
-        function do_check_create(err, res) {
-          var parsed = JSON.parse(res.body);
-          keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-          keysToDelete['user:localhost:mapviews:global'] = 5;
-          assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-          done();
-        }
-      );
     });
 
 
@@ -262,9 +274,9 @@ describe(suiteName, function() {
           var parsedBody = JSON.parse(res.body);
           expected_token = parsedBody.layergroupid.split(':')[0];
           helper.checkCache(res);
-          helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
-              'public.test_table',
-              'public.test_table_2'
+          helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
+            {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
+            {dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
           ]).key().join(' '));
 
 
@@ -392,7 +404,8 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = 'test/fixtures/test_multilayer_bbox.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err/*, similarity*/) {
                   next(err);
               });
@@ -431,7 +444,8 @@ describe(suiteName, function() {
                       ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
               }
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = 'test/fixtures/test_multilayer_bbox.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err/*, similarity*/) {
                   next(err);
               });
@@ -1007,7 +1021,7 @@ describe(suiteName, function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
+              assert.imageBufferIsSimilarToFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png',
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err/*, similarity*/) {
                       next(err);
                   }

test/acceptance/multilayer_server.js

@@ -7,6 +7,7 @@ var _ = require('underscore');
 var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
+var QueryTables = require('cartodb-query-tables');
 var CartodbWindshaft = require('../../lib/cartodb/server');
 var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
@@ -334,7 +335,7 @@ describe('tests from old api translated to multilayer', function() {
 
                 var parsed = JSON.parse(res.body);
                 assert.deepEqual(parsed, {
-                    errors: ["could not get overviews metadata: fake error message"]
+                    errors: ["fake error message"]
                 });
 
                 done();
@@ -360,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
                 keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
                 keysToDelete['user:localhost:mapviews:global'] = 5;
 
-                var runQueryFn = PgQueryRunner.prototype.run;
-                PgQueryRunner.prototype.run = function(username, query, callback) {
-                    return callback(new Error('failed to query database for affected tables'), []);
+                var affectedFn = QueryTables.getAffectedTablesFromQuery;
+                QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
+                    affectedFn({query: function(query, callback) {
+                        return callback(new Error('fake error message'), []);
+                    }}, username, query, callback);
                 };
 
                 // reset internal cacheChannel cache
@@ -387,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
                     },
                     function(res) {
                         assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
-                        PgQueryRunner.prototype.run = runQueryFn;
+                        QueryTables.getAffectedTablesFromQuery = affectedFn;
                         done();
                     }
                 );

test/acceptance/overviews_metadata_named_maps.js

@@ -0,0 +1,174 @@
+var test_helper = require('../support/test_helper');
+
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+
+var RedisPool = require('redis-mpool');
+
+var step = require('step');
+
+var windshaft = require('windshaft');
+
+
+describe('overviews metadata for named maps', function() {
+    // configure redis pool instance to use in tests
+    var redisPool = new RedisPool(global.environment.redis);
+
+    var overviews_layer = {
+        type: 'cartodb',
+        options: {
+            sql: 'SELECT * FROM test_table_overviews',
+            cartocss: '#layer { marker-fill: black; }',
+            cartocss_version: '2.3.0'
+        }
+    };
+
+    var non_overviews_layer = {
+        type: 'cartodb',
+        options: {
+            sql: 'SELECT * FROM test_table',
+            cartocss: '#layer { marker-fill: black; }',
+            cartocss_version: '2.3.0'
+        }
+    };
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        test_helper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    var templateId = 'overviews-template-1';
+
+    var template = {
+        version: '0.0.1',
+        name: templateId,
+        auth: { method: 'open' },
+        layergroup:  {
+            version: '1.0.0',
+            layers: [overviews_layer, non_overviews_layer]
+        }
+    };
+
+    it("should add overviews data to layers", function(done) {
+      step(
+        function postTemplate()
+        {
+          var next = this;
+
+          assert.response(server, {
+              url: '/api/v1/map/named?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template)
+          }, {}, function(res, err) {
+                     next(err, res);
+          });
+        },
+        function checkTemplate(err, res) {
+            assert.ifError(err);
+
+            var next = this;
+            assert.equal(res.statusCode, 200);
+            assert.deepEqual(JSON.parse(res.body), {
+                template_id: templateId
+            });
+            next(null);
+        },
+        function instantiateTemplate(err) {
+            assert.ifError(err);
+
+            var next = this;
+            assert.response(server, {
+                url: '/api/v1/map/named/' + templateId,
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                }
+            }, {},
+            function(res, err) {
+                return next(err, res);
+            });
+
+        },
+        function checkInstanciation(err, res) {
+            assert.ifError(err);
+
+            var next = this;
+
+            assert.equal(res.statusCode, 200);
+
+            var parsedBody = JSON.parse(res.body);
+
+            keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+            keysToDelete['user:localhost:mapviews:global'] = 5;
+
+            assert.ok(parsedBody.layergroupid);
+            assert.ok(parsedBody.last_updated);
+
+            next(null, parsedBody.layergroupid);
+        },
+
+        function checkMapconfig(err, layergroupId)
+        {
+            assert.ifError(err);
+
+            var next = this;
+
+            var mapStore  = new windshaft.storage.MapStore({
+                pool: redisPool,
+                expire_time: 500000
+            });
+            mapStore.load(LayergroupToken.parse(layergroupId).token, function(err, mapConfig) {
+              assert.ifError(err);
+              assert.deepEqual(non_overviews_layer, mapConfig._cfg.layers[1]);
+              assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
+              assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
+              var expected_data = {
+                overviews: {
+                  test_table_overviews: {
+                    1: { table: '_vovw_1_test_table_overviews' },
+                    2: { table: '_vovw_2_test_table_overviews' }
+                  }
+                }
+              };
+              assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
+            });
+
+            next(err);
+        },
+        function deleteTemplate(err) {
+            assert.ifError(err);
+
+            var next = this;
+
+            assert.response(server, {
+                url: '/api/v1/map/named/' + templateId + '?api_key=1234',
+                method: 'DELETE',
+                headers: { host: 'localhost' }
+            }, {}, function (res, err) {
+                next(err, res);
+            });
+        },
+        function checkDeleteTemplate(err, res) {
+            assert.ifError(err);
+            assert.equal(res.statusCode, 204);
+            assert.ok(!res.body);
+
+            return null;
+        },
+        function finish(err) {
+            done(err);
+        }
+      );
+    });
+});

test/acceptance/overviews_queries.js

@@ -30,7 +30,8 @@ describe('overviews_queries', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(tile.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            var referenceImagePath = './test/fixtures/' + fixture;
+            assert.imageBufferIsSimilarToFile(tile.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
         };
     }
 

test/acceptance/ported/blend.js

@@ -95,10 +95,12 @@ describe('blend png renderer', function() {
         var zxy = [tileRequest.z, tileRequest.x, tileRequest.y];
         it('tile all/' + zxy.join('/') + '.png', function (done) {
             testClient.getTileLayer(plainTorqueMapConfig(testScenario.plainColor), tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(zxy), IMAGE_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(zxy), IMAGE_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/blend_filtering.js

@@ -156,10 +156,12 @@ describe('blend layer filtering', function() {
 
         it('should filter on ' + layerFilter + '/1/0/0.png', function (done) {
             testClient.getTileLayer(mapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/blend_http_fallback.js

@@ -111,10 +111,12 @@ describe('blend http fallback', function() {
 
         it('should fallback on http error while blending layers ' + layerFilter + '/1/0/0.png', function (done) {
             testClient.getTileLayer(mapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err, err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(filteredLayers), IMG_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err, err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/external_resources.js

@@ -57,7 +57,8 @@ describe('external resources', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            var referenceImagePath = './test/fixtures/' + fixture;
+            assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
         };
     }
 

test/acceptance/ported/limits.js

@@ -84,11 +84,13 @@ describe.skip('render limits', function() {
         testClient.withLayergroup(slowQueryMapConfig, options, function(err, requestTile, finish) {
             var tileUrl = '/0/0/0.png';
             requestTile(tileUrl, options, function(err, res) {
-                assert.imageEqualsFile(res.body, fixtureImage, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
-                    finish(function(finishErr) {
-                        done(err || finishErr);
-                    });
-                });
+                assert.imageBufferIsSimilarToFile(res.body, fixtureImage, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                    function(err) {
+                        finish(function(finishErr) {
+                            done(err || finishErr);
+                        });
+                    }
+                );
             });
         });
     });

test/acceptance/ported/multilayer.js

@@ -117,7 +117,8 @@ describe('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
               checkCORSHeaders(res);
-              assert.imageEqualsFile(res.body, './test/fixtures/test_bigpoint_red.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              var referenceImagePath = './test/fixtures/test_bigpoint_red.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                   function(err) {
                   next(err);
               });
@@ -191,7 +192,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -302,7 +304,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -425,7 +428,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -541,7 +545,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer1.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -727,7 +732,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer2.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer2.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -761,7 +767,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer3.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer3.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -856,7 +863,8 @@ describe('multilayer', function() {
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer4.png',
+              var referenceImagePath = './test/acceptance/ported/fixtures/test_table_0_0_0_multilayer4.png';
+              assert.imageBufferIsSimilarToFile(res.body, referenceImagePath,
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });
@@ -1259,7 +1267,7 @@ describe('multilayer', function() {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
               checkCORSHeaders(res);
-              assert.imageEqualsFile(res.body, './test/fixtures/test_bigpoint_red.png',
+              assert.imageBufferIsSimilarToFile(res.body, './test/fixtures/test_bigpoint_red.png',
                   IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
                   next(err);
               });

test/acceptance/ported/raster.js

@@ -87,7 +87,7 @@ describe('raster', function() {
           assert.equal(res.statusCode, 200, res.body);
           assert.deepEqual(res.headers['content-type'], "image/png");
           var next = this;
-          assert.imageEqualsFile(res.body,
+          assert.imageBufferIsSimilarToFile(res.body,
             './test/fixtures/raster_gray_rect.png',
             IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
               try {

test/acceptance/ported/server_gettile.js

@@ -34,7 +34,9 @@ describe('server_gettile', function() {
             if (err) {
                 return done(err);
             }
-            assert.imageEqualsFile(res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done);
+            assert.imageBufferIsSimilarToFile(
+                res.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done
+            );
         };
     }
 
@@ -113,12 +115,13 @@ describe('server_gettile', function() {
                 assert.ok(res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time");
                 assert.ok(res.headers['x-windshaft-cache'] >= 0);
 
-                assert.imageEqualsFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
-
-                    finish(function(finishErr) {
-                        done(err || finishErr);
-                    });
-                });
+                assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                    function(err) {
+                        finish(function(finishErr) {
+                            done(err || finishErr);
+                        });
+                    }
+                );
             });
         });
     });

test/acceptance/ported/server_png8_format.js

@@ -108,7 +108,7 @@ describe('server_png8_format', function() {
                             assert.equal(responsePng8.headers['content-type'], "image/png");
                             bufferPng8 = responsePng8.body;
                             assert.ok(bufferPng8.length < bufferPng32.length);
-                            assert.imageBuffersAreEqual(bufferPng32, bufferPng8, IMAGE_EQUALS_TOLERANCE_PER_MIL,
+                            assert.imageBuffersAreSimilar(bufferPng32, bufferPng8, IMAGE_EQUALS_TOLERANCE_PER_MIL,
                                 function(err, imagePaths, similarity) {
 
                                     keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;

test/acceptance/ported/torque_png.js

@@ -83,10 +83,12 @@ describe('torque png renderer', function() {
         var zxy = [z, x, y];
         it('tile ' + zxy.join('/') + '.torque.png', function (done) {
             testClient.getTileLayer(torquePngPointsMapConfig, tileRequest, function(err, res) {
-                assert.imageEqualsFile(res.body, torquePngFixture(zxy), IMAGE_TOLERANCE_PER_MIL, function(err) {
-                    assert.ok(!err);
-                    done();
-                });
+                assert.imageBufferIsSimilarToFile(res.body, torquePngFixture(zxy), IMAGE_TOLERANCE_PER_MIL,
+                    function(err) {
+                        assert.ok(!err);
+                        done();
+                    }
+                );
             });
         });
     });

test/acceptance/ported/wrap.js

@@ -107,10 +107,12 @@ describe('wrap x coordinate', function() {
             var fixtureZxy = [testScenario.fixture.z, testScenario.fixture.x, testScenario.fixture.y];
             it('tile all/' + zxy.join('/') + '.png', function (done) {
                 testClient.getTileLayer(plainTorqueMapConfig(testScenario.plainColor), tileRequest, function(err, res) {
-                    assert.imageEqualsFile(res.body, blendPngFixture(fixtureZxy), IMG_TOLERANCE_PER_MIL, function(err) {
-                        assert.ok(!err);
-                        done();
-                    });
+                    assert.imageBufferIsSimilarToFile(res.body, blendPngFixture(fixtureZxy), IMG_TOLERANCE_PER_MIL,
+                        function(err) {
+                            assert.ok(!err);
+                            done();
+                        }
+                    );
                 });
             });
         });

test/acceptance/templates.js

@@ -3,8 +3,8 @@ var _           = require('underscore');
 var redis       = require('redis');
 var step        = require('step');
 var strftime    = require('strftime');
+var QueryTables = require('cartodb-query-tables');
 var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
-var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
 var redis_stats_db = 5;
 
 // Pollute the PG environment to make sure
@@ -313,51 +313,63 @@ describe('template_api', function() {
       });
     });
 
-    it("instance endpoint should return server metadata", function(done){
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
-      var tmpl = _.clone(template_acceptance1);
-      tmpl.name = "rambotemplate2";
+    describe('server-metadata', function() {
+        var serverMetadata;
+        beforeEach(function() {
+            serverMetadata = global.environment.serverMetadata;
+            global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } };
+        });
 
-      step(function postTemplate1() {
-          var next = this;
-          var post_request = {
-              url: '/api/v1/map/named?api_key=1234',
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' },
-              data: JSON.stringify(tmpl)
-          };
-          assert.response(server, post_request, {}, function(res) {
-            next(null, res);
-          });
-        },
-        function testCORS() {
-          var next = this;
-          assert.response(server, {
-              url: '/api/v1/map/named/' + tmpl.name,
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          },{
-              status: 200
-          }, function(res) {
-            var parsed = JSON.parse(res.body);
-            keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-            keysToDelete['user:localhost:mapviews:global'] = 5;
-            assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-            next(null);
-          });
-        },
-        function deleteTemplate(err) {
-            assert.ifError(err);
-          var del_request = {
-              url: '/api/v1/map/named/' + tmpl.name + '?api_key=1234',
-              method: 'DELETE',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          };
-          assert.response(server, del_request, {}, function() {
-              done();
-          });
-        }
-      );
+        afterEach(function() {
+            global.environment.serverMetadata = serverMetadata;
+        });
+
+
+        it("instance endpoint should return server metadata", function(done){
+          var tmpl = _.clone(template_acceptance1);
+          tmpl.name = "rambotemplate2";
+
+          step(function postTemplate1() {
+              var next = this;
+              var post_request = {
+                  url: '/api/v1/map/named?api_key=1234',
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' },
+                  data: JSON.stringify(tmpl)
+              };
+              assert.response(server, post_request, {}, function(res) {
+                next(null, res);
+              });
+            },
+            function testCORS() {
+              var next = this;
+              assert.response(server, {
+                  url: '/api/v1/map/named/' + tmpl.name,
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' }
+              },{
+                  status: 200
+              }, function(res) {
+                var parsed = JSON.parse(res.body);
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+                next(null);
+              });
+            },
+            function deleteTemplate(err) {
+                assert.ifError(err);
+              var del_request = {
+                  url: '/api/v1/map/named/' + tmpl.name + '?api_key=1234',
+                  method: 'DELETE',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' }
+              };
+              assert.response(server, del_request, {}, function() {
+                  done();
+              });
+            }
+          );
+        });
     });
 
 
@@ -1393,7 +1405,8 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-              new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+                                   table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
           helper.checkSurrogateKey(res, expectedSurrogateKey);
@@ -1476,7 +1489,8 @@ describe('template_api', function() {
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
           helper.checkCache(res);
           var expectedSurrogateKey = [
-              new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
+            new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
+                                   table_name: 'test_table_private_1'}]).key(),
               new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
           ].join(' ');
           helper.checkSurrogateKey(res, expectedSurrogateKey);

test/acceptance/turbo-cartocss-anonymous-maps.js

@@ -0,0 +1,128 @@
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+var IMAGE_TOLERANCE_PER_MIL = 20;
+
+function imageCompareFn(fixture, done) {
+    return function(err, res, image) {
+        assert.ok(!err, err);
+        assert.imageIsSimilarToFile(image, './test/fixtures/' + fixture, IMAGE_TOLERANCE_PER_MIL, done);
+    };
+}
+
+function makeMapconfig(cartocss) {
+    return {
+        "version": "1.4.0",
+        "layers": [
+            {
+                "type": 'mapnik',
+                "options": {
+                    "cartocss_version": '2.3.0',
+                    "sql": [
+                        'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
+                        '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                        '  UNION',
+                        '  SELECT 2, 10.50',
+                        '  UNION',
+                        '  SELECT 3, 11.00',
+                        '  UNION',
+                        '  SELECT 4, 12.00',
+                        '  UNION',
+                        '  SELECT 5, 21.00',
+                        ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
+                    ].join('\n'),
+                    "cartocss": cartocss
+                }
+            }
+        ]
+    };
+}
+
+describe('turbo-cartocss for anonymous maps', function() {
+    describe('parsing ramp function with colorbrewer for greens and mapnik renderer', function () {
+        beforeEach(function () {
+            var turboCartocss = '#layer { marker-fill: ramp([price], colorbrewer(Greens)); }';
+            this.testClient = new TestClient(makeMapconfig(turboCartocss));
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+
+        it('should get a tile with turbo-cartocss parsed properly', function (done) {
+            var fixturePath = 'test_turbo_cartocss_greens_13_4011_3088.png';
+            this.testClient.getTile(13, 4011, 3088, imageCompareFn(fixturePath, done));
+        });
+    });
+
+    describe('parsing ramp function with colorbrewer for reds and mapnik renderer', function () {
+        beforeEach(function () {
+            var turboCartocss = '#layer { marker-fill: ramp([price], colorbrewer(Reds)); }';
+            this.testClient = new TestClient(makeMapconfig(turboCartocss));
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+
+        it('should get a tile with turbo-cartocss parsed properly', function (done) {
+            var fixtureFileName = 'test_turbo_cartocss_reds_13_4011_3088.png';
+            this.testClient.getTile(13, 4011, 3088, imageCompareFn(fixtureFileName, done));
+        });
+    });
+
+    describe('parsing ramp function with colorbrewer for greens and toque renderer', function () {
+        var mapConfig = {
+            version: '1.2.0',
+            layers: [
+                {
+                    type: 'torque',
+                    options: {
+                        sql: "SELECT * FROM populated_places_simple_reduced where the_geom" +
+                            " && ST_MakeEnvelope(-90, 0, 90, 65)",
+                        cartocss: [
+                            'Map {',
+                            '    buffer-size:0;',
+                            '    -torque-frame-count:1;',
+                            '    -torque-animation-duration:30;',
+                            '    -torque-time-attribute:"cartodb_id";',
+                            '    -torque-aggregation-function:"count(cartodb_id)";',
+                            '    -torque-resolution:1;',
+                            '    -torque-data-aggregation:linear;',
+                            '};',
+                            '#populated_places_simple_reduced {',
+                            '    comp-op: multiply;',
+                            '    marker-fill-opacity: 1;',
+                            '    marker-line-color: #FFF;',
+                            '    marker-line-width: 0;',
+                            '    marker-line-opacity: 1;',
+                            '    marker-type: rectangle;',
+                            '    marker-width: 3;',
+                            '    marker-fill: ramp([pop_max], colorbrewer(Greens));',
+                            '};'
+                        ].join(' '),
+                        cartocss_version: '2.3.0'
+                    }
+                }
+            ]
+        };
+
+        beforeEach(function () {
+            this.testClient = new TestClient(mapConfig);
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+
+        it('should get a tile with turbo-cartocss parsed properly', function (done) {
+            var z = 2;
+            var x = 2;
+            var y = 1;
+
+            var pngFixture = 'torque/populated_places_simple_reduced-turbo-cartocss-' + [z, x, y].join('.') + '.png';
+
+            this.testClient.getTile(z, x, y, { layers: 0, format: 'torque.png' }, imageCompareFn(pngFixture, done));
+        });
+    });
+});

test/acceptance/turbo-cartocss-named-maps.js

@@ -0,0 +1,236 @@
+var assert = require('../support/assert');
+var step = require('step');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
+var testHelper = require(__dirname + '/../support/test_helper');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+var mapnik = require('windshaft').mapnik;
+var IMAGE_TOLERANCE_PER_MIL = 10;
+
+describe('turbo-cartocss for named maps', function() {
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        testHelper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    var templateId = 'turbo-cartocss-template-1';
+
+    var template = {
+        version: '0.0.1',
+        name: templateId,
+        auth: { method: 'open' },
+        placeholders: {
+            color: {
+                type: "css_color",
+                default: "Reds"
+            }
+        },
+        layergroup:  {
+            version: '1.0.0',
+                layers: [{
+                    options: {
+                        sql: [
+                            'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
+                            '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                            '  UNION',
+                            '  SELECT 2, 10.50',
+                            '  UNION',
+                            '  SELECT 3, 11.00',
+                            '  UNION',
+                            '  SELECT 4, 12.00',
+                            '  UNION',
+                            '  SELECT 5, 21.00',
+                            ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
+                        ].join('\n'),
+                        cartocss: [
+                            '#layer {',
+                            '  marker-fill: ramp([price], colorbrewer(<%= color %>));',
+                            '  marker-allow-overlap:true;',
+                            '}'
+                        ].join('\n'),
+                        cartocss_version: '2.0.2'
+                    }
+                }
+            ]
+        }
+    };
+
+    var templateParamsReds = { color: 'Reds' };
+    var templateParamsBlues = { color: 'Blues' };
+
+    it('should create a template with turbo-cartocss parsed properly', function (done) {
+        step(
+            function postTemplate() {
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/named?api_key=1234',
+                    method: 'POST',
+                    headers: { host: 'localhost', 'Content-Type': 'application/json' },
+                    data: JSON.stringify(template)
+                }, {},
+                function (res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTemplate(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 200);
+                assert.deepEqual(JSON.parse(res.body), {
+                    template_id: templateId
+                });
+
+                return null;
+            },
+            function instantiateTemplateWithReds(err) {
+                assert.ifError(err);
+
+                var next = this;
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(templateParamsReds)
+                }, {},
+                function(res, err) {
+                    return next(err, res);
+                });
+            },
+            function checkInstanciationWithReds(err, res) {
+                assert.ifError(err);
+
+                assert.equal(res.statusCode, 200);
+
+                var parsedBody = JSON.parse(res.body);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                assert.ok(parsedBody.layergroupid);
+                assert.ok(parsedBody.last_updated);
+
+                return parsedBody.layergroupid;
+            },
+            function requestTileReds(err, layergroupId) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
+                    method: 'GET',
+                    headers: { host: 'localhost' },
+                    encoding: 'binary'
+                }, {},
+                function(res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTileReds(err, res) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.equal(res.statusCode, 200);
+                assert.equal(res.headers['content-type'], 'image/png');
+
+                var fixturePath = './test/fixtures/turbo-cartocss-named-maps-reds.png';
+                var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+
+                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
+            },
+            function instantiateTemplateWithBlues(err) {
+                assert.ifError(err);
+
+                var next = this;
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(templateParamsBlues)
+                }, {},
+                function(res, err) {
+                    return next(err, res);
+                });
+            },
+            function checkInstanciationWithBlues(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 200);
+
+                var parsedBody = JSON.parse(res.body);
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                assert.ok(parsedBody.layergroupid);
+                assert.ok(parsedBody.last_updated);
+
+                return parsedBody.layergroupid;
+            },
+            function requestTileBlues(err, layergroupId) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
+                    method: 'GET',
+                    headers: { host: 'localhost' },
+                    encoding: 'binary'
+                }, {},
+                function(res, err) {
+                    next(err, res);
+                });
+            },
+            function checkTileBlues(err, res) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.equal(res.statusCode, 200);
+                assert.equal(res.headers['content-type'], 'image/png');
+
+                var fixturePath = './test/fixtures/turbo-cartocss-named-maps-blues.png';
+                var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+
+                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
+            },
+            function deleteTemplate(err) {
+                assert.ifError(err);
+
+                var next = this;
+
+                assert.response(server, {
+                    url: '/api/v1/map/named/' + templateId + '?api_key=1234',
+                    method: 'DELETE',
+                    headers: { host: 'localhost' }
+                }, {}, function (res, err) {
+                    next(err, res);
+                });
+            },
+            function checkDeleteTemplate(err, res) {
+                assert.ifError(err);
+                assert.equal(res.statusCode, 204);
+                assert.ok(!res.body);
+
+                return null;
+            },
+            function finish(err) {
+                done(err);
+            }
+        );
+    });
+});

test/acceptance/turbo-cartocss-regressions.js

@@ -0,0 +1,75 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+function makeMapconfig(cartocss) {
+    return {
+        "version": "1.4.0",
+        "layers": [
+            {
+                "type": 'mapnik',
+                "options": {
+                    "cartocss_version": '2.3.0',
+                    "sql": [
+                        'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
+                        '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                        '  UNION',
+                        '  SELECT 2, 10.50',
+                        '  UNION',
+                        '  SELECT 3, 11.00',
+                        '  UNION',
+                        '  SELECT 4, 12.00',
+                        '  UNION',
+                        '  SELECT 5, 21.00',
+                        ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
+                    ].join('\n'),
+                    "cartocss": cartocss
+                }
+            }
+        ]
+    };
+}
+
+describe('turbo-cartocss regressions', function() {
+
+    var cartocss = [
+        "/** simple visualization */",
+        "",
+        "Map {",
+        "  buffer-size: 256;",
+        "}",
+        "",
+        "#county_points_with_population{",
+        "  marker-fill-opacity: 0.1;",
+        "  marker-line-color:#FFFFFF;//#CF1C90;",
+        "  marker-line-width: 0;",
+        "  marker-line-opacity: 0.3;",
+        "  marker-placement: point;",
+        "  marker-type: ellipse;",
+        "  //marker-comp-op: overlay;",
+        "  marker-width: [price];",
+        "  [zoom=5]{marker-width: [price]*2;}",
+        "  [zoom=6]{marker-width: [price]*4;}",
+        "  marker-fill: #000000;",
+        "  marker-allow-overlap: true;",
+        "  ",
+        "",
+        "}"
+    ].join('\n');
+
+    beforeEach(function () {
+        this.testClient = new TestClient(makeMapconfig(cartocss));
+    });
+
+    afterEach(function (done) {
+        this.testClient.drain(done);
+    });
+
+    it('should accept // comments', function(done) {
+        this.testClient.getTile(0, 0, 0, function(err) {
+            assert.ok(!err, err);
+            done();
+        });
+    });
+});

test/acceptance/widgets/named-maps.js

@@ -0,0 +1,250 @@
+var assert = require('../../support/assert');
+var step = require('step');
+
+var url = require('url');
+var queue = require('queue-async');
+
+var helper = require('../../support/test_helper');
+
+var CartodbWindshaft = require('../../../lib/cartodb/server');
+var serverOptions = require('../../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
+
+describe('named-maps widgets', function() {
+
+    var username = 'localhost';
+    var widgetsTemplateName = 'widgets-template';
+
+    var layergroupid;
+    var layergroup;
+    var keysToDelete;
+
+    beforeEach(function(done) {
+        keysToDelete = {};
+
+        var widgetsTemplate =  {
+            version: '0.0.1',
+            name: widgetsTemplateName,
+            layergroup:  {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'cartodb',
+                        options: {
+                            sql: "select * from populated_places_simple_reduced_private",
+                            cartocss: '#layer { marker-fill: blue; }',
+                            cartocss_version: '2.3.0',
+                            widgets: {
+                                pop_max_formula_sum: {
+                                    type: 'formula',
+                                    options: {
+                                        column: 'pop_max',
+                                        operation: 'sum'
+                                    }
+                                },
+                                country_places_count: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm0_a3',
+                                        aggregation: 'count'
+                                    }
+                                },
+                                pop_max: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        };
+
+        var template_params = {};
+
+        step(
+            function createTemplate()
+            {
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named?api_key=1234',
+                        method: 'POST',
+                        headers: {
+                            host: username,
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(widgetsTemplate)
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res, err) {
+                        next(err, res);
+                    }
+                );
+            },
+            function instantiateTemplate(err, res) {
+                assert.ifError(err);
+
+                assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName });
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + widgetsTemplateName,
+                        method: 'POST',
+                        headers: {
+                            host: username,
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(template_params)
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res) {
+                        next(null, res);
+                    }
+                );
+            },
+            function fetchTile(err, res) {
+                assert.ifError(err);
+
+                layergroup = JSON.parse(res.body);
+                assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body);
+                layergroupid = layergroup.layergroupid;
+
+                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                return done();
+            }
+        );
+
+    });
+
+    afterEach(function(done) {
+        step(
+            function deleteTemplate(err) {
+                assert.ifError(err);
+                var next = this;
+                assert.response(
+                    server,
+                    {
+                        url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234',
+                        method: 'DELETE',
+                        headers: {
+                            host: username
+                        }
+                    },
+                    {
+                        status: 204
+                    },
+                    function(res, err) {
+                        next(err, res);
+                    }
+                );
+            },
+            function deleteRedisKeys(err) {
+                assert.ifError(err);
+                helper.deleteRedisKeys(keysToDelete, done);
+            }
+        );
+    });
+
+    function getWidget(widgetName, callback) {
+        assert.response(
+            server,
+            {
+                url: '/api/v1/map/' + layergroupid + '/0/widget/' + widgetName,
+                method: 'GET',
+                headers: {
+                    host: username
+                }
+            },
+            {
+                status: 200
+            },
+            function(res, err) {
+                if (err) {
+                    return callback(err);
+                }
+                var parsedBody = JSON.parse(res.body);
+                return callback(err, res, parsedBody);
+            }
+        );
+    }
+
+    it('should be able to retrieve widgets from all URLs', function(done) {
+        var widgetsPaths = layergroup.metadata.layers.reduce(function(paths, layer) {
+            var widgets = layer.widgets || {};
+            Object.keys(widgets).forEach(function(widget) {
+                paths.push(url.parse(widgets[widget].url.http).path);
+            });
+
+            return paths;
+        }, []);
+
+        var widgetsQueue = queue(widgetsPaths.length);
+
+        widgetsPaths.forEach(function(path) {
+            widgetsQueue.defer(function(path, done) {
+                assert.response(
+                    server,
+                    {
+                        url: path,
+                        method: 'GET',
+                        headers: {
+                            host: username
+                        }
+                    },
+                    {
+                        status: 200
+                    },
+                    function(res, err) {
+                        if (err) {
+                            return done(err);
+                        }
+                        var parsedBody = JSON.parse(res.body);
+                        return done(null, parsedBody);
+                    }
+                );
+            }, path);
+        });
+
+        widgetsQueue.awaitAll(function(err, results) {
+            assert.equal(results.length, 3);
+            done(err);
+        });
+    });
+
+
+    it("should retrieve aggregation", function(done) {
+        getWidget('country_places_count', function(err, response, aggregation) {
+            assert.ok(!err, err);
+
+            assert.equal(aggregation.type, 'aggregation');
+            assert.equal(aggregation.max, 769);
+
+            return done();
+        });
+    });
+
+    it("should retrieve histogram", function(done) {
+        getWidget('pop_max', function(err, response, histogram) {
+            assert.ok(!err, err);
+
+            assert.equal(histogram.type, 'histogram');
+            assert.equal(histogram.bin_width, 743250);
+
+            return done();
+        });
+    });
+
+});

test/integration/overviews-metadata-api.js

@@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis');
 
 var PgConnection = require('../../lib/cartodb/backends/pg_connection');
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
-var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
 var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
 
 
 describe('OverviewsMetadataApi', function() {
 
-    var queryTablesApi, overviewsMetadataApi;
+    var overviewsMetadataApi;
 
     before(function() {
         var redisPool = new RedisPool(global.environment.redis);
         var metadataBackend = cartodbRedis({pool: redisPool});
         var pgConnection = new PgConnection(metadataBackend);
         var pgQueryRunner = new PgQueryRunner(pgConnection);
-        queryTablesApi = new QueryTablesApi(pgQueryRunner);
         overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
     });
 

test/integration/query-tables-api.js

@@ -1,55 +0,0 @@
-require('../support/test_helper');
-
-var assert = require('assert');
-
-var RedisPool = require('redis-mpool');
-var cartodbRedis = require('cartodb-redis');
-
-var PgConnection = require('../../lib/cartodb/backends/pg_connection');
-var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
-var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
-
-
-describe('QueryTablesApi', function() {
-
-    var queryTablesApi;
-
-    before(function() {
-        var redisPool = new RedisPool(global.environment.redis);
-        var metadataBackend = cartodbRedis({pool: redisPool});
-        var pgConnection = new PgConnection(metadataBackend);
-        var pgQueryRunner = new PgQueryRunner(pgConnection);
-        queryTablesApi = new QueryTablesApi(pgQueryRunner);
-    });
-
-    // Check test/support/sql/windshaft.test.sql to understand where the values come from.
-
-    it('should return an object with affected tables array and last updated time', function(done) {
-        var query = 'select * from test_table';
-        queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
-            assert.ok(!err, err);
-
-            assert.deepEqual(result, {
-                affectedTables: [ 'public.test_table' ],
-                lastUpdatedTime: 1234567890123
-            });
-
-            done();
-        });
-    });
-
-    it('should work with private tables', function(done) {
-        var query = 'select * from test_table_private_1';
-        queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
-            assert.ok(!err, err);
-
-            assert.deepEqual(result, {
-                affectedTables: [ 'public.test_table_private_1' ],
-                lastUpdatedTime: 1234567890123
-            });
-
-            done();
-        });
-    });
-
-});

test/integration/query-tables.js

@@ -0,0 +1,71 @@
+require('../support/test_helper');
+
+var assert = require('assert');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var PgConnection = require('../../lib/cartodb/backends/pg_connection');
+
+var QueryTables = require('cartodb-query-tables');
+
+
+describe('QueryTables', function() {
+
+    var connection;
+
+    before(function(done) {
+        var redisPool = new RedisPool(global.environment.redis);
+        var metadataBackend = cartodbRedis({pool: redisPool});
+        var pgConnection = new PgConnection(metadataBackend);
+        pgConnection.getConnection('localhost', function(err, pgConnection) {
+            if (err) {
+                return done(err);
+            }
+            connection = pgConnection;
+
+            return done();
+        });
+    });
+
+    // Check test/support/sql/windshaft.test.sql to understand where the values come from.
+
+    it('should return an object with affected tables array and last updated time', function(done) {
+        var query = 'select * from test_table';
+        QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.equal(result.getLastUpdatedAt(), 1234567890123);
+
+            assert.equal(result.tables.length, 1);
+            assert.deepEqual(result.tables[0], {
+                dbname: 'test_windshaft_cartodb_user_1_db',
+                schema_name: 'public',
+                table_name: 'test_table',
+                updated_at: new Date(1234567890123)
+            });
+
+            done();
+        });
+    });
+
+    it('should work with private tables', function(done) {
+        var query = 'select * from test_table_private_1';
+        QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.equal(result.getLastUpdatedAt(), 1234567890123);
+
+            assert.equal(result.tables.length, 1);
+            assert.deepEqual(result.tables[0], {
+                dbname: 'test_windshaft_cartodb_user_1_db',
+                schema_name: 'public',
+                table_name: 'test_table_private_1',
+                updated_at: new Date(1234567890123)
+            });
+
+            done();
+        });
+    });
+
+});

test/support/assert.js

@@ -1,7 +1,6 @@
 // Cribbed from the ever prolific Konstantin Kaefer
 // https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js
 
-var exec = require('child_process').exec;
 var fs = require('fs');
 var path = require('path');
 var util = require('util');
@@ -13,7 +12,7 @@ var request = require('request');
 var assert = module.exports = exports = require('assert');
 
 /**
- * Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
+ * Takes an image data as an input and an image path and compare them using mapnik.Image.compare mechanism, in case the
  * similarity is not within the tolerance limit it will callback with an error.
  *
  * @param buffer The image data to compare from
@@ -22,70 +21,39 @@ var assert = module.exports = exports = require('assert');
  * @param {function} callback Will call to home with null in case there is no error, otherwise with the error itself
  * @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
  */
-assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
+assert.imageBufferIsSimilarToFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
     callback = callback || function(err) { assert.ifError(err); };
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
-        testImageFilePath = createImageFromBuffer(buffer, 'test');
 
-    imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, function(err) {
-        fs.unlinkSync(testImageFilePath);
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
+    var referenceImageBuffer = fs.readFileSync(referenceImageFilePath, { encoding: null });
+
+    assert.imageBuffersAreSimilar(buffer, referenceImageBuffer, tolerance, callback);
+};
+
+assert.imageBuffersAreSimilar = function(bufferA, bufferB, tolerance, callback) {
+    var testImage = mapnik.Image.fromBytes(Buffer.isBuffer(bufferA) ? bufferA : new Buffer(bufferA, 'binary'));
+    var referenceImage = mapnik.Image.fromBytes(Buffer.isBuffer(bufferB) ? bufferB : new Buffer(bufferB, 'binary'));
+
+    imagesAreSimilar(testImage, referenceImage, tolerance, callback);
+};
+
+assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
+    callback = callback || function(err) { assert.ifError(err); };
+
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
+
+    var referenceImage = mapnik.Image.fromBytes(fs.readFileSync(referenceImageFilePath,  { encoding: null }));
+
+    imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
+        if (err) {
+            var testImageFilePath = randomImagePath();
+            testImage.save(testImageFilePath);
+        }
         callback(err);
     });
 };
 
-assert.imageBuffersAreEqual = function(bufferA, bufferB, tolerance, callback) {
-    var randStr = (Math.random() * 1e16).toString().substring(0, 8);
-    var imageFilePathA = createImageFromBuffer(bufferA, randStr + '-a'),
-        imageFilePathB = createImageFromBuffer(bufferB, randStr + '-b');
-
-    imageFilesAreEqual(imageFilePathA, imageFilePathB, tolerance, function(err, similarity) {
-        callback(err, [imageFilePathA, imageFilePathB], similarity);
-    });
-};
-
-function createImageFromBuffer(buffer, nameHint) {
-    var imageFilePath = path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
-    var err = fs.writeFileSync(imageFilePath, buffer, 'binary');
-    assert.ifError(err);
-    return imageFilePath;
-}
-
-function imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, callback) {
-    var resultFilePath = path.resolve(util.format('/tmp/windshaft-result-%s-diff.png', Date.now()));
-    var imageMagickCmd = util.format(
-        'compare -metric fuzz "%s" "%s" "%s"',
-        testImageFilePath, referenceImageFilePath, resultFilePath
-    );
-
-    exec(imageMagickCmd, function(err, stdout, stderr) {
-        if (err) {
-            fs.unlinkSync(testImageFilePath);
-            callback(err);
-        } else {
-            stderr = stderr.trim();
-            var metrics = stderr.match(/([0-9]*) \((.*)\)/);
-            if ( ! metrics ) {
-                callback(new Error("No match for " + stderr));
-                return;
-            }
-            var similarity = parseFloat(metrics[2]),
-                tolerancePerMil = (tolerance / 1000);
-            if (similarity > tolerancePerMil) {
-                err = new Error(util.format(
-                        'Images %s and %s are not equal (got %d similarity, expected %d). Result %s',
-                        testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil, resultFilePath)
-                );
-                err.similarity = similarity;
-                callback(err, similarity);
-            } else {
-                fs.unlinkSync(resultFilePath);
-                callback(null, similarity);
-            }
-        }
-    });
-}
-
-assert.imagesAreSimilar = function(testImage, referenceImage, tolerance, callback) {
+function imagesAreSimilar(testImage, referenceImage, tolerance, callback) {
     if (testImage.width() !== referenceImage.width() || testImage.height() !== referenceImage.height()) {
         return callback(new Error('Images are not the same size'));
     }
@@ -103,27 +71,10 @@ assert.imagesAreSimilar = function(testImage, referenceImage, tolerance, callbac
     } else {
         callback(null, similarity);
     }
-};
+}
 
-assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
-    callback = callback || function(err) { assert.ifError(err); };
-
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
-
-    var referenceImage = mapnik.Image.fromBytes(fs.readFileSync(referenceImageFilePath,  { encoding: null }));
-
-    assert.imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
-        if (err) {
-            var testImageFilePath = randomImagePath();
-            testImage.save(testImageFilePath);
-        }
-        callback(err);
-    });
-};
-
-function randomImagePath(nameHint) {
-    nameHint = nameHint || 'test';
-    return path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
+function randomImagePath() {
+    return path.resolve('test/results/png/image-test-' + Date.now() + '.png');
 }
 
 // jshint maxcomplexity:9

test/support/prepare_db.sh

@@ -78,11 +78,15 @@ if test x"$PREPARE_PGSQL" = xyes; then
     sed "s/:TESTPASS/${TESTPASS}/" |
     psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
-  psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
-  curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql
-  curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
-  cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql sql/CDB_Overviews.sql |
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+
+  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins'
+  for i in ${SQL_SCRIPTS}
+  do
+    curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
+    cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
+        | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  done
 
 fi
 

test/support/sql/.gitignore

@@ -0,0 +1 @@
+CDB_*.sql

test/support/sql/CDB_Overviews.sql

@@ -1,46 +0,0 @@
--- Mockup for CDB_Overviews
-CREATE OR REPLACE FUNCTION CDB_Overviews(table_names regclass[])
-RETURNS TABLE(base_table regclass, z integer, overview_table regclass)
-AS $$
-  BEGIN
-    IF (SELECT 'test_table_overviews'::regclass = ANY (table_names)) THEN
-      RETURN QUERY
-        SELECT 'test_table_overviews'::regclass AS base_table, 1 AS z, '_vovw_1_test_table_overviews'::regclass AS overview_table
-        UNION ALL
-        SELECT 'test_table_overviews'::regclass AS base_table, 2 AS z, '_vovw_2_test_table_overviews'::regclass AS overview_table;
-    ELSE
-      RETURN;
-    END IF;
-  END
-$$ LANGUAGE PLPGSQL;
-
-CREATE OR REPLACE FUNCTION CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
-BEGIN
-  CASE
-    WHEN scaleDenominator > 500000000 THEN RETURN 0;
-    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
-    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
-    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
-    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
-    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
-    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
-    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
-    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
-    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
-    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
-    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
-    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
-    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
-    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
-    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
-    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
-    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
-    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
-    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
-    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
-    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
-    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
-    WHEN scaleDenominator <= 100 THEN RETURN 23;
-  END CASE;
-END
-$$ LANGUAGE plpgsql IMMUTABLE;

test/support/sql/CDB_QueryStatements.sql

@@ -1,14 +0,0 @@
--- Return an array of statements found in the given query text
---
--- Regexp curtesy of Hubert Lubaczewski (depesz)
--- Implemented in plpython for performance reasons
---
-CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text) 
-RETURNS SETOF TEXT AS $$
-  import re
-  pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL )
-  for match in pat.findall(query):
-    cleaned = match[0].strip()
-    if ( cleaned ):
-      yield cleaned
-$$ language 'plpythonu' IMMUTABLE STRICT;

test/support/sql/CDB_QueryTables.sql

@@ -1,78 +0,0 @@
--- Return an array of table names scanned by a given query
---
--- Requires PostgreSQL 9.x+
---
-CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
-RETURNS text[]
-AS $$
-DECLARE
-  exp XML;
-  tables text[];
-  rec RECORD;
-  rec2 RECORD;
-BEGIN
-  
-  tables := '{}';
-
-  FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
-
-    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
-        --RAISE WARNING 'Skipping %', rec.q;
-        CONTINUE;
-    END IF;
-
-    BEGIN
-      EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
-    EXCEPTION WHEN others THEN
-      -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
-      --       the affected table ?
-      RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
-      RAISE EXCEPTION '%', SQLERRM;
-      CONTINUE;
-    END;
-
-    -- Now need to extract all values of <Relation-Name>
-
-    -- RAISE DEBUG 'Explain: %', exp;
-
-    FOR rec2 IN WITH
-      inp AS (
-        SELECT
-          xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
-          xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
-      )
-      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
-    LOOP
-      -- RAISE DEBUG 'tab: %', rec2.p;
-      -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
-    END LOOP;
-
-    -- RAISE DEBUG 'Tables: %', tables;
-
-  END LOOP;
-
-  -- RAISE DEBUG 'Tables: %', tables;
-
-  -- Remove duplicates and sort by name
-  IF array_upper(tables, 1) > 0 THEN
-    WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
-       SELECT array_agg(p) from dist into tables;
-  END IF;
-
-  --RAISE DEBUG 'Tables: %', tables;
-
-  return tables;
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-
-
--- Keep CDB_QueryTables with same signature for backwards compatibility.
--- It should probably be removed in the future.
-CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
-RETURNS name[]
-AS $$
-BEGIN
-  RETURN CDB_QueryTablesText(query)::name[];
-END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

test/support/sql/_CDB_QueryStatements.sql

@@ -0,0 +1,16 @@
+-- DUMMY IMPLEMENTATION
+-- Ref: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_QueryStatements.sql
+-- Originally implemented in plpython for performance reasons
+
+-- Return an array of statements found in the given query text
+--
+-- Regexp curtesy of Hubert Lubaczewski (depesz)
+--
+CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
+RETURNS SETOF TEXT AS $$
+  with matches as (
+    select regexp_matches($1, $regexp$((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)$regexp$, 'g') as m
+  )
+  select btrim(m[1]) from matches
+$$
+LANGUAGE SQL IMMUTABLE STRICT;

test/support/sql/ported/populated_places_simple_reduced.sql

@@ -7385,3 +7385,8 @@ GRANT ALL ON TABLE populated_places_simple_reduced TO :TESTUSER;
 GRANT SELECT ON TABLE populated_places_simple_reduced TO :PUBLICUSER;
 
 VACUUM ANALYZE populated_places_simple_reduced;
+
+create table populated_places_simple_reduced_private AS
+select * from populated_places_simple_reduced;
+
+GRANT ALL ON TABLE populated_places_simple_reduced_private TO :TESTUSER;

test/support/test-client.js

@@ -3,6 +3,8 @@
 var qs = require('querystring');
 var step = require('step');
 
+var mapnik = require('windshaft').mapnik;
+
 var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
 
 var assert = require('./assert');
@@ -13,8 +15,9 @@ var serverOptions = require('../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
 
-function TestClient(mapConfig) {
+function TestClient(mapConfig, apiKey) {
     this.mapConfig = mapConfig;
+    this.apiKey = apiKey;
     this.keysToDelete = {};
 }
 
@@ -114,6 +117,114 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
     );
 };
 
+TestClient.prototype.getTile = function(z, x, y, params, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = params;
+        params = {};
+    }
+
+    var url = '/api/v1/map';
+
+    if (this.apiKey) {
+        url += '?' + qs.stringify({api_key: this.apiKey});
+    }
+
+    var layergroupId;
+    step(
+        function createLayergroup() {
+            var next = this;
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(self.mapConfig)
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+                    return next(null, JSON.parse(res.body).layergroupid);
+                }
+            );
+        },
+        function getTileResult(err, _layergroupId) {
+            assert.ifError(err);
+
+            var next = this;
+            layergroupId = _layergroupId;
+
+            url = '/api/v1/map/' + layergroupId + '/';
+
+            var layers = params.layers;
+
+            if (layers !== undefined) {
+                layers = Array.isArray(layers) ? layers : [layers];
+                url += layers.join(',') + '/';
+            }
+
+            var format = params.format || 'png';
+
+            url += [z,x,y].join('/');
+            url += '.' + format;
+
+            if (self.apiKey) {
+                url += '?' + qs.stringify({api_key: self.apiKey});
+            }
+
+            var request = {
+                url: url,
+                method: 'GET',
+                headers: {
+                    host: 'localhost'
+                }
+            };
+
+            var expectedResponse = {
+                status: 200,
+                headers: {
+                    'Content-Type': 'application/json; charset=utf-8'
+                }
+            };
+
+            var isPng = format === 'png' || format === 'torque.png';
+
+            if (isPng) {
+                request.encoding = 'binary';
+                expectedResponse.headers['Content-Type'] = 'image/png';
+            }
+
+            assert.response(server, request, expectedResponse, function(res, err) {
+                assert.ifError(err);
+
+                var image;
+
+                if (isPng) {
+                    image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+                }
+
+                next(null, res, image);
+            });
+        },
+        function finish(err, res, image) {
+            self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+            self.keysToDelete['user:localhost:mapviews:global'] = 5;
+            return callback(err, res, image);
+        }
+    );
+};
+
 TestClient.prototype.drain = function(callback) {
     helper.deleteRedisKeys(this.keysToDelete, callback);
 };

test/support/test_helper.js

@@ -63,9 +63,25 @@ function checkCache(res) {
 
 function checkSurrogateKey(res, expectedKey) {
     assert.ok(res.headers.hasOwnProperty('surrogate-key'));
-    assert.equal(res.headers['surrogate-key'], expectedKey);
+
+    function createSet(keys, key) {
+        keys[key] = true;
+        return keys;
+    }
+    var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {});
+    var expectedKeys = expectedKey.split(' ').reduce(createSet, {});
+
+    assert.deepEqual(keys, expectedKeys);
 }
 
+var redisClient;
+
+beforeEach(function() {
+    if (!redisClient) {
+        redisClient = redis.createClient(global.environment.redis.port);
+    }
+});
+
 //global afterEach to capture test suites that leave keys in redis
 afterEach(function(done) {
 
@@ -102,7 +118,6 @@ afterEach(function(done) {
     }
 
     Object.keys(databasesTasks).forEach(function(db) {
-        var redisClient = redis.createClient(global.environment.redis.port);
         redisClient.select(db, function() {
             // Check that we start with an empty redis db
             redisClient.keys("*", function(err, keys) {
@@ -129,6 +144,7 @@ function deleteRedisKeys(keysToDelete, callback) {
         var redisClient = redis.createClient(global.environment.redis.port);
         redisClient.select(keysToDelete[k], function() {
             redisClient.del(k, function(err, deletedKeysCount) {
+                redisClient.quit();
                 assert.notStrictEqual(deletedKeysCount, 0, 'No KEYS deleted for: [db=' + keysToDelete[k] + ']' + k);
                 taskDone(k);
             });

test/unit/cartodb/overviews_query_rewriter.js

@@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
         }
     };
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM  (\
             SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
             UNION ALL\
             SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
             SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
             UNION ALL\
             SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
             SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
 
     assertSameSql(overviews_sql, expected_sql);
@@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          \"_vovw_table 1\" AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM \"_vovw_table 1\"\
+          ) AS \"_vovw_table 1\"\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          \"_vovw_table 1\" AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM \"_vovw_table 1\"\
+          ) AS \"_vovw_table 1\"\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT column1, column2, column3 FROM _vovw_table1\
-    ";
-    assertSameSql(overviews_sql, expected_sql);
-    done();
-  });
-
-  it('generates query using overviews for queries with selected columns and all columns', function(done){
-    var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
-    var data = {
-        overviews: {
-            table1: {
-              2: { table: 'table1_ov2' }
-            }
-        }
-    };
-    var overviews_sql = overviewsQueryRewriter.query(sql, data);
-    var expected_sql = "\
-        WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
-            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
-            UNION ALL\
-            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
   });
 
   it('generates query using overviews for queries with a semicolon', function(done){
-    var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
+    var sql = "SELECT column1, column2, column3 FROM table1;";
     var data = {
         overviews: {
             table1: {
@@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
         }
     };
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
+          ) AS _vovw_table1;\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
   });
 
   it('generates query using overviews for queries with extra whitespace', function(done){
-    var sql = "  SELECT  table1.* , column1,column2,  column3 FROM  table1  ";
+    var sql = "  SELECT  column1,column2,  column3 FROM  table1  ";
     var data = {
         overviews: {
             table1: {
@@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1,column2, column3 FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -422,8 +388,39 @@ describe('Overviews query rewriter', function() {
     overviews_sql = overviewsQueryRewriter.query(sql, data);
     assert.equal(overviews_sql, sql);
 
+    sql = "SELECT table1.*, column1, column2, column3 FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
     done();
   });
 
-
+  it('generates overviews for wrapped query', function(done){
+    var sql = "SELECT * FROM (SELECT * FROM table1) AS wrapped_query WHERE 1=1";
+    var data = {
+        overviews: {
+            table1: {
+              0: { table: 'table1_ov0' },
+              1: { table: 'table1_ov1' },
+              2: { table: 'table1_ov2' }
+            }
+        }
+    };
+    var overviews_sql = overviewsQueryRewriter.query(sql, data);
+    var expected_sql = "\
+        WITH\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (SELECT * FROM  (\
+            SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+            UNION ALL\
+            SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
+            UNION ALL\
+            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
+            UNION ALL\
+            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
+          ) AS _vovw_table1) AS wrapped_query WHERE 1=1\
+    ";
+    assertSameSql(overviews_sql, expected_sql);
+    done();
+  });
 });

tools/README.md

@@ -1,8 +0,0 @@
-Deprecated tools
-================
-
-All tools and scripts found in this directory are deprecated and no longer maintained.
-
-Use at your own peril.
-
-In future releases they might get removed.

tools/convert_database_styles

@@ -1,81 +0,0 @@
-#!/usr/bin/env node
-
-var path = require('path');
-
-var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
-var mapnik = require('mapnik');
-var redis = require('redis');
-
-function usage(me, exitcode) {
-  console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
-  process.exit(exitcode);
-}
-
-var node_path = process.argv.shift(); 
-var script_path = process.argv.shift(); 
-var me = path.basename(script_path);
-
-var database_name = process.argv.shift()
-var table_name = process.argv.shift()
-var MAPNIK_VERSION = process.argv.shift()
-
-
-if ( ! MAPNIK_VERSION ) {
-  MAPNIK_VERSION = mapnik.versions.mapnik;
-}
-
-if ( ! database_name || ! table_name) {
-  usage(me, 1);
-}
-
-var REDIS_PORT = 6379; // TODO: make a command line parameter
-
-var dbnum = 0;
-
-var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
-
-var failures = [];
-
-var client = redis.createClient(REDIS_PORT, 'localhost');
-client.on('connect', function() {
-  client.select(dbnum);
-  client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
-
-    processNext = function() {
-      if ( ! matches.length ) process.exit(failures.length);
-      var k = matches.shift();
-
-      if ( /map_style\|.*\|.*\|/.test(k) ) {
-        //console.warn("Key " + k + " is EXTENDED, skipping");
-        processNext();
-      } 
-
-      var out = 'map_style|' + database_name + '|' + table_name + ': ';
-
-      var mml_builder = mml_store.mml_builder({
-        dbname:database_name, 
-        table:table_name},
-        function(err, payload) {
-          if ( err ) { 
-            console.warn(out + err.message); 
-            failures.push(k); processNext(); 
-          }
-          else {
-            mml_builder.resetStyle(function(err, data) {
-              if ( err ) { 
-                console.warn(out + err.message); 
-                failures.push(k); 
-              }
-              else console.log(out + 'OK');
-              processNext();
-            }, true);
-          }
-      });
-
-    };
-
-    processNext();
-
-  });
-
-});

tools/create_multilayer

@@ -1,40 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
-
-# This is for direct windshaft connection
-#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    tiler_url="$1"
-  fi
-  shift
-done
-
-if test -z "$cfg"; then
-  echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  exit 1
-fi
-
-cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
-echo $tok

tools/create_template

@@ -1,49 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$cfg"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-# Successful response contains no space
-echo "$res" | grep " " && { echo $res && exit 1; }
-
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
-echo $tok
-

tools/delete_template

@@ -1,45 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$tpl"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
-echo $tok

tools/examples/mapconfig_simple.js

@@ -1,11 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"cartodb",
-	  "options":{
-      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
-	    "cartocss":"#style{ marker-width: 12;}",
-	    "cartocss_version":"2.1.1",
-      "Interactivity":"id"
-    }
-  }]
-}

tools/examples/mapconfig_slow.js

@@ -1,11 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"cartodb",
-	  "options":{
-      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
-	    "cartocss":"#style{ marker-width: 12;}",
-	    "cartocss_version":"2.1.1",
-      "Interactivity":"id"
-    }
-  }]
-}

tools/examples/mapconfig_torque.js

@@ -1,10 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"torque",
-	  "options":{
-      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
-	    "cartocss":"Map{ -torque-time-attribute:'id'; -torque-aggregation-function:'count(id)'; -torque-frame-count:2; -torque-resolution:2}",
-      "cartocss_version": "2.1.1"
-    }
-  }]
-}

tools/examples/template_simple.js

@@ -1,17 +0,0 @@
-{
- "version":"0.0.1",
- "name":"simple",
- "placeholders":{},
- "auth":{ "method":"open" },
- "layergroup":{
-   "version":"1.0.1",
-    "layers":[{
-      "type":"cartodb",
-      "options":{
-        "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
-        "cartocss":"#s{ marker-width: 12;}",
-        "cartocss_version":"2.1.1"
-      }
-    }]
-  }
-}

tools/instanciate_template

@@ -1,53 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$tpl"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> [<template_params>]" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-if test -z "$cfg"; then
-  cfg="/dev/null"
-fi
-
-tiler_url="${tiler_url}/${tpl}"
-
-cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
-echo $tok

tools/list_templates

@@ -1,45 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test "$1" = "-h" -o "$1" = "-?"; then
-    echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
-    echo "Default <tiler_url> is ${tiler_url}" >&2
-    echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-    exit 0
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-
-cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-node <<EOF
-var parsed = JSON.parse('$res');
-console.dir(parsed);
-EOF
-

tools/munin/Makefile

@@ -1,19 +0,0 @@
-MUNIN_PLUGINS_DIR=/etc/munin/plugins
-MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
-PWD=$(shell pwd)
-
-all: windshaft.conf
-
-windshaft.conf: windshaft.conf.in
-	sed 's#@PWD@#$(PWD)#' < $< > $@
-
-install-munin-plugin-conf: windshaft.conf 
-	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
-
-install-munin-plugin: windshaft
-	install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
-
-install: install-munin-plugin install-munin-plugin-conf
-
-clean:
-	rm -f windshaft.conf

tools/munin/windshaft

@@ -1,87 +0,0 @@
-#!/bin/sh
-
-envnik=$(basename "${TILER_ENVIRONMENT}" .js)
-
-if test "$1" = "config"; then
-echo "graph_title fd usage (${envnik})"
-cat <<"EOM"
-graph_vlabel number of file descriptors
-graph_category windshaft
-graph_scale no
-procs.label Number of tiler processes
-pgsql.label PostgreSQL connections (max)
-redis.label Redis connections (max)
-http.label Incoming http requests (max)
-caches.label Number of renderer caches (max)
-nfd.label Total file descriptors (max)
-EOM
-exit 0
-fi
-
-if test x"$1" != x; then
-  TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
-fi
-
-if test -z "$TILER_ENVIRONMENT"; then
-  echo "Usage: $0 [<environment>]" >&2
-  echo "   or: [TILER_ENVIRONMENT=<environment>] $0" >&2
-  exit 1
-fi
-
-http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
-pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
-redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
-
-pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
-nworkers=$(echo "${pids}" | wc -l)
-pids=$(echo "${pids}" | paste -sd ' ')
-
-if test -z "${pids}"; then
-  echo "No processes found listening on tcp port '${http_port}'" >&2
-  exit 1
-fi
-
-tmpreport="/tmp/checkfd.$$.txt"
-
-lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
-
-maxdb=0
-maxredis=0
-maxhttp=0
-maxtot=0
-maxcache=0
-
-for pid in ${pids}; do
-
-  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
-  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
-  
-  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
-  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
-
-  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
-  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
-
-  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
-  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
-
-  log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
-  if test -e "${log}"; then
-    kill -USR2 "${pid}"
-    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l)
-    if test $cnt -gt $maxcache; then maxcache=$cnt; fi
-  else
-    # report the error...
-    maxcache=-1
-  fi
-
-done
-
-echo "procs.value ${nworkers}"
-echo "pgsql.value ${maxdb}"
-echo "redis.value ${maxredis}"
-echo "http.value ${maxhttp}"
-echo "caches.value ${maxcache}"
-echo "nfd.value ${maxtot}"
-
-rm -f "${tmpreport}"

tools/munin/windshaft.conf.in

@@ -1,5 +0,0 @@
-# Configuration file for munin plugin
-
-[windshaft]
-user root
-env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js

tools/performance/stresstest_templates.sh

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$tpl"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-basedir=$(cd $(dirname $0); cd ..; pwd)
-export CDB_APIKEY=${apikey}
-max=3000000
-i=0
-while test "$i" -le "$max"; do
-  tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"`
-  tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin`
-  if test $? -ne 0; then
-    echo $tpl_id >&2
-    break
-  fi
-  tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin`
-  if test $? -ne 0; then
-    echo $tpl_id >&2
-    break
-  fi
-  out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}`
-  if test $? -ne 0; then
-    echo $out >&2
-    break
-  fi
-  i=$((i+1))
-  if test `expr $i % 100` -eq 0; then
-    echo -n "."
-  fi
-done

tools/reset_styles

@@ -1,105 +0,0 @@
-#!/usr/bin/env node
-
-/*
-
-This scripts drops all extended map_style keys in redis and regenerates
-the XML caches in all the base ones to target the configured mapnik_version.
-
-Optionally (with --convert) it also re-writes the CartoCSS if needed
-to target the configured mapnik_version.
-
-It is recommended to make a backup of the redis database before using
-this script.
-
-*/
-
-var path = require('path');
-
-// Reset all styles in the store
-var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
-var mapnik = require('mapnik');
-var redis = require('redis');
-
-function usage(me, exitcode) {
-  console.log("Usage: " + me + " [--convert] <environment>");
-  process.exit(exitcode);
-}
-
-var doConvert = false;
-
-var node_path = process.argv.shift(); 
-var script_path = process.argv.shift(); 
-var me = path.basename(script_path);
-var ENV;
-var arg;
-while ( arg = process.argv.shift() ) {
-  if ( arg == '--convert' ) {
-    doConvert = true;
-  }
-  else if ( ! ENV ) {
-    ENV = arg;
-  }
-  else {
-    usage(me, 1);
-  }
-}
-
-if ( ! ENV ) usage(me, 1);
-
-global.environment = require('../config/environments/' + ENV);
-var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
-
-var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
-
-console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
-
-var dbnum = 0;
-
-var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
-
-var failures = [];
-
-var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
-client.on('connect', function() {
-  client.select(dbnum);
-  client.keys('map_style|*', function(err, matches) {
-
-    processNext = function() {
-      if ( ! matches.length ) process.exit(failures.length);
-      var k = matches.shift();
-
-      if ( /map_style\|.*\|.*\|/.test(k) ) {
-        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
-        //console.warn("Key " + k + " is EXTENDED, dropping");
-        client.del(k, function(err) {
-          if ( err ) console.warn("Error dropping key " + k);
-          processNext();
-        });
-        return;
-      } 
-
-      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
-      var db = params[1];
-      var tab = params[2];
-      var out = 'map_style|' + db + '|' + tab + ': ';
-
-      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
-                                              function(err, payload) {
-
-          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
-          else {
-            mml_builder.resetStyle(function(err, data) {
-              if ( err ) { console.warn(out + err.message); failures.push(k); }
-              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
-              processNext();
-            }, doConvert);
-          }
-      });
-
-    };
-
-    processNext();
-
-  });
-
-});

tools/show_style

@@ -1,148 +0,0 @@
-#!/usr/bin/env node
-
-var path = require('path');
-var redis = require('redis');
-var Step = require('step');
-
-
-function usage(me, exitcode) {
-  console.log("Usage: " + me + " [--env <environment>] <username> [<tablename>|~<token>]");
-  process.exit(exitcode);
-}
-
-var node_path = process.argv.shift(); 
-var script_path = process.argv.shift(); 
-var basedir = path.dirname(script_path);
-var me = path.basename(script_path);
-
-var ENV = 'development.js';
-var username, token;
-var arg;
-while ( arg = process.argv.shift() ) {
-  if ( arg == '--env' ) {
-    ENV = process.argv.shift();
-  }
-  else if ( ! username ) {
-    username = arg;
-  }
-  else if ( ! token ) {
-    token = arg;
-  }
-  else {
-    console.warn("Unused parameter " + arg);
-  }
-}
-
-if ( ! username ) usage(me, 1);
-
-console.log("Using environment " + ENV);
-
-global.environment = require('../config/environments/' + ENV);
-// _after_ setting global.environment
-var serverOptions = require('../lib/cartodb/server_options')();
-
-var client;
-var dbname;
-Step(
-  function getClient() {
-    client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
-    client.on('connect', this);
-  },
-  function getUserMeta(err) {
-    if ( err ) throw err;
-    client.select(5);
-    client.hgetall('rails:users:' + username, this);
-  },
-  function readDB(err, data) {
-    if ( err ) throw err;
-    if ( ! data )
-      throw new Error('Username ' + username + ' unknown by redis on port '
-        + serverOptions.redis.port + ' (try CARTODB/script/restore_redis?)');
-    //console.log("Data:"); console.dir(data);
-    dbname = data['database_name'];
-    console.log("Database name for user " + username + ": " + dbname);
-    client.select(0);
-    return null;
-  },
-  function showTokens(err) {
-    if ( err ) throw err;
-    if ( token ) return null;
-    var next = this;
-    Step(
-      function getTokens() {
-        client.keys('map_style|' + dbname + '|*', this);
-      },
-      function showTokens(err, data) {
-        if (err) throw err;
-        if ( data ) console.log(data.join('\n'));
-        return null;
-      },
-      function showTokensFinish(err) {
-        next(err);
-      }
-    );
-  },
-  function showStyle(err) {
-    if ( err ) throw err;
-    if ( ! token ) return null;
-    var next = this;
-    Step(
-      function getStyle() {
-        client.get('map_style|' + dbname + '|' + token, this);
-      },
-      function showStyle(err, data) {
-        if ( err ) throw err;
-        if ( ! data ) {
-          throw new Error(token + ': no such map style known by redis on port '
-            + serverOptions.redis.port);
-        }
-//console.log("data: " + data);
-        var x=JSON.parse(data);
-        printMapnikStyle(x, this);
-      },
-      function showStyleFinish(err) {
-        next(err);
-      }
-    );
-  },
-  function finish(err) {
-    if ( err ) {
-      console.error(err.message)
-      process.exit(1);
-    }
-    process.exit(0);
-  }
-);
-
-function printMapnikStyle(x, callback) {
-  console.log('style: ' + x.style);
-  console.log('version: ' + x.version);
-  var grainstore = require(basedir + '/../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
-  var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
-  var builderconfig = {dbname:dbname};
-  if ( token.match(/^~/) ) {
-    builderconfig.token = token.substring(1);
-  } else {
-    builderconfig.table = token;
-  }
-  var mml_builder;
-  Step(
-    function getBuilder() {
-      mml_builder = mml_store.mml_builder(builderconfig, this);
-    },
-    function getXML(err, builder) {
-      if ( err ) throw err;
-      mml_builder.toXML(this);
-    },
-    function showXML(err, xml) {
-      if ( err ) throw err;
-      console.log('- XML - ');
-      console.log(xml);
-      return null;
-    },
-    function finish(err) {
-      callback(err);
-    }
-  );
-}
-

tools/update_template

@@ -1,47 +0,0 @@
-#!/bin/sh
-
-verbose=no
-tiler_url=http://dev.localhost.lan:8181/tiles/template
-apikey=${CDB_APIKEY}
-
-while test -n "$1"; do
-  if test "$1" = "-v"; then
-    verbose=yes
-  elif test "$1" = "-k"; then
-    shift
-    apikey="$1"
-  elif test "$1" = "-u"; then
-    shift
-    tiler_url="$1"
-  elif test -z "$tpl"; then
-    tpl="$1"
-  elif test -z "$cfg"; then
-    cfg="$1"
-  else
-    echo "Unused parameter $1" >&2
-  fi
-  shift
-done
-
-if test -z "$cfg"; then
-  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> <template_config>" >&2
-  echo "Default <tiler_url> is ${tiler_url}" >&2
-  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
-  exit 1
-fi
-
-cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}"
-if test x${verbose} = xyes; then
-  cmd="${cmd} -v"
-fi
-res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
-if test $? -gt 0; then
-  echo "curl command failed: ${cmd}"
-fi
-
-if test x${verbose} = xyes; then
-  echo "${res}"
-fi
-
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
-echo $tok