Release 2.87.5

Cast all dataview overrides values to Number

.gitignore

@@ -2,6 +2,7 @@ node_modules*
 config.status*
 config/environments/*.js
 .idea
+.vscode
 tools/munin/windshaft.conf
 logs/
 pids/

.travis.yml

@@ -1,9 +1,9 @@
-dist: trusty # only environment that supports Postgres 9.5 at this time
-sudo: required
+dist: trusty
 addons:
   postgresql: "9.5"
   apt:
     packages:
+      - postgresql-9.5-postgis-2.3
       - postgresql-plpython-9.5
       - pkg-config
       - libcairo2-dev
@@ -22,9 +22,3 @@ env:
 language: node_js
 node_js:
   - "0.10"
-
-notifications:
-  irc:
-    channels:
-      - "irc.freenode.org#cartodb"
-    use_notice: true

CONTRIBUTING.md

@@ -8,4 +8,4 @@ We love pull requests from everyone, see [Contributing to Open Source on GitHub]
 
 ## Submitting Contributions
 
-* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

NEWS.md

@@ -1,5 +1,170 @@
 # Changelog
 
+## 2.87.5
+Released 2017-02-02
+
+Bug fixes:
+ - Cast dataview override values to Number or throw error.
+
+
+## 2.87.4
+Released 2017-01-20
+
+Bug fixes:
+ - Be able to not compute NULL categories and null values  in aggregation dataviews #617.
+
+
+## 2.87.3
+Released 2016-12-19
+
+Bug fixes:
+ - Fix overviews-related dataviews problems. See https://github.com/CartoDB/Windshaft-cartodb/pull/604
+
+
+## 2.87.2
+Released 2016-12-19
+
+- Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
+
+## 2.87.1
+Released 2016-12-13
+
+Announcements:
+ - Upgrades windshaft to [2.6.4](https://github.com/CartoDB/Windshaft/releases/tag/2.6.4).
+ - Upgrades request dependency.
+ - Regenerate npm-shrinkwrap.json: missing dependency updates.
+
+
+## 2.87.0
+Released 2016-12-12
+
+Enhancements:
+ - Upgrade turbo-carto dependency to version 0.19.0.
+
+## 2.86.1
+Released 2016-12-02
+
+Bug fixes:
+ - Maps with analyses and `sql_wrap` were broken #599.
+
+
+## 2.86.0
+Released 2016-12-02
+
+Announcements:
+ - Upgrades windshaft to [2.6.3](https://github.com/CartoDB/Windshaft/releases/tag/2.6.3).
+
+
+## 2.85.1
+Released 2016-11-30
+
+Announcements:
+ - Upgrades camshaft to [0.48.4](https://github.com/CartoDB/camshaft/releases/tag/0.48.4).
+
+
+## 2.85.0
+Released 2016-11-24
+
+New features:
+ - Allow to set resource URL templates with substitution tokens #594.
+
+
+## 2.84.2
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.3](https://github.com/CartoDB/camshaft/releases/tag/0.48.3).
+
+
+## 2.84.1
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.2](https://github.com/CartoDB/camshaft/releases/tag/0.48.2).
+
+
+## 2.84.0
+Released 2016-11-11
+
+New features:
+ - Analyses limit configuration allows to set other limits than timeout.
+
+
+## 2.83.1
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.1](https://github.com/CartoDB/camshaft/releases/tag/0.48.1).
+
+
+## 2.83.0
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.0](https://github.com/CartoDB/camshaft/releases/tag/0.48.0).
+
+
+## 2.82.0
+Released 2016-11-08
+
+Announcements:
+ - Upgrades camshaft to [0.47.0](https://github.com/CartoDB/camshaft/releases/tag/0.47.0).
+
+
+## 2.81.1
+Released 2016-11-05
+
+Announcements:
+ - Upgrades windshaft to [2.6.2](https://github.com/CartoDB/windshaft/releases/tag/2.6.2).
+ - Upgrades camshaft to [0.46.3](https://github.com/CartoDB/camshaft/releases/tag/0.46.3).
+
+
+## 2.81.0
+Released 2016-11-02
+
+Enhancements:
+ - Returns errors with context when query layer does not retrieve geometry column
+
+Announcements:
+ - Upgrades windshaft to [2.6.1](https://github.com/CartoDB/windshaft/releases/tag/2.6.1).
+ - Upgrades camshaft to [0.46.2](https://github.com/CartoDB/camshaft/releases/tag/0.46.2).
+
+
+## 2.80.2
+Released 2016-10-26
+
+Bug fixes:
+ - Fix order in categories query to get ramps
+
+
+## 2.80.1
+Released 2016-10-25
+
+Announcements:
+ - Upgrades camshaft to [0.46.1](https://github.com/CartoDB/camshaft/releases/tag/0.46.1).
+
+
+## 2.80.0
+Released 2016-10-20
+
+Announcements:
+ - Upgrades camshaft to [0.46.0](https://github.com/CartoDB/camshaft/releases/tag/0.46.0).
+
+New features:
+ - Default analyses limits can be defined in configuration.
+
+
+## 2.79.0
+Released 2016-10-11
+
+New features:
+ - Retrieve analysis limits and pass them into camshaft.
+
+Announcements:
+ - Upgrades turbo-carto to [0.18.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.18.0).
+ - Upgrades camshaft to [0.45.0](https://github.com/CartoDB/camshaft/releases/tag/0.45.0).
+
+
 ## 2.78.1
 Released 2016-09-30
 

config/environments/development.js.example

@@ -23,6 +23,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -96,6 +111,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -216,6 +235,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/production.js.example

@@ -23,6 +23,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -90,6 +105,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -210,6 +229,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/staging.js.example

@@ -23,6 +23,21 @@ var config = {
     // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -90,6 +105,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -210,6 +229,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/test.js.example

@@ -23,6 +23,20 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -90,6 +104,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -211,6 +229,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

docs/named_maps.md

@@ -73,10 +73,6 @@ The `name` argument defines how to name this "template_name".json. Note that the
       }
     ]
   },
-  "preview_layers": {
-    "0": true,
-    "layer1": false
-  },
   "view": {
     "zoom": 4,
     "center": {
@@ -88,6 +84,10 @@ The `name` argument defines how to name this "template_name".json. Note that the
       "south": -45,
       "east": 45,
       "north": 45
+    },
+    "preview_layers": {
+      "0": true,
+      "layer1": false
     }
   }
 }
@@ -105,7 +105,7 @@ auth |
 |_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.carto.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
 placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
 layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.carto.com/carto-engine/maps-api/mapconfig/) for more information.
-view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box. Also it is possible to choose which layers are visible or not with `preview_layers` indicating its visibility by layer index or id (visible by default).
 --- | ---
 |_ zoom | The zoom level to use
 

docs/static_maps_api.md

@@ -146,10 +146,10 @@ It is important to note that generated images are cached from the live data refe
 
 ### Limits
 
-* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
-* Image resolution by default is set to 72 DPI
-* JPEG quality by default is 85%
-* Timeout limits for generating static maps are the same across the CARTO Editor and CARTO Engine. It is important to ensure timely processing of queries.
+* While images can encompass an entirety of a map, the limit for pixel range is 8192 x 8192.
+* Image resolution is set to 72 DPI
+* JPEG quality is 85%
+* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
 
 ## Examples
 

lib/cartodb/backends/analysis.js

@@ -1,10 +1,20 @@
+'use strict';
+
+var _ = require('underscore');
 var camshaft = require('camshaft');
 var fs = require('fs');
 
-function AnalysisBackend (options) {
-    options = options || {};
-    this.setBatchConfig(options.batch);
-    this.setLoggerConfig(options.logger);
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.options.limits = this.options.limits || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
 }
 
 module.exports = AnalysisBackend;
@@ -18,11 +28,7 @@ AnalysisBackend.prototype.setBatchConfig = function (options) {
 };
 
 AnalysisBackend.prototype.setLoggerConfig = function (options) {
-    var loggerConfig = options || {};
-
-    loggerConfig.filename = loggerConfig.filename;
-
-    this.loggerConfig = loggerConfig;
+    this.loggerConfig = options || {};
 
     if (this.loggerConfig.filename) {
         this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
@@ -46,5 +52,42 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
         stream: this.stream ? this.stream : process.stdout
     };
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+
+    var analysesLimits = {
+        analyses: {
+            // buffer: {
+            //     timeout: 1000,
+            //     maxNumberOfRows: 1e6
+            // }
+        }
+    };
+
+    Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) {
+        analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]);
+    });
+
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        // analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 }
+        analysesTimeouts = analysesTimeouts || {};
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = _.defaults(
+                {
+                    timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+                },
+                analysesLimits.analyses[analysisType]
+            );
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/dataview.js

@@ -79,7 +79,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
 
             var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
                 function castNumbers(overrides, val, k) {
-                    overrides[k] = Number.isFinite(+val) ? +val : val;
+                    if (!Number.isFinite(+val)) {
+                        throw new Error('Invalid number format for parameter \'' + k + '\'');
+                    }
+                    overrides[k] = +val;
                     return overrides;
                 },
                 {ownFilter: ownFilter}

lib/cartodb/backends/template_maps.js

@@ -55,11 +55,9 @@ util.inherits(TemplateMaps, EventEmitter);
 module.exports = TemplateMaps;
 
 
-var o = TemplateMaps.prototype;
-
 //--------------- PRIVATE METHODS --------------------------------
 
-o._userTemplateLimit = function() {
+TemplateMaps.prototype._userTemplateLimit = function() {
   return this.opts.max_user_templates || 0;
 };
 
@@ -70,7 +68,7 @@ o._userTemplateLimit = function() {
  * @param redisArgs - the arguments for the redis function in an array
  * @param callback - function to pass results too.
  */
-o._redisCmd = function(redisFunc, redisArgs, callback) {
+TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
   var redisClient;
   var that = this;
   var db = that.db_signatures;
@@ -97,7 +95,7 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
 var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
 // jshint maxcomplexity:15
-o._checkInvalidTemplate = function(template) {
+TemplateMaps.prototype._checkInvalidTemplate = function(template) {
   if ( template.version !== '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
   }
@@ -200,7 +198,7 @@ function templateDefaults(template) {
 // @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
-o.addTemplate = function(owner, template, callback) {
+TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
     var self = this;
 
     template = templateDefaults(template);
@@ -258,7 +256,7 @@ o.addTemplate = function(owner, template, callback) {
 //
 // @param callback function(err)
 //
-o.delTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function deleteTemplate() {
@@ -297,7 +295,8 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //
 // @param callback function(err)
 //
-o.updTemplate = function(owner, tpl_id, template, callback) {
+TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
+    
     var self = this;
 
     template = templateDefaults(template);
@@ -356,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
 //
-o.listTemplates = function(owner, callback) {
+TemplateMaps.prototype.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
 
@@ -370,7 +369,7 @@ o.listTemplates = function(owner, callback) {
 // @param callback function(err, template)
 //        Return full template definition
 //
-o.getTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function getTemplate() {
@@ -386,7 +385,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
     );
 };
 
-o.isAuthorized = function(template, authTokens) {
+TemplateMaps.prototype.isAuthorized = function(template, authTokens) {
     if (!template) {
         return false;
     }
@@ -438,7 +437,7 @@ function _replaceVars (str, params) {
     });
     return str;
 }
-o.instance = function(template, params) {
+TemplateMaps.prototype.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
   Object.keys(phold).forEach(function(k) {
@@ -500,7 +499,7 @@ o.instance = function(template, params) {
 };
 
 // Return a fingerPrint of the object
-o.fingerPrint = function(template) {
+TemplateMaps.prototype.fingerPrint = function(template) {
   return crypto.createHash('md5')
     .update(JSON.stringify(template))
     .digest('hex')

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -32,7 +32,7 @@ methodTemplates.category = dot.template([
     '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
     '  FROM ({{=it._sql}}) _cdb_aggregation_all',
     '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
+    '  ORDER BY 2 DESC, 1 ASC',
     '),',
     'agg_categories AS (',
     '  SELECT category',

lib/cartodb/controllers/map.js

@@ -1,4 +1,6 @@
 var _ = require('underscore');
+var dot = require('dot');
+dot.templateSettings.strip = false;
 var assert = require('assert');
 var step = require('step');
 var windshaft = require('windshaft');
@@ -44,6 +46,20 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.mapConfigAdapter = mapConfigAdapter;
+
+    this.resourcesUrlTemplates = null;
+    if (global.environment.resources_url_templates) {
+        var templates = global.environment.resources_url_templates;
+
+        if (templates.http) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
+        }
+        if (templates.https) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
+        }
+    }
 }
 
 util.inherits(MapController, BaseController);
@@ -166,11 +182,29 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function finish(err, layergroup) {
             if (err) {
+                if (Number.isFinite(err.layerIndex)) {
+                    var error = new Error(err.message);
+                    error.http_status = err.http_status;
+
+                    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
+                        error.http_status = 400;
+                    }
+
+                    error.type = 'layer';
+                    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
+                    error.layer = {
+                        id: mapConfig.getLayerId(err.layerIndex),
+                        index: err.layerIndex,
+                        type: mapConfig.layerType(err.layerIndex)
+                    };
+
+                    err = error;
+                }
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
                 var analysesResults = context.analysesResults || [];
-                addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
-                addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                self.addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
+                self.addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
                 addContextMetadata(layergroup, mapConfig.obj(), context);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
@@ -242,8 +276,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
                 var _mapConfig = mapConfig.obj();
-                addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
-                addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+                self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
+                self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
                 addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context);
 
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
@@ -352,7 +386,7 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
     }, lastUpdateTime);
 }
 
-function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) {
+MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
     includeQuery = includeQuery || false;
     analysesResults = analysesResults || [];
     layergroup.metadata.analyses = [];
@@ -365,7 +399,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                     var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
                     var nodeRepr = {
                         status: node.getStatus(),
-                        url: getUrls(username, nodeResource)
+                        url: this.getUrls(username, nodeResource)
                     };
                     if (includeQuery) {
                         nodeRepr.query = node.getQuery();
@@ -377,30 +411,30 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                 }
 
                 return nodesIdMap;
-            }, {})
+            }.bind(this), {})
         });
-    });
-}
+    }.bind(this));
+};
 
 // TODO this should take into account several URL patterns
-function addDataviewsAndWidgetsUrls(username, layergroup, mapConfig) {
-    addDataviewsUrls(username, layergroup, mapConfig);
-    addWidgetsUrl(username, layergroup, mapConfig);
-}
+MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) {
+    this.addDataviewsUrls(username, layergroup, mapConfig);
+    this.addWidgetsUrl(username, layergroup, mapConfig);
+};
 
-function addDataviewsUrls(username, layergroup, mapConfig) {
+MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) {
     layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
     var dataviews = mapConfig.dataviews || {};
 
     Object.keys(dataviews).forEach(function(dataviewName) {
         var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
         layergroup.metadata.dataviews[dataviewName] = {
-            url: getUrls(username, resource)
+            url: this.getUrls(username, resource)
         };
-    });
-}
+    }.bind(this));
+};
 
-function addWidgetsUrl(username, layergroup, mapConfig) {
+MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) {
     if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
         layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
             var mapConfigLayer = mapConfig.layers[layerIndex];
@@ -410,16 +444,19 @@ function addWidgetsUrl(username, layergroup, mapConfig) {
                     var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
                     layer.widgets[widgetName] = {
                         type: mapConfigLayer.options.widgets[widgetName].type,
-                        url: getUrls(username, resource)
+                        url: this.getUrls(username, resource)
                     };
-                });
+                }.bind(this));
             }
             return layer;
-        });
+        }.bind(this));
     }
-}
+};
 
-function getUrls(username, resource) {
+MapController.prototype.getUrls = function(username, resource) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, resource);
+    }
     var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
     if (cdnUrl) {
         return {
@@ -432,4 +469,29 @@ function getUrls(username, resource) {
             http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
         };
     }
-}
+};
+
+MapController.prototype.getUrlsFromTemplate = function(username, resource) {
+    var urls = {};
+    var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url || {};
+
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = this.resourcesUrlTemplates.http({
+            cdn_url: cdnUrl.http,
+            user: username,
+            port: global.environment.port,
+            resource: resource
+        });
+    }
+
+    if (this.resourcesUrlTemplates.https) {
+        urls.https = this.resourcesUrlTemplates.https({
+            cdn_url: cdnUrl.https,
+            user: username,
+            port: global.environment.port,
+            resource: resource
+        });
+    }
+
+    return urls;
+};

lib/cartodb/models/dataview/aggregation.js

@@ -19,25 +19,37 @@ var rankedCategoriesQueryTpl = dot.template([
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
     '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM ({{=it._query}}) _cdb_categories',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
     'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
     'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
 ].join('\n'));
@@ -45,7 +57,7 @@ var rankedAggregationQueryTpl = dot.template([
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
     '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
     'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
     'ORDER BY value DESC'
 ].join('\n'));
@@ -114,6 +126,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
     var _query = this.query;
 
     var aggregationSql;
+
     if (!!override.ownFilter) {
         aggregationSql = [
             "WITH",
@@ -125,9 +138,14 @@ Aggregation.prototype.sql = function(psql, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
@@ -150,9 +168,14 @@ Aggregation.prototype.sql = function(psql, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -18,25 +18,37 @@ var rankedCategoriesQueryTpl = dot.template([
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
     '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM ({{=it._query}}) _cdb_categories',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
     'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
     'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
 ].join('\n'));
@@ -44,7 +56,7 @@ var rankedAggregationQueryTpl = dot.template([
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
     '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
     'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
     'ORDER BY value DESC'
 ].join('\n'));
@@ -65,7 +77,7 @@ Aggregation.prototype.constructor = Aggregation;
 
 module.exports = Aggregation;
 
-Aggregation.prototype.sql = function(psql, filters, override, callback) {
+Aggregation.prototype.sql = function(psql, override, callback) {
     if (!callback) {
         callback = override;
         override = {};
@@ -85,9 +97,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
@@ -110,9 +127,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })

lib/cartodb/models/dataview/overviews/base.js

@@ -55,20 +55,20 @@ BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
 };
 
 // Default behaviour
-BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
+BaseOverviewsDataview.prototype.defaultSql = function(psql, override, callback) {
     var query = this.query;
     var dataview = this.baseDataview;
     if ( SETTINGS.defaultOverviews ) {
         query = this.rewrittenQuery(query);
         dataview = new this.BaseDataview(query, this.queryOptions);
     }
-    return dataview.sql(psql, filters, override, callback);
+    return dataview.sql(psql, override, callback);
 };
 
 // default implementation that can be override in derived classes:
 
-BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
-    return this.defaultSql(psql, filters, override, callback);
+BaseOverviewsDataview.prototype.sql = function(psql, override, callback) {
+    return this.defaultSql(psql, override, callback);
 };
 
 BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {

lib/cartodb/models/dataview/overviews/formula.js

@@ -36,7 +36,7 @@ Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-Formula.prototype.sql = function(psql, filters, override, callback) {
+Formula.prototype.sql = function(psql, override, callback) {
     var formulaQueryTpl = formulaQueryTpls[this.operation];
 
     if ( formulaQueryTpl ) {
@@ -52,5 +52,5 @@ Formula.prototype.sql = function(psql, filters, override, callback) {
     }
 
     // default behaviour
-    return this.defaultSql(psql, filters, override, callback);
+    return this.defaultSql(psql, override, callback);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -8,7 +8,6 @@ dot.templateSettings.strip = false;
 var columnTypeQueryTpl = dot.template(
     'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
 );
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
 
 var BIN_MIN_NUMBER = 6;
 var BIN_MAX_NUMBER = 48;
@@ -149,7 +148,9 @@ Histogram.prototype.sql = function(psql, override, callback) {
     }
 
     if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
+        // overviews currently aggregate dates to NULL
+        // to avoid problem we don't use overviews for histograms of date columns
+        return this.defaultSql(psql, override, callback);
     }
 
     var _query = this.rewrittenQuery(this.query);

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -33,7 +33,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf
                step(
                    function collectFiltersData() {
                        var filters, unfiltered_query;
-                       if ( layer.options.source && analysesResults ) {
+                       if ( layer.options.source && analysesResults && !layer.options.sql_wrap) {
                            var sourceId = layer.options.source.id;
                            var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
                            if ( node ) {

lib/cartodb/server.js

@@ -148,7 +148,7 @@ module.exports = function(serverOptions) {
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
 
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

lib/cartodb/server_options.js

@@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
     },
     logger: {
         filename: undefined
-    }
+    },
+    limits: {}
 });
 
 module.exports = {
@@ -69,6 +70,7 @@ module.exports = {
         },
         datasource: global.environment.postgres,
         cachedir: global.environment.millstone.cache_basedir,
+        use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
         mapnik_version: global.environment.mapnik_version,
         mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
         default_layergroup_ttl: global.environment.mapConfigTTL || 7200
@@ -101,7 +103,8 @@ module.exports = {
         },
         logger: {
             filename: analysisConfig.logger.filename
-        }
+        },
+        limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.78.1",
+  "version": "2.87.5",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.44.2",
+    "camshaft": "0.48.5",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "0.13.1",
@@ -34,13 +34,14 @@
     "node-statsd": "~0.0.7",
     "queue-async": "~1.0.7",
     "redis-mpool": "~0.4.0",
-    "request": "~2.62.0",
+    "request": "~2.79.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.17.1",
+    "turbo-carto": "0.19.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.5.0",
-    "yargs": "~5.0.0"
+    "windshaft": "2.6.5",
+    "yargs": "~5.0.0",
+    "zipfile": "cartodb/node-zipfile#0.5.11-cdb1"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",

test/acceptance/dataviews/aggregation.js

@@ -3,7 +3,7 @@ require('../../support/test_helper');
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
-describe('aggregations', function() {
+describe('aggregations happy cases', function() {
 
     afterEach(function(done) {
         if (this.testClient) {
@@ -13,30 +13,36 @@ describe('aggregations', function() {
         }
     });
 
-    function aggregationOperationMapConfig(operation) {
-        return {
+    function aggregationOperationMapConfig(operation, query, column, aggregationColumn) {
+        column = column || 'adm0name';
+        aggregationColumn = aggregationColumn || 'pop_max';
+        query = query || 'select * from populated_places_simple_reduced';
+
+        var mapConfig = {
             version: '1.5.0',
             layers: [
                 {
                     type: 'mapnik',
                     options: {
-                        sql: 'select * from populated_places_simple_reduced',
+                        sql: query,
                         cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
                         cartocss_version: '2.0.1',
-                        widgets: {
-                            adm0name: {
-                                type: 'aggregation',
-                                options: {
-                                    column: 'adm0name',
-                                    aggregation: operation,
-                                    aggregationColumn: 'pop_max'
-                                }
-                            }
-                        }
+                        widgets: {}
                     }
                 }
             ]
         };
+
+        mapConfig.layers[0].options.widgets[column] = {
+            type: 'aggregation',
+            options: {
+                column: column,
+                aggregation: operation,
+                aggregationColumn: aggregationColumn
+            }
+        };
+
+        return mapConfig;
     }
 
     var operations = ['count', 'sum', 'avg', 'max', 'min'];
@@ -56,4 +62,49 @@ describe('aggregations', function() {
             });
         });
     });
+
+    var query = [
+        'select 1 as val, \'a\' as cat, ST_Transform(ST_SetSRID(ST_MakePoint(0,0),4326),3857) as the_geom_webmercator',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(0,1),4326),3857)',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(1,0),4326),3857)',
+        'select null, null, ST_Transform(ST_SetSRID(ST_MakePoint(1,1),4326),3857)'
+    ].join(' UNION ALL ');
+
+    operations.forEach(function (operation) {
+        var not = operation === 'count' ? ' not ' : ' ';
+        var description = 'should' +
+            not +
+            'handle NULL values in category and aggregation columns using "' +
+            operation +
+            '" as aggregation operation';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 3);
+                assert.equal(aggregation.count, 4);
+                assert.equal(aggregation.nulls, 1);
+
+                var hasNullCategory = false;
+                aggregation.categories.forEach(function (category) {
+                    if (category.category === null) {
+                        hasNullCategory = true;
+                    }
+                });
+
+                if (operation === 'count') {
+                    assert.ok(hasNullCategory, 'aggregation has not a category NULL');
+                } else {
+                    assert.ok(!hasNullCategory, 'aggregation has category NULL');
+                }
+
+                done();
+            });
+        });
+    });
 });

test/acceptance/dataviews/histogram.js

@@ -77,4 +77,24 @@ describe('histogram-dataview', function() {
             done();
         });
     });
+
+    it('should cast all overridable params to numbers', function(done) {
+        var params = {
+            bins: '256 AS other, (select 256 * 2) AS bins_number--',
+            start: 1e3,
+            end: 0,
+            response: TestClient.RESPONSE.ERROR
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('pop_max_histogram', params, function(err, res) {
+            assert.ok(!err, err);
+
+            assert.ok(res.errors);
+            assert.equal(res.errors.length, 1);
+            assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
+
+            done();
+        });
+    });
 });

test/acceptance/errors-with-context.js

@@ -0,0 +1,68 @@
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+describe('error with context', function () {
+    var layerOK = {
+        options: {
+            sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator ' +
+                 'from test_table',
+            cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.1',
+            interactivity: 'cartodb_id'
+        }
+    };
+    var layerKO = {
+        options: {
+            sql: 'select cartodb_id from test_table offset 3', // it doesn't return the_geom_webmercator so it must fail
+            cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.2',
+            interactivity: 'cartodb_id'
+        }
+    };
+
+    var DB_ERROR_MESSAGE = 'Postgis Plugin: ERROR:  column "the_geom_webmercator" does not exist';
+    var scenarios = [{
+        description: 'layergroup with 2 layers, second one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerOK, layerKO]
+        },
+        expectedFailingLayer: { id: 'layer1', index: 1, type: 'mapnik' }
+    }, {
+        description: 'layergroup with 2 layers, first one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerKO, layerOK]
+        },
+        expectedFailingLayer: { id: 'layer0', index: 0, type: 'mapnik' }
+    }];
+
+    scenarios.forEach(function (scenario) {
+        it(scenario.description, function (done) {
+            assert.response(server, {
+                url: '/api/v1/map',
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(scenario.layergroup)
+            }, {
+                status: 400
+            }, function (res) {
+                var parsedBody = JSON.parse(res.body);
+
+                assert.ok(Array.isArray(parsedBody.errors_with_context));
+
+                var err = parsedBody.errors_with_context[0];
+                assert.equal(err.type, 'layer');
+                assert.equal(err.subtype, 'query');
+                assert.ok(err.message.indexOf(DB_ERROR_MESSAGE) >= 0);
+                assert.deepEqual(err.layer, scenario.expectedFailingLayer);
+                done();
+            });
+        });
+    });
+});

test/integration/analysis-backend-limits.js

@@ -0,0 +1,167 @@
+var testHelper = require('../support/test_helper');
+
+var assert = require('assert');
+var redis = require('redis');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
+
+describe('analysis-backend limits', function() {
+
+    var redisClient;
+    var keysToDelete;
+    var user = 'localhost';
+
+    beforeEach(function() {
+        redisClient = redis.createClient(global.environment.redis.port);
+        keysToDelete = {};
+        var redisPool = new RedisPool(global.environment.redis);
+        this.metadataBackend = cartodbRedis({pool: redisPool});
+    });
+
+    afterEach(function(done) {
+        redisClient.quit(function() {
+            testHelper.deleteRedisKeys(keysToDelete, done);
+        });
+    });
+
+    function withAnalysesLimits(limits, callback) {
+        redisClient.SELECT(5, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            var analysesLimitsKey = 'limits:analyses:' + user;
+            redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
+                if (err) {
+                    return callback(err);
+                }
+                keysToDelete[analysesLimitsKey] = 5;
+                return callback();
+            });
+        });
+    }
+
+    it("should use limits from configuration", function(done) {
+        var analysisBackend = new AnalysisBackend(this.metadataBackend, {
+            limits: {
+                moran: { timeout: 5000 },
+                kmeans: { timeout: 5000 }
+            }
+        });
+        analysisBackend.getAnalysesLimits(user, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.ok(result.analyses.moran);
+            assert.equal(result.analyses.moran.timeout, 5000);
+
+            assert.ok(result.analyses.kmeans);
+            assert.equal(result.analyses.kmeans.timeout, 5000);
+
+            done();
+        });
+    });
+
+    it("should use limits from redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend);
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, redis takes priority", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    moran: { timeout: 1000 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    moran: { timeout: 1000 },
+                    kmeans: { timeout: 1000 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                assert.ok(result.analyses.kmeans);
+                assert.equal(result.analyses.kmeans.timeout, 1000);
+
+                done();
+            });
+        });
+    });
+
+    it("should allow to set other limits per analysis via configuration, and keep timeout from redis", function(done) {
+        var self = this;
+        var limits = ['aggregate-intersection', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    'aggregate-intersection': { timeout: 10000, maxNumberOfRows: 1e5 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses['aggregate-intersection']);
+                assert.equal(result.analyses['aggregate-intersection'].timeout, 5000);
+                assert.equal(result.analyses['aggregate-intersection'].maxNumberOfRows, 1e5);
+
+                done();
+            });
+        });
+    });
+
+});

test/support/prepare_db.sh

@@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced'
+  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
   REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
   CURL_ARGS=""

test/support/sql/cdb_analysis_check.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+BEGIN
+END;
+$$ LANGUAGE PLPGSQL;

test/support/sql/windshaft.test.sql

@@ -634,7 +634,8 @@ GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER;
 --
 GRANT SELECT, UPDATE, INSERT, DELETE ON cdb_analysis_catalog TO :TESTUSER;
 
-create schema cdb_crankshaft;
+DROP EXTENSION IF EXISTS crankshaft;
+CREATE SCHEMA IF NOT EXISTS cdb_crankshaft;
 GRANT USAGE ON SCHEMA cdb_crankshaft TO :TESTUSER;
 CREATE TYPE kmeans_type as (cartodb_id numeric, cluster_no numeric);
 CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters integer,no_init integer default 20)

test/support/test-client.js

@@ -307,6 +307,13 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
         url += '?' + qs.stringify(extraParams);
     }
 
+    var expectedResponse = params.response || {
+        status: 200,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+    };
+
     var layergroupId;
     step(
         function createLayergroup() {
@@ -372,12 +379,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
                         host: 'localhost'
                     }
                 },
-                {
-                    status: 200,
-                    headers: {
-                        'Content-Type': 'application/json; charset=utf-8'
-                    }
-                },
+                expectedResponse,
                 function(res, err) {
                     if (err) {
                         return next(err);