Release 2.86.0

Upgrade windshaft

.gitignore

@@ -2,6 +2,7 @@ node_modules*
 config.status*
 config/environments/*.js
 .idea
+.vscode
 tools/munin/windshaft.conf
 logs/
 pids/

NEWS.md

@@ -1,5 +1,129 @@
 # Changelog
 
+## 2.86.0
+Released 2016-12-02
+
+Announcements:
+ - Upgrades windshaft to [2.6.3](https://github.com/CartoDB/Windshaft/releases/tag/2.6.3).
+
+
+## 2.85.1
+Released 2016-11-30
+
+Announcements:
+ - Upgrades camshaft to [0.48.4](https://github.com/CartoDB/camshaft/releases/tag/0.48.4).
+
+
+## 2.85.0
+Released 2016-11-24
+
+New features:
+ - Allow to set resource URL templates with substitution tokens #594.
+
+
+## 2.84.2
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.3](https://github.com/CartoDB/camshaft/releases/tag/0.48.3).
+
+
+## 2.84.1
+Released 2016-11-23
+
+Announcements:
+ - Upgrades camshaft to [0.48.2](https://github.com/CartoDB/camshaft/releases/tag/0.48.2).
+
+
+## 2.84.0
+Released 2016-11-11
+
+New features:
+ - Analyses limit configuration allows to set other limits than timeout.
+
+
+## 2.83.1
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.1](https://github.com/CartoDB/camshaft/releases/tag/0.48.1).
+
+
+## 2.83.0
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.0](https://github.com/CartoDB/camshaft/releases/tag/0.48.0).
+
+
+## 2.82.0
+Released 2016-11-08
+
+Announcements:
+ - Upgrades camshaft to [0.47.0](https://github.com/CartoDB/camshaft/releases/tag/0.47.0).
+
+
+## 2.81.1
+Released 2016-11-05
+
+Announcements:
+ - Upgrades windshaft to [2.6.2](https://github.com/CartoDB/windshaft/releases/tag/2.6.2).
+ - Upgrades camshaft to [0.46.3](https://github.com/CartoDB/camshaft/releases/tag/0.46.3).
+
+
+## 2.81.0
+Released 2016-11-02
+
+Enhancements:
+ - Returns errors with context when query layer does not retrieve geometry column
+
+Announcements:
+ - Upgrades windshaft to [2.6.1](https://github.com/CartoDB/windshaft/releases/tag/2.6.1).
+ - Upgrades camshaft to [0.46.2](https://github.com/CartoDB/camshaft/releases/tag/0.46.2).
+
+
+## 2.80.2
+Released 2016-10-26
+
+Bug fixes:
+ - Fix order in categories query to get ramps
+
+
+## 2.80.1
+Released 2016-10-25
+
+Announcements:
+ - Upgrades camshaft to [0.46.1](https://github.com/CartoDB/camshaft/releases/tag/0.46.1).
+
+
+## 2.80.0
+Released 2016-10-20
+
+Announcements:
+ - Upgrades camshaft to [0.46.0](https://github.com/CartoDB/camshaft/releases/tag/0.46.0).
+
+New features:
+ - Default analyses limits can be defined in configuration.
+
+
+## 2.79.0
+Released 2016-10-11
+
+New features:
+ - Retrieve analysis limits and pass them into camshaft.
+
+Announcements:
+ - Upgrades turbo-carto to [0.18.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.18.0).
+ - Upgrades camshaft to [0.45.0](https://github.com/CartoDB/camshaft/releases/tag/0.45.0).
+
+
+## 2.78.1
+Released 2016-09-30
+
+Announcements:
+ - Upgrades camshaft to [0.44.2](https://github.com/CartoDB/camshaft/releases/tag/0.44.2).
+
+
 ## 2.78.0
 Released 2016-09-29
 

config/environments/development.js.example

@@ -23,6 +23,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -96,6 +111,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -216,6 +235,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/production.js.example

@@ -23,6 +23,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -90,6 +105,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -210,6 +229,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/staging.js.example

@@ -23,6 +23,21 @@ var config = {
     // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -90,6 +105,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -210,6 +229,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

config/environments/test.js.example

@@ -23,6 +23,20 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -90,6 +104,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -211,6 +229,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {

docs/named_maps.md

@@ -73,10 +73,6 @@ The `name` argument defines how to name this "template_name".json. Note that the
       }
     ]
   },
-  "preview_layers": {
-    "0": true,
-    "layer1": false
-  },
   "view": {
     "zoom": 4,
     "center": {
@@ -88,6 +84,10 @@ The `name` argument defines how to name this "template_name".json. Note that the
       "south": -45,
       "east": 45,
       "north": 45
+    },
+    "preview_layers": {
+      "0": true,
+      "layer1": false
     }
   }
 }
@@ -105,7 +105,7 @@ auth |
 |_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.carto.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
 placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
 layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.carto.com/carto-engine/maps-api/mapconfig/) for more information.
-view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box. Also it is possible to choose which layers are visible or not with `preview_layers` indicating its visibility by layer index or id (visible by default).
 --- | ---
 |_ zoom | The zoom level to use
 

docs/static_maps_api.md

@@ -149,7 +149,7 @@ It is important to note that generated images are cached from the live data refe
 * While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
 * Image resolution by default is set to 72 DPI
 * JPEG quality by default is 85%
-* Timeout limits for generating static maps are the same across the CARTO Editor and CARTO Engine. It is important to ensure timely processing of queries.
+* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
 
 ## Examples
 

lib/cartodb/backends/analysis.js

@@ -1,10 +1,20 @@
+'use strict';
+
+var _ = require('underscore');
 var camshaft = require('camshaft');
 var fs = require('fs');
 
-function AnalysisBackend (options) {
-    options = options || {};
-    this.setBatchConfig(options.batch);
-    this.setLoggerConfig(options.logger);
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.options.limits = this.options.limits || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
 }
 
 module.exports = AnalysisBackend;
@@ -18,11 +28,7 @@ AnalysisBackend.prototype.setBatchConfig = function (options) {
 };
 
 AnalysisBackend.prototype.setLoggerConfig = function (options) {
-    var loggerConfig = options || {};
-
-    loggerConfig.filename = loggerConfig.filename;
-
-    this.loggerConfig = loggerConfig;
+    this.loggerConfig = options || {};
 
     if (this.loggerConfig.filename) {
         this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
@@ -46,5 +52,42 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
         stream: this.stream ? this.stream : process.stdout
     };
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+
+    var analysesLimits = {
+        analyses: {
+            // buffer: {
+            //     timeout: 1000,
+            //     maxNumberOfRows: 1e6
+            // }
+        }
+    };
+
+    Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) {
+        analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]);
+    });
+
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        // analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 }
+        analysesTimeouts = analysesTimeouts || {};
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = _.defaults(
+                {
+                    timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+                },
+                analysesLimits.analyses[analysisType]
+            );
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/template_maps.js

@@ -55,11 +55,9 @@ util.inherits(TemplateMaps, EventEmitter);
 module.exports = TemplateMaps;
 
 
-var o = TemplateMaps.prototype;
-
 //--------------- PRIVATE METHODS --------------------------------
 
-o._userTemplateLimit = function() {
+TemplateMaps.prototype._userTemplateLimit = function() {
   return this.opts.max_user_templates || 0;
 };
 
@@ -70,7 +68,7 @@ o._userTemplateLimit = function() {
  * @param redisArgs - the arguments for the redis function in an array
  * @param callback - function to pass results too.
  */
-o._redisCmd = function(redisFunc, redisArgs, callback) {
+TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
   var redisClient;
   var that = this;
   var db = that.db_signatures;
@@ -97,7 +95,7 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
 var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
 // jshint maxcomplexity:15
-o._checkInvalidTemplate = function(template) {
+TemplateMaps.prototype._checkInvalidTemplate = function(template) {
   if ( template.version !== '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
   }
@@ -200,7 +198,7 @@ function templateDefaults(template) {
 // @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
-o.addTemplate = function(owner, template, callback) {
+TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
     var self = this;
 
     template = templateDefaults(template);
@@ -258,7 +256,7 @@ o.addTemplate = function(owner, template, callback) {
 //
 // @param callback function(err)
 //
-o.delTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function deleteTemplate() {
@@ -297,7 +295,8 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //
 // @param callback function(err)
 //
-o.updTemplate = function(owner, tpl_id, template, callback) {
+TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
+    
     var self = this;
 
     template = templateDefaults(template);
@@ -356,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
 //
-o.listTemplates = function(owner, callback) {
+TemplateMaps.prototype.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
 
@@ -370,7 +369,7 @@ o.listTemplates = function(owner, callback) {
 // @param callback function(err, template)
 //        Return full template definition
 //
-o.getTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function getTemplate() {
@@ -386,7 +385,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
     );
 };
 
-o.isAuthorized = function(template, authTokens) {
+TemplateMaps.prototype.isAuthorized = function(template, authTokens) {
     if (!template) {
         return false;
     }
@@ -438,7 +437,7 @@ function _replaceVars (str, params) {
     });
     return str;
 }
-o.instance = function(template, params) {
+TemplateMaps.prototype.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
   Object.keys(phold).forEach(function(k) {
@@ -500,7 +499,7 @@ o.instance = function(template, params) {
 };
 
 // Return a fingerPrint of the object
-o.fingerPrint = function(template) {
+TemplateMaps.prototype.fingerPrint = function(template) {
   return crypto.createHash('md5')
     .update(JSON.stringify(template))
     .digest('hex')

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -32,7 +32,7 @@ methodTemplates.category = dot.template([
     '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
     '  FROM ({{=it._sql}}) _cdb_aggregation_all',
     '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
+    '  ORDER BY 2 DESC, 1 ASC',
     '),',
     'agg_categories AS (',
     '  SELECT category',

lib/cartodb/controllers/map.js

@@ -1,4 +1,6 @@
 var _ = require('underscore');
+var dot = require('dot');
+dot.templateSettings.strip = false;
 var assert = require('assert');
 var step = require('step');
 var windshaft = require('windshaft');
@@ -44,6 +46,20 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.mapConfigAdapter = mapConfigAdapter;
+
+    this.resourcesUrlTemplates = null;
+    if (global.environment.resources_url_templates) {
+        var templates = global.environment.resources_url_templates;
+
+        if (templates.http) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
+        }
+        if (templates.https) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
+        }
+    }
 }
 
 util.inherits(MapController, BaseController);
@@ -166,11 +182,29 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function finish(err, layergroup) {
             if (err) {
+                if (Number.isFinite(err.layerIndex)) {
+                    var error = new Error(err.message);
+                    error.http_status = err.http_status;
+
+                    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
+                        error.http_status = 400;
+                    }
+
+                    error.type = 'layer';
+                    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
+                    error.layer = {
+                        id: mapConfig.getLayerId(err.layerIndex),
+                        index: err.layerIndex,
+                        type: mapConfig.layerType(err.layerIndex)
+                    };
+
+                    err = error;
+                }
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
                 var analysesResults = context.analysesResults || [];
-                addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
-                addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                self.addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
+                self.addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
                 addContextMetadata(layergroup, mapConfig.obj(), context);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
@@ -242,8 +276,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
                 var _mapConfig = mapConfig.obj();
-                addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
-                addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+                self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
+                self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
                 addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context);
 
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
@@ -352,7 +386,7 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
     }, lastUpdateTime);
 }
 
-function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) {
+MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
     includeQuery = includeQuery || false;
     analysesResults = analysesResults || [];
     layergroup.metadata.analyses = [];
@@ -365,7 +399,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                     var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
                     var nodeRepr = {
                         status: node.getStatus(),
-                        url: getUrls(username, nodeResource)
+                        url: this.getUrls(username, nodeResource)
                     };
                     if (includeQuery) {
                         nodeRepr.query = node.getQuery();
@@ -377,30 +411,30 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
                 }
 
                 return nodesIdMap;
-            }, {})
+            }.bind(this), {})
         });
-    });
-}
+    }.bind(this));
+};
 
 // TODO this should take into account several URL patterns
-function addDataviewsAndWidgetsUrls(username, layergroup, mapConfig) {
-    addDataviewsUrls(username, layergroup, mapConfig);
-    addWidgetsUrl(username, layergroup, mapConfig);
-}
+MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) {
+    this.addDataviewsUrls(username, layergroup, mapConfig);
+    this.addWidgetsUrl(username, layergroup, mapConfig);
+};
 
-function addDataviewsUrls(username, layergroup, mapConfig) {
+MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) {
     layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
     var dataviews = mapConfig.dataviews || {};
 
     Object.keys(dataviews).forEach(function(dataviewName) {
         var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
         layergroup.metadata.dataviews[dataviewName] = {
-            url: getUrls(username, resource)
+            url: this.getUrls(username, resource)
         };
-    });
-}
+    }.bind(this));
+};
 
-function addWidgetsUrl(username, layergroup, mapConfig) {
+MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) {
     if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
         layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
             var mapConfigLayer = mapConfig.layers[layerIndex];
@@ -410,16 +444,19 @@ function addWidgetsUrl(username, layergroup, mapConfig) {
                     var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
                     layer.widgets[widgetName] = {
                         type: mapConfigLayer.options.widgets[widgetName].type,
-                        url: getUrls(username, resource)
+                        url: this.getUrls(username, resource)
                     };
-                });
+                }.bind(this));
             }
             return layer;
-        });
+        }.bind(this));
     }
-}
+};
 
-function getUrls(username, resource) {
+MapController.prototype.getUrls = function(username, resource) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, resource);
+    }
     var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
     if (cdnUrl) {
         return {
@@ -432,4 +469,29 @@ function getUrls(username, resource) {
             http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
         };
     }
-}
+};
+
+MapController.prototype.getUrlsFromTemplate = function(username, resource) {
+    var urls = {};
+    var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url || {};
+
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = this.resourcesUrlTemplates.http({
+            cdn_url: cdnUrl.http,
+            user: username,
+            port: global.environment.port,
+            resource: resource
+        });
+    }
+
+    if (this.resourcesUrlTemplates.https) {
+        urls.https = this.resourcesUrlTemplates.https({
+            cdn_url: cdnUrl.https,
+            user: username,
+            port: global.environment.port,
+            resource: resource
+        });
+    }
+
+    return urls;
+};

lib/cartodb/server.js

@@ -148,7 +148,7 @@ module.exports = function(serverOptions) {
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
 
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

lib/cartodb/server_options.js

@@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
     },
     logger: {
         filename: undefined
-    }
+    },
+    limits: {}
 });
 
 module.exports = {
@@ -69,6 +70,7 @@ module.exports = {
         },
         datasource: global.environment.postgres,
         cachedir: global.environment.millstone.cache_basedir,
+        use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
         mapnik_version: global.environment.mapnik_version,
         mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
         default_layergroup_ttl: global.environment.mapConfigTTL || 7200
@@ -101,7 +103,8 @@ module.exports = {
         },
         logger: {
             filename: analysisConfig.logger.filename
-        }
+        },
+        limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.78.0",
+  "version": "2.86.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.44.1",
+    "camshaft": "0.48.4",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "0.13.1",
@@ -37,9 +37,9 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.17.1",
+    "turbo-carto": "0.18.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.5.0",
+    "windshaft": "2.6.3",
     "yargs": "~5.0.0"
   },
   "devDependencies": {

test/acceptance/errors-with-context.js

@@ -0,0 +1,68 @@
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+describe('error with context', function () {
+    var layerOK = {
+        options: {
+            sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator ' +
+                 'from test_table',
+            cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.1',
+            interactivity: 'cartodb_id'
+        }
+    };
+    var layerKO = {
+        options: {
+            sql: 'select cartodb_id from test_table offset 3', // it doesn't return the_geom_webmercator so it must fail
+            cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.2',
+            interactivity: 'cartodb_id'
+        }
+    };
+
+    var DB_ERROR_MESSAGE = 'Postgis Plugin: ERROR:  column "the_geom_webmercator" does not exist';
+    var scenarios = [{
+        description: 'layergroup with 2 layers, second one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerOK, layerKO]
+        },
+        expectedFailingLayer: { id: 'layer1', index: 1, type: 'mapnik' }
+    }, {
+        description: 'layergroup with 2 layers, first one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerKO, layerOK]
+        },
+        expectedFailingLayer: { id: 'layer0', index: 0, type: 'mapnik' }
+    }];
+
+    scenarios.forEach(function (scenario) {
+        it(scenario.description, function (done) {
+            assert.response(server, {
+                url: '/api/v1/map',
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(scenario.layergroup)
+            }, {
+                status: 400
+            }, function (res) {
+                var parsedBody = JSON.parse(res.body);
+
+                assert.ok(Array.isArray(parsedBody.errors_with_context));
+
+                var err = parsedBody.errors_with_context[0];
+                assert.equal(err.type, 'layer');
+                assert.equal(err.subtype, 'query');
+                assert.ok(err.message.indexOf(DB_ERROR_MESSAGE) >= 0);
+                assert.deepEqual(err.layer, scenario.expectedFailingLayer);
+                done();
+            });
+        });
+    });
+});

test/integration/analysis-backend-limits.js

@@ -0,0 +1,167 @@
+var testHelper = require('../support/test_helper');
+
+var assert = require('assert');
+var redis = require('redis');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
+
+describe('analysis-backend limits', function() {
+
+    var redisClient;
+    var keysToDelete;
+    var user = 'localhost';
+
+    beforeEach(function() {
+        redisClient = redis.createClient(global.environment.redis.port);
+        keysToDelete = {};
+        var redisPool = new RedisPool(global.environment.redis);
+        this.metadataBackend = cartodbRedis({pool: redisPool});
+    });
+
+    afterEach(function(done) {
+        redisClient.quit(function() {
+            testHelper.deleteRedisKeys(keysToDelete, done);
+        });
+    });
+
+    function withAnalysesLimits(limits, callback) {
+        redisClient.SELECT(5, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            var analysesLimitsKey = 'limits:analyses:' + user;
+            redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
+                if (err) {
+                    return callback(err);
+                }
+                keysToDelete[analysesLimitsKey] = 5;
+                return callback();
+            });
+        });
+    }
+
+    it("should use limits from configuration", function(done) {
+        var analysisBackend = new AnalysisBackend(this.metadataBackend, {
+            limits: {
+                moran: { timeout: 5000 },
+                kmeans: { timeout: 5000 }
+            }
+        });
+        analysisBackend.getAnalysesLimits(user, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.ok(result.analyses.moran);
+            assert.equal(result.analyses.moran.timeout, 5000);
+
+            assert.ok(result.analyses.kmeans);
+            assert.equal(result.analyses.kmeans.timeout, 5000);
+
+            done();
+        });
+    });
+
+    it("should use limits from redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend);
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, redis takes priority", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    moran: { timeout: 1000 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    moran: { timeout: 1000 },
+                    kmeans: { timeout: 1000 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                assert.ok(result.analyses.kmeans);
+                assert.equal(result.analyses.kmeans.timeout, 1000);
+
+                done();
+            });
+        });
+    });
+
+    it("should allow to set other limits per analysis via configuration, and keep timeout from redis", function(done) {
+        var self = this;
+        var limits = ['aggregate-intersection', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, {
+                limits: {
+                    'aggregate-intersection': { timeout: 10000, maxNumberOfRows: 1e5 }
+                }
+            });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses['aggregate-intersection']);
+                assert.equal(result.analyses['aggregate-intersection'].timeout, 5000);
+                assert.equal(result.analyses['aggregate-intersection'].maxNumberOfRows, 1e5);
+
+                done();
+            });
+        });
+    });
+
+});

test/support/prepare_db.sh

@@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced'
+  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
   REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
   CURL_ARGS=""

test/support/sql/cdb_analysis_check.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+BEGIN
+END;
+$$ LANGUAGE PLPGSQL;

test/support/sql/windshaft.test.sql

@@ -634,7 +634,8 @@ GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER;
 --
 GRANT SELECT, UPDATE, INSERT, DELETE ON cdb_analysis_catalog TO :TESTUSER;
 
-create schema cdb_crankshaft;
+DROP EXTENSION IF EXISTS crankshaft;
+CREATE SCHEMA IF NOT EXISTS cdb_crankshaft;
 GRANT USAGE ON SCHEMA cdb_crankshaft TO :TESTUSER;
 CREATE TYPE kmeans_type as (cartodb_id numeric, cluster_no numeric);
 CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters integer,no_init integer default 20)