Merge pull request #589 from CartoDB/upgrade-camshaft-to-0.48.1

Release 2.83.1

NEWS.md

@@ -1,5 +1,114 @@
 # Changelog
 
+## 2.83.1
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.1](https://github.com/CartoDB/camshaft/releases/tag/0.48.1).
+
+
+## 2.83.0
+Released 2016-11-10
+
+Announcements:
+ - Upgrades camshaft to [0.48.0](https://github.com/CartoDB/camshaft/releases/tag/0.48.0).
+
+
+## 2.82.0
+Released 2016-11-08
+
+Announcements:
+ - Upgrades camshaft to [0.47.0](https://github.com/CartoDB/camshaft/releases/tag/0.47.0).
+
+
+## 2.81.1
+Released 2016-11-05
+
+Announcements:
+ - Upgrades windshaft to [2.6.2](https://github.com/CartoDB/windshaft/releases/tag/2.6.2).
+ - Upgrades camshaft to [0.46.3](https://github.com/CartoDB/camshaft/releases/tag/0.46.3).
+
+
+## 2.81.0
+Released 2016-11-02
+
+Enhancements:
+ - Returns errors with context when query layer does not retrieve geometry column
+
+Announcements:
+ - Upgrades windshaft to [2.6.1](https://github.com/CartoDB/windshaft/releases/tag/2.6.1).
+ - Upgrades camshaft to [0.46.2](https://github.com/CartoDB/camshaft/releases/tag/0.46.2).
+
+
+## 2.80.2
+Released 2016-10-26
+
+Bug fixes:
+ - Fix order in categories query to get ramps
+
+
+## 2.80.1
+Released 2016-10-25
+
+Announcements:
+ - Upgrades camshaft to [0.46.1](https://github.com/CartoDB/camshaft/releases/tag/0.46.1).
+
+
+## 2.80.0
+Released 2016-10-20
+
+Announcements:
+ - Upgrades camshaft to [0.46.0](https://github.com/CartoDB/camshaft/releases/tag/0.46.0).
+
+New features:
+ - Default analyses limits can be defined in configuration.
+
+
+## 2.79.0
+Released 2016-10-11
+
+New features:
+ - Retrieve analysis limits and pass them into camshaft.
+
+Announcements:
+ - Upgrades turbo-carto to [0.18.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.18.0).
+ - Upgrades camshaft to [0.45.0](https://github.com/CartoDB/camshaft/releases/tag/0.45.0).
+
+
+## 2.78.1
+Released 2016-09-30
+
+Announcements:
+ - Upgrades camshaft to [0.44.2](https://github.com/CartoDB/camshaft/releases/tag/0.44.2).
+
+
+## 2.78.0
+Released 2016-09-29
+
+New features:
+ - Add metadata about processed turbo-carto rules.
+
+Announcements:
+ - Upgrades turbo-carto to [0.17.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.17.1).
+
+
+## 2.77.1
+
+Released 2016-09-28
+
+Announcements:
+ - Upgrades camshaft to [0.44.1](https://github.com/CartoDB/camshaft/releases/tag/0.44.1).
+
+
+## 2.77.0
+
+Released 2016-09-26
+
+Announcements:
+ - Upgrades camshaft to [0.44.0](https://github.com/CartoDB/camshaft/releases/tag/0.44.0).
+ - Adds a new configuration for camshaft: logger stream.
+
+
 ## 2.76.0
 
 Released 2016-09-15

config/environments/development.js.example

@@ -210,6 +210,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

config/environments/production.js.example

@@ -204,6 +204,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

config/environments/staging.js.example

@@ -204,6 +204,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

config/environments/test.js.example

@@ -205,6 +205,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

lib/cartodb/backends/analysis.js

@@ -1,19 +1,84 @@
-var camshaft = require('camshaft');
+'use strict';
 
-function AnalysisBackend(options) {
-    var batchConfig = options.batch || {};
+var _ = require('underscore');
+var camshaft = require('camshaft');
+var fs = require('fs');
+
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
+}
+
+module.exports = AnalysisBackend;
+
+AnalysisBackend.prototype.setBatchConfig = function (options) {
+    var batchConfig = options || {};
     batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job';
     batchConfig.inlineExecution = batchConfig.inlineExecution || false;
     batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan';
     this.batchConfig = batchConfig;
-}
+};
 
-module.exports = AnalysisBackend;
+AnalysisBackend.prototype.setLoggerConfig = function (options) {
+    this.loggerConfig = options || {};
+
+    if (this.loggerConfig.filename) {
+        this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+
+        process.on('SIGHUP', function () {
+            if (this.stream) {
+                this.stream.destroy();
+            }
+
+            this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+        }.bind(this));
+    }
+};
 
 AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) {
     analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
     analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
     analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    analysisConfiguration.logger = {
+        stream: this.stream ? this.stream : process.stdout
+    };
+
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        analysesTimeouts = analysesTimeouts || {};
+
+        _.defaults(analysesTimeouts, self.options.limits);
+
+        var analysesLimits = {
+            analyses: {
+                // buffer: {
+                //     timeout: 1000
+                // }
+            }
+        };
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = {
+                timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+            };
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -6,6 +6,9 @@ dot.templateSettings.strip = false;
 function createTemplate(method) {
     return dot.template([
         'SELECT',
+        'min({{=it._column}}) min_val,',
+        'max({{=it._column}}) max_val,',
+        'avg({{=it._column}}) avg_val,',
         method,
         'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
     ].join('\n'));
@@ -29,7 +32,7 @@ methodTemplates.category = dot.template([
     '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
     '  FROM ({{=it._sql}}) _cdb_aggregation_all',
     '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
+    '  ORDER BY 2 DESC, 1 ASC',
     '),',
     'agg_categories AS (',
     '  SELECT category',
@@ -74,11 +77,14 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             return callback(err);
         }
 
-        resultSet = resultSet || {};
-        var result = resultSet.rows || [];
-
+        var result = getResult(resultSet);
         var strategy = method2strategy[methodName];
-        var ramp = result[0][methodName] || [];
+        var ramp = result[methodName] || [];
+        var stats = {
+            min_val: result.min_val,
+            max_val: result.max_val,
+            avg_val: result.avg_val
+        };
         // Skip null values from ramp
         // Generated turbo-carto won't be correct, but better to keep it working than failing
         // TODO fix cartodb-postgres extension quantification functions
@@ -89,8 +95,16 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             });
         }
 
-        return callback(null, { ramp: ramp, strategy: strategy });
+        return callback(null, { ramp: ramp, strategy: strategy, stats: stats });
     }, true); // use read-only transaction
 };
 
+function getResult(resultSet) {
+    resultSet = resultSet || {};
+    var result = resultSet.rows || [];
+    result = result[0] || {};
+
+    return result;
+}
+
 module.exports = PostgresDatasource;

lib/cartodb/controllers/map.js

@@ -166,11 +166,30 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function finish(err, layergroup) {
             if (err) {
+                if (Number.isFinite(err.layerIndex)) {
+                    var error = new Error(err.message);
+                    error.http_status = err.http_status;
+
+                    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
+                        error.http_status = 400;
+                    }
+
+                    error.type = 'layer';
+                    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
+                    error.layer = {
+                        id: mapConfig.getLayerId(err.layerIndex),
+                        index: err.layerIndex,
+                        type: mapConfig.layerType(err.layerIndex)
+                    };
+
+                    err = error;
+                }
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
                 var analysesResults = context.analysesResults || [];
                 addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
                 addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                addContextMetadata(layergroup, mapConfig.obj(), context);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -178,6 +197,17 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
     );
 };
 
+function addContextMetadata(layergroup, mapConfig, context) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
+                layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
+            }
+            return layer;
+        });
+    }
+}
+
 MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
     var self = this;
 
@@ -229,8 +259,10 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
-                addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj());
+                var _mapConfig = mapConfig.obj();
+                addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
                 addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+                addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context);
 
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -48,6 +48,11 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
         }
 
         requestMapConfig.layers = results.map(function(result) { return result.layer; });
+        context.turboCarto = {
+            layers: results.map(function(result) {
+                return result.meta;
+            })
+        };
 
         return callback(null, requestMapConfig);
     });
@@ -84,7 +89,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
     }
 
     var pg = new PSQL(dbParamsFromReqParams(params));
-    function processCallback(err, cartocss) {
+    function processCallback(err, cartocss, meta) {
         // Only return turbo-carto errors
         if (err && err.name === 'TurboCartoError') {
             var error = new Error(err.message);
@@ -105,7 +110,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
         if (cartocss) {
             layer.options.cartocss = cartocss;
         }
-        return callback(null, { layer: layer });
+        return callback(null, { layer: layer, meta: meta });
     }
 
     var layerSql = layer.options.sql;

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -120,6 +120,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
             self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
             self.analysesResults = context.analysesResults || [];
             self.rendererParams = rendererParams;
+            self.context = context;
             self.context.limits = renderLimits || {};
             return callback(self.err, self.mapConfig, self.rendererParams, self.context);
         }

lib/cartodb/server.js

@@ -147,7 +147,8 @@ module.exports = function(serverOptions) {
     var tileBackend = new windshaft.backend.Tile(rendererCache);
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

lib/cartodb/server_options.js

@@ -36,7 +36,11 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
         inlineExecution: false,
         endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
         hostHeaderTemplate: '{{=it.username}}.localhost.lan'
-    }
+    },
+    logger: {
+        filename: undefined
+    },
+    limits: {}
 });
 
 module.exports = {
@@ -95,7 +99,11 @@ module.exports = {
             inlineExecution: analysisConfig.batch.inlineExecution,
             endpoint: analysisConfig.batch.endpoint,
             hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate
-        }
+        },
+        logger: {
+            filename: analysisConfig.logger.filename
+        },
+        limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.76.0",
+  "version": "2.83.1",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.43.0",
+    "camshaft": "0.48.1",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "0.13.1",
@@ -37,9 +37,9 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.16.0",
+    "turbo-carto": "0.18.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.5.0",
+    "windshaft": "2.6.2",
     "yargs": "~5.0.0"
   },
   "devDependencies": {

test/acceptance/errors-with-context.js

@@ -0,0 +1,68 @@
+var assert = require('../support/assert');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+
+describe('error with context', function () {
+    var layerOK = {
+        options: {
+            sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator ' +
+                 'from test_table',
+            cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.1',
+            interactivity: 'cartodb_id'
+        }
+    };
+    var layerKO = {
+        options: {
+            sql: 'select cartodb_id from test_table offset 3', // it doesn't return the_geom_webmercator so it must fail
+            cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
+            cartocss_version: '2.0.2',
+            interactivity: 'cartodb_id'
+        }
+    };
+
+    var DB_ERROR_MESSAGE = 'Postgis Plugin: ERROR:  column "the_geom_webmercator" does not exist';
+    var scenarios = [{
+        description: 'layergroup with 2 layers, second one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerOK, layerKO]
+        },
+        expectedFailingLayer: { id: 'layer1', index: 1, type: 'mapnik' }
+    }, {
+        description: 'layergroup with 2 layers, first one has query error',
+        layergroup: {
+            version: '1.0.0',
+            layers: [layerKO, layerOK]
+        },
+        expectedFailingLayer: { id: 'layer0', index: 0, type: 'mapnik' }
+    }];
+
+    scenarios.forEach(function (scenario) {
+        it(scenario.description, function (done) {
+            assert.response(server, {
+                url: '/api/v1/map',
+                method: 'POST',
+                headers: {
+                    host: 'localhost',
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(scenario.layergroup)
+            }, {
+                status: 400
+            }, function (res) {
+                var parsedBody = JSON.parse(res.body);
+
+                assert.ok(Array.isArray(parsedBody.errors_with_context));
+
+                var err = parsedBody.errors_with_context[0];
+                assert.equal(err.type, 'layer');
+                assert.equal(err.subtype, 'query');
+                assert.ok(err.message.indexOf(DB_ERROR_MESSAGE) >= 0);
+                assert.deepEqual(err.layer, scenario.expectedFailingLayer);
+                done();
+            });
+        });
+    });
+});

test/integration/analysis-backend-limits.js

@@ -0,0 +1,127 @@
+var testHelper = require('../support/test_helper');
+
+var assert = require('assert');
+var redis = require('redis');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
+
+describe('analysis-backend limits', function() {
+
+    var redisClient;
+    var keysToDelete;
+    var user = 'localhost';
+
+    beforeEach(function() {
+        redisClient = redis.createClient(global.environment.redis.port);
+        keysToDelete = {};
+        var redisPool = new RedisPool(global.environment.redis);
+        this.metadataBackend = cartodbRedis({pool: redisPool});
+    });
+
+    afterEach(function(done) {
+        redisClient.quit(function() {
+            testHelper.deleteRedisKeys(keysToDelete, done);
+        });
+    });
+
+    function withAnalysesLimits(limits, callback) {
+        redisClient.SELECT(5, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            var analysesLimitsKey = 'limits:analyses:' + user;
+            redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
+                if (err) {
+                    return callback(err);
+                }
+                keysToDelete[analysesLimitsKey] = 5;
+                return callback();
+            });
+        });
+    }
+
+    it("should use limits from configuration", function(done) {
+        var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } });
+        analysisBackend.getAnalysesLimits(user, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.ok(result.analyses.moran);
+            assert.equal(result.analyses.moran.timeout, 5000);
+
+            assert.ok(result.analyses.kmeans);
+            assert.equal(result.analyses.kmeans.timeout, 5000);
+
+            done();
+        });
+    });
+
+    it("should use limits from redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend);
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, redis takes priority", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                assert.ok(result.analyses.kmeans);
+                assert.equal(result.analyses.kmeans.timeout, 1000);
+
+                done();
+            });
+        });
+    });
+
+});

test/support/prepare_db.sh

@@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced'
+  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
   REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
   CURL_ARGS=""

test/support/sql/cdb_analysis_check.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+BEGIN
+END;
+$$ LANGUAGE PLPGSQL;

test/support/sql/windshaft.test.sql

@@ -634,7 +634,8 @@ GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER;
 --
 GRANT SELECT, UPDATE, INSERT, DELETE ON cdb_analysis_catalog TO :TESTUSER;
 
-create schema cdb_crankshaft;
+DROP EXTENSION IF EXISTS crankshaft;
+CREATE SCHEMA IF NOT EXISTS cdb_crankshaft;
 GRANT USAGE ON SCHEMA cdb_crankshaft TO :TESTUSER;
 CREATE TYPE kmeans_type as (cartodb_id numeric, cluster_no numeric);
 CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters integer,no_init integer default 20)