Release 2.80.0

Upgrade camshaft to 0.46.0

NEWS.md

@@ -1,5 +1,51 @@
 # Changelog
 
+## 2.80.0
+Released 2016-10-20
+
+Announcements:
+ - Upgrades camshaft to [0.46.0](https://github.com/CartoDB/camshaft/releases/tag/0.46.0).
+
+New features:
+ - Default analyses limits can be defined in configuration.
+
+
+## 2.79.0
+Released 2016-10-11
+
+New features:
+ - Retrieve analysis limits and pass them into camshaft.
+
+Announcements:
+ - Upgrades turbo-carto to [0.18.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.18.0).
+ - Upgrades camshaft to [0.45.0](https://github.com/CartoDB/camshaft/releases/tag/0.45.0).
+
+
+## 2.78.1
+Released 2016-09-30
+
+Announcements:
+ - Upgrades camshaft to [0.44.2](https://github.com/CartoDB/camshaft/releases/tag/0.44.2).
+
+
+## 2.78.0
+Released 2016-09-29
+
+New features:
+ - Add metadata about processed turbo-carto rules.
+
+Announcements:
+ - Upgrades turbo-carto to [0.17.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.17.1).
+
+
+## 2.77.1
+
+Released 2016-09-28
+
+Announcements:
+ - Upgrades camshaft to [0.44.1](https://github.com/CartoDB/camshaft/releases/tag/0.44.1).
+
+
 ## 2.77.0
 
 Released 2016-09-26

config/environments/development.js.example

@@ -216,6 +216,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

config/environments/production.js.example

@@ -210,6 +210,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

config/environments/staging.js.example

@@ -210,6 +210,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

config/environments/test.js.example

@@ -211,6 +211,12 @@ var config = {
             // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
             // Log file will be re-opened on receiving the HUP signal
             filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: 120000,
+            cpu2x: 60000
         }
     }
     ,millstone: {

lib/cartodb/backends/analysis.js

@@ -1,10 +1,19 @@
+'use strict';
+
+var _ = require('underscore');
 var camshaft = require('camshaft');
 var fs = require('fs');
 
-function AnalysisBackend (options) {
-    options = options || {};
-    this.setBatchConfig(options.batch);
-    this.setLoggerConfig(options.logger);
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
 }
 
 module.exports = AnalysisBackend;
@@ -18,11 +27,7 @@ AnalysisBackend.prototype.setBatchConfig = function (options) {
 };
 
 AnalysisBackend.prototype.setLoggerConfig = function (options) {
-    var loggerConfig = options || {};
-
-    loggerConfig.filename = loggerConfig.filename;
-
-    this.loggerConfig = loggerConfig;
+    this.loggerConfig = options || {};
 
     if (this.loggerConfig.filename) {
         this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
@@ -46,5 +51,34 @@ AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefin
         stream: this.stream ? this.stream : process.stdout
     };
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        analysesTimeouts = analysesTimeouts || {};
+
+        _.defaults(analysesTimeouts, self.options.limits);
+
+        var analysesLimits = {
+            analyses: {
+                // buffer: {
+                //     timeout: 1000
+                // }
+            }
+        };
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = {
+                timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+            };
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -6,6 +6,9 @@ dot.templateSettings.strip = false;
 function createTemplate(method) {
     return dot.template([
         'SELECT',
+        'min({{=it._column}}) min_val,',
+        'max({{=it._column}}) max_val,',
+        'avg({{=it._column}}) avg_val,',
         method,
         'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
     ].join('\n'));
@@ -74,11 +77,14 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             return callback(err);
         }
 
-        resultSet = resultSet || {};
-        var result = resultSet.rows || [];
-
+        var result = getResult(resultSet);
         var strategy = method2strategy[methodName];
-        var ramp = result[0][methodName] || [];
+        var ramp = result[methodName] || [];
+        var stats = {
+            min_val: result.min_val,
+            max_val: result.max_val,
+            avg_val: result.avg_val
+        };
         // Skip null values from ramp
         // Generated turbo-carto won't be correct, but better to keep it working than failing
         // TODO fix cartodb-postgres extension quantification functions
@@ -89,8 +95,16 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             });
         }
 
-        return callback(null, { ramp: ramp, strategy: strategy });
+        return callback(null, { ramp: ramp, strategy: strategy, stats: stats });
     }, true); // use read-only transaction
 };
 
+function getResult(resultSet) {
+    resultSet = resultSet || {};
+    var result = resultSet.rows || [];
+    result = result[0] || {};
+
+    return result;
+}
+
 module.exports = PostgresDatasource;

lib/cartodb/controllers/map.js

@@ -171,6 +171,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
                 var analysesResults = context.analysesResults || [];
                 addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
                 addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                addContextMetadata(layergroup, mapConfig.obj(), context);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -178,6 +179,17 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
     );
 };
 
+function addContextMetadata(layergroup, mapConfig, context) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
+                layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
+            }
+            return layer;
+        });
+    }
+}
+
 MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
     var self = this;
 
@@ -229,8 +241,10 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
-                addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj());
+                var _mapConfig = mapConfig.obj();
+                addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
                 addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+                addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context);
 
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -48,6 +48,11 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
         }
 
         requestMapConfig.layers = results.map(function(result) { return result.layer; });
+        context.turboCarto = {
+            layers: results.map(function(result) {
+                return result.meta;
+            })
+        };
 
         return callback(null, requestMapConfig);
     });
@@ -84,7 +89,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
     }
 
     var pg = new PSQL(dbParamsFromReqParams(params));
-    function processCallback(err, cartocss) {
+    function processCallback(err, cartocss, meta) {
         // Only return turbo-carto errors
         if (err && err.name === 'TurboCartoError') {
             var error = new Error(err.message);
@@ -105,7 +110,7 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
         if (cartocss) {
             layer.options.cartocss = cartocss;
         }
-        return callback(null, { layer: layer });
+        return callback(null, { layer: layer, meta: meta });
     }
 
     var layerSql = layer.options.sql;

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -120,6 +120,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
             self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
             self.analysesResults = context.analysesResults || [];
             self.rendererParams = rendererParams;
+            self.context = context;
             self.context.limits = renderLimits || {};
             return callback(self.err, self.mapConfig, self.rendererParams, self.context);
         }

lib/cartodb/server.js

@@ -148,7 +148,7 @@ module.exports = function(serverOptions) {
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
 
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

lib/cartodb/server_options.js

@@ -39,7 +39,8 @@ var analysisConfig = _.defaults(global.environment.analysis || {}, {
     },
     logger: {
         filename: undefined
-    }
+    },
+    limits: {}
 });
 
 module.exports = {
@@ -101,7 +102,8 @@ module.exports = {
         },
         logger: {
             filename: analysisConfig.logger.filename
-        }
+        },
+        limits: analysisConfig.limits
     },
     // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.77.0",
+  "version": "2.80.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.44.0",
+    "camshaft": "0.46.0",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "0.13.1",
@@ -37,7 +37,7 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.16.0",
+    "turbo-carto": "0.18.0",
     "underscore": "~1.6.0",
     "windshaft": "2.5.0",
     "yargs": "~5.0.0"

test/integration/analysis-backend-limits.js

@@ -0,0 +1,127 @@
+var testHelper = require('../support/test_helper');
+
+var assert = require('assert');
+var redis = require('redis');
+
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+
+var AnalysisBackend = require('../../lib/cartodb/backends/analysis');
+
+describe('analysis-backend limits', function() {
+
+    var redisClient;
+    var keysToDelete;
+    var user = 'localhost';
+
+    beforeEach(function() {
+        redisClient = redis.createClient(global.environment.redis.port);
+        keysToDelete = {};
+        var redisPool = new RedisPool(global.environment.redis);
+        this.metadataBackend = cartodbRedis({pool: redisPool});
+    });
+
+    afterEach(function(done) {
+        redisClient.quit(function() {
+            testHelper.deleteRedisKeys(keysToDelete, done);
+        });
+    });
+
+    function withAnalysesLimits(limits, callback) {
+        redisClient.SELECT(5, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            var analysesLimitsKey = 'limits:analyses:' + user;
+            redisClient.HMSET([analysesLimitsKey].concat(limits), function(err) {
+                if (err) {
+                    return callback(err);
+                }
+                keysToDelete[analysesLimitsKey] = 5;
+                return callback();
+            });
+        });
+    }
+
+    it("should use limits from configuration", function(done) {
+        var analysisBackend = new AnalysisBackend(this.metadataBackend, { limits: { moran: 5000, kmeans: 5000 } });
+        analysisBackend.getAnalysesLimits(user, function(err, result) {
+            assert.ok(!err, err);
+
+            assert.ok(result.analyses.moran);
+            assert.equal(result.analyses.moran.timeout, 5000);
+
+            assert.ok(result.analyses.kmeans);
+            assert.equal(result.analyses.kmeans.timeout, 5000);
+
+            done();
+        });
+    });
+
+    it("should use limits from redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend);
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, redis takes priority", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000 } });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                done();
+            });
+        });
+    });
+
+    it("should use limits from redis and configuration, defaulting for values not present in redis", function(done) {
+        var self = this;
+        var limits = ['moran', 5000];
+
+        withAnalysesLimits(limits, function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            var analysisBackend = new AnalysisBackend(self.metadataBackend, { limits: { moran: 1000, kmeans: 1000 } });
+            analysisBackend.getAnalysesLimits(user, function(err, result) {
+                assert.ok(!err, err);
+
+                assert.ok(result.analyses.moran);
+                assert.equal(result.analyses.moran.timeout, 5000);
+
+                assert.ok(result.analyses.kmeans);
+                assert.equal(result.analyses.kmeans.timeout, 1000);
+
+                done();
+            });
+        });
+    });
+
+});

test/support/prepare_db.sh

@@ -75,7 +75,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced'
+  LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check'
   REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
   CURL_ARGS=""

test/support/sql/cdb_analysis_check.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+BEGIN
+END;
+$$ LANGUAGE PLPGSQL;