Release 2.89.0

Upgrade windshaft to 2.8.0

.gitignore

@@ -3,6 +3,7 @@ config.status*
 config/environments/*.js
 .idea
 .vscode
+.nvmrc
 tools/munin/windshaft.conf
 logs/
 pids/

HOWTO_RELEASE

@@ -1,11 +1,10 @@
 1. Test (make clean all check), fix if broken before proceeding
 2. Ensure proper version in package.json
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
+4. Recreate npm-shrinkwrap.json with: `make shrinkwrap`
 5. Commit package.json, npm-shrinwrap.json, NEWS
 6. git tag -a Major.Minor.Patch # use NEWS section as content
-7. Announce on cartodb@googlegroups.com
-8. Stub NEWS/package for next version
+7. Stub NEWS/package for next version
 
 Versions:
 

Makefile

@@ -7,7 +7,13 @@ all:
 	@$(SHELL) ./scripts/install.sh
 
 clean:
-	rm -rf node_modules/*
+	rm -rf node_modules/
+
+shrinkwrap: clean
+	rm npm-shrinkwrap.json
+	npm install --no-shrinkwrap --production
+	npm prune
+	npm shrinkwrap
 
 distclean: clean
 	rm config.status*

NEWS.md

@@ -1,5 +1,53 @@
 # Changelog
 
+## 2.89.0
+Released 2017-03-17
+
+**Deprecation warning**: v2.89.0 is the last release that supports Node v0.10.x. Next mayor release will support Node v6.9.x and further versions.
+
+Announcements:
+ - Upgrades windshaft to [2.8.0](https://github.com/CartoDB/windshaft/releases/tag/2.8.0).
+
+Bug fixes:
+ - Histogram column type discovery query uses non-filtered query #637
+
+
+## 2.88.4
+Released 2017-03-10
+
+Announcements:
+ - Upgrades camshaft to [0.50.3](https://github.com/CartoDB/camshaft/releases/tag/0.50.3).
+
+
+## 2.88.3
+Released 2017-03-02
+
+Bug fixes:
+- Category dataviews now uses the proper aggregation function for the 'Other' category. See https://github.com/CartoDB/Windshaft-cartodb/issues/628
+
+## 2.88.2
+Released 2017-02-23
+
+Announcements:
+ - Upgrades camshaft to [0.50.2](https://github.com/CartoDB/camshaft/releases/tag/0.50.2).
+
+
+## 2.88.1
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.1](https://github.com/CartoDB/camshaft/releases/tag/0.50.1)
+
+
+## 2.88.0
+Released 2017-02-21
+
+Announcements:
+ - Upgrades camshaft to [0.50.0](https://github.com/CartoDB/camshaft/releases/tag/0.50.0).
+ - Upgrades cartodb-psql to [0.7.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.7.1).
+ - Upgrades windshaft to [2.7.0](https://github.com/CartoDB/windshaft/releases/tag/2.7.0).
+
+
 ## 2.87.5
 Released 2017-02-02
 
@@ -26,6 +74,7 @@ Released 2016-12-19
 
 - Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
 
+
 ## 2.87.1
 Released 2016-12-13
 

lib/cartodb/controllers/layergroup.js

@@ -338,6 +338,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
 LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
     var self = this;
 
+    req.profiler.done('res');
+
     res.set('Cache-Control', 'public,max-age=31536000');
 
     // Set Last-Modified header

lib/cartodb/models/dataview/aggregation.js

@@ -48,7 +48,8 @@ var rankedAggregationQueryTpl = dot.template([
     '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
+    'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
+    '       count, categories_count',
     '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
@@ -129,27 +130,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
 
     if (!!override.ownFilter) {
         aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql(),
-                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
-                }),
-                categoriesSummaryMinMaxQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                categoriesSummaryCountQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
+            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
             aggregationQueryTpl({
                 _query: _query,
                 _column: this.column,
@@ -159,30 +140,11 @@ Aggregation.prototype.sql = function(psql, override, callback) {
         ].join('\n');
     } else {
         aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql(),
-                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
-                }),
-                categoriesSummaryMinMaxQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                categoriesSummaryCountQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
+            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
             rankedAggregationQueryTpl({
                 _query: _query,
                 _column: this.column,
+                _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
                 _limit: CATEGORIES_LIMIT
             })
         ].join('\n');
@@ -193,6 +155,32 @@ Aggregation.prototype.sql = function(psql, override, callback) {
     return callback(null, aggregationSql);
 };
 
+Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
+    return [
+                "WITH",
+                [
+                    summaryQueryTpl({
+                        _query: query,
+                        _column: column
+                    }),
+                    rankedCategoriesQueryTpl({
+                        _query: query,
+                        _column: column,
+                        _aggregation: this.getAggregationSql(),
+                        _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
+                    }),
+                    categoriesSummaryMinMaxQueryTpl({
+                        _query: query,
+                        _column: column
+                    }),
+                    categoriesSummaryCountQueryTpl({
+                        _query: query,
+                        _column: column
+                    })
+                ].join(',\n')
+            ].join('\n');
+};
+
 var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
 Aggregation.prototype.getAggregationSql = function() {
     return aggregationFnQueryTpl({

lib/cartodb/models/dataview/factory.js

@@ -11,7 +11,7 @@ var DataviewFactory = {
         if (!this.dataviews[type]) {
             throw new Error('Invalid dataview type: "' + type + '"');
         }
-        return new this.dataviews[type](query, dataviewDefinition.options);
+        return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql);
     }
 };
 

lib/cartodb/models/dataview/histogram.js

@@ -109,12 +109,13 @@ var TYPE = 'histogram';
      }
  }
  */
-function Histogram(query, options) {
+function Histogram(query, options, queries) {
     if (!_.isString(options.column)) {
         throw new Error('Histogram expects `column` in widget options');
     }
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.bins = options.bins;
 
@@ -143,7 +144,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
     var _column = this.column;
 
     var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.query
+        column: _column, query: this.queries.no_filters
     });
 
     if (this._columnType === null) {

lib/cartodb/models/dataview/overviews/factory.js

@@ -14,7 +14,8 @@ OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinit
         return parentFactory.getDataview(query, dataviewDefinition);
     }
     return new dataviews[type](
-        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
+        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options,
+        dataviewDefinition.sql
     );
 };
 

lib/cartodb/models/dataview/overviews/histogram.js

@@ -96,10 +96,11 @@ var histogramQueryTpl = dot.template([
     'ORDER BY bin'
 ].join('\n'));
 
-function Histogram(query, options, queryRewriter, queryRewriteData, params) {
+function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
     BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.bins = options.bins;
 
@@ -129,7 +130,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
     var _column = this.column;
 
     var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.rewrittenQuery(this.query)
+        column: _column, query: this.rewrittenQuery(this.queries.no_filters)
     });
 
     if (this._columnType === null) {

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -4,6 +4,13 @@ var dot = require('dot');
 dot.templateSettings.strip = false;
 var queue = require('queue-async');
 var PSQL = require('cartodb-psql');
+/**
+ * cartodb-psql creates `global.Promise` as an empty constructor.
+ * However, `turbo-carto` relies on a polyfil that fails to create the polyfil
+ * as it finds `global.Promise` but it doesn't find `Promise.resolve`.
+ */
+global.Promise = global.Promise || function() {};
+global.Promise.resolve = global.Promise.resolve || function() {};
 var turboCarto = require('turbo-carto');
 
 var SubstitutionTokens = require('../../../utils/substitution-tokens');

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.87.5",
+  "version": "2.89.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,8 +20,8 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.48.5",
-    "cartodb-psql": "~0.6.1",
+    "camshaft": "0.50.3",
+    "cartodb-psql": "~0.7.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "0.13.1",
     "debug": "~2.2.0",
@@ -39,7 +39,7 @@
     "step-profiler": "~0.3.0",
     "turbo-carto": "0.19.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.6.5",
+    "windshaft": "2.8.0",
     "yargs": "~5.0.0",
     "zipfile": "cartodb/node-zipfile#0.5.11-cdb1"
   },

test/acceptance/dataviews/aggregation.js

@@ -1,5 +1,4 @@
 require('../../support/test_helper');
-
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
@@ -107,4 +106,43 @@ describe('aggregations happy cases', function() {
             });
         });
     });
+
+    var operations_and_values = {'count': 9, 'sum': 45, 'avg': 5, 'max': 9, 'min': 1};
+
+    var query_other = [
+        'select generate_series(1,3) as val, \'other_a\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(4,6) as val, \'other_b\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(7,9) as val, \'other_c\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_1\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_2\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_3\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_4\' as cat, NULL as the_geom_webmercator',
+        'select generate_series(10,12) as val, \'category_5\' as cat, NULL as the_geom_webmercator'
+    ].join(' UNION ALL ');
+
+    Object.keys(operations_and_values).forEach(function (operation) {
+        var description = 'should aggregate OTHER category using "' + operation + '"';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 8);
+                assert.equal(aggregation.count, 24);
+                assert.equal(aggregation.nulls, 0);
+
+                var aggregated_categories = aggregation.categories.filter( function(category) {
+                    return category.agg === true;
+                });
+                assert.equal(aggregated_categories.length, 1);
+                assert.equal(aggregated_categories[0].value, operations_and_values[operation]);
+
+                done();
+            });
+        });
+    });
 });

test/acceptance/dataviews/overviews.js

@@ -144,6 +144,22 @@ describe('dataviews using tables with overviews', function() {
                     aggregationColumn: 'name',
                 }
             },
+            test_histogram: {
+                type: 'histogram',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    bins: 2
+                }
+            },
+            test_histogram_date: {
+                type: 'histogram',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'updated_at',
+                    bins: 2
+                }
+            },
             test_avg: {
                 type: 'formula',
                 source: {id: 'data-source'},
@@ -265,8 +281,83 @@ describe('dataviews using tables with overviews', function() {
         });
     });
 
+    it("should expose a histogram", function (done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_histogram', function (err, histogram) {
+            if (err) {
+                return done(err);
+            }
+            assert.ok(histogram);
+            assert.equal(histogram.type, 'histogram');
+            assert.ok(Array.isArray(histogram.bins));
+            testClient.drain(done);
+        });
+    });
+
     describe('filters', function() {
 
+        describe('histogram', function () {
+
+            it("should expose a filtered histogram", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram: { min: 2 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 4);
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered histogram with no results", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram: { max: -1 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 0);
+                    testClient.drain(done);
+                });
+            });
+
+            it("should expose a filtered date histogram with no results", function (done) {
+                // This most likely works because the overviews will pass
+                // the responsibility to the normal dataviews.
+                var params = {
+                    filters: {
+                        dataviews: { test_histogram_date: { max: -1 } }
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram_date', params, function (err, histogram) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+                    assert.ok(Array.isArray(histogram.bins));
+                    assert.equal(histogram.bins.length, 0);
+                    testClient.drain(done);
+                });
+            });
+        });
+
         describe('category', function () {
 
             var params = {

test/acceptance/multilayer.js

@@ -574,7 +574,7 @@ describe(suiteName, function() {
             if ( err ) {
               return done(err);
             }
-            // trip epoch
+            // strip epoch
             expected_token = expected_token.split(':')[0];
             keysToDelete['map_cfg|' + expected_token] = 0;
             keysToDelete['user:localhost:mapviews:global'] = 5;

test/acceptance/widgets/ported/histogram.js

@@ -304,6 +304,37 @@ describe('widgets', function() {
                 });
             });
 
+            it('can use a datetime filtered column with no results', function(done) {
+                this.testClient = new TestClient(histogramsMapConfig({
+                    updated_at: {
+                        type: 'histogram',
+                        options: {
+                            column: 'updated_at'
+                        }
+                    }
+                }));
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [{
+                            updated_at: {
+                                // this will remove all results
+                                max: -1
+                            }
+                        }]
+                    }
+                };
+                this.testClient.getWidget('updated_at', params, function (err, res, histogram) {
+                    assert.ok(!err, err);
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+
+                    assert.equal(histogram.bins.length, 0);
+
+                    done();
+                });
+            });
+
             it('can getTile with datetime filtered column', function(done) {
                 this.testClient = new TestClient(histogramsMapConfig({
                     updated_at: {

test/integration/mapconfig_named_layers_datasource.js

@@ -145,7 +145,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[0].type, 'cartodb');
                 assert.equal(layers[0].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(0), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(0), {});
 
                 done();
             }
@@ -160,7 +160,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[0].type, 'cartodb');
                 assert.equal(layers[0].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(0), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(0), {});
 
                 assert.equal(layers[1].type, 'cartodb');
                 assert.equal(layers[1].options.sql, wadusTemplateSql);
@@ -181,7 +181,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[0].type, 'cartodb');
                 assert.equal(layers[0].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(0), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(0), {});
 
                 assert.equal(layers[1].type, 'mapnik');
                 assert.equal(layers[1].options.sql, wadusMapnikSql);
@@ -263,7 +263,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[3].type, 'cartodb');
                 assert.equal(layers[3].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(3), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(3), {});
 
                 assert.equal(layers[4].type, 'cartodb');
                 assert.equal(layers[4].options.sql, wadusTemplateSql);
@@ -273,7 +273,7 @@ describe('named_layers datasources', function() {
 
                 assert.equal(layers[5].type, 'cartodb');
                 assert.equal(layers[5].options.sql, wadusSql);
-                assert.equal(datasource.getLayerDatasource(5), undefined);
+                assert.deepEqual(datasource.getLayerDatasource(5), {});
 
                 assert.equal(layers[6].type, 'mapnik');
                 assert.equal(layers[6].options.sql, wadusMapnikSql);
@@ -298,6 +298,7 @@ describe('named_layers datasources', function() {
             var context = {};
             mapConfigNamedLayersAdapter.getMapConfig(username, testScenario.config, params, context,
                 function(err, mapConfig) {
+                    assert.ifError(err);
                     testScenario.test(err, mapConfig.layers, context.datasource, done);
                 }
             );