Release 2.87.4

Be able to not compute NULL categories and null values

.travis.yml

@@ -1,13 +1,10 @@
-dist: trusty # only environment that supports Postgres 9.5 at this time
-sudo: required
+dist: trusty
 addons:
   postgresql: "9.5"
   apt:
     packages:
       - postgresql-9.5-postgis-2.3
-      - postgresql-contrib-9.5
       - postgresql-plpython-9.5
-      - postgis
       - pkg-config
       - libcairo2-dev
       - libjpeg8-dev
@@ -25,9 +22,3 @@ env:
 language: node_js
 node_js:
   - "0.10"
-
-notifications:
-  irc:
-    channels:
-      - "irc.freenode.org#cartodb"
-    use_notice: true

NEWS.md

@@ -1,12 +1,39 @@
 # Changelog
 
-## 2.86.2
-Released 2017-09-01
+## 2.87.4
+Released 2017-01-20
 
 Bug fixes:
- - Adding a new cartodb-psql fixing a pg vulnerability
+ - Be able to not compute NULL categories and null values  in aggregation dataviews #617.
 
 
+## 2.87.3
+Released 2016-12-19
+
+Bug fixes:
+ - Fix overviews-related dataviews problems. See https://github.com/CartoDB/Windshaft-cartodb/pull/604
+
+
+## 2.87.2
+Released 2016-12-19
+
+- Use exception safe Dataservices API functions. See https://github.com/CartoDB/dataservices-api/issues/314 and https://github.com/CartoDB/camshaft/issues/242
+
+## 2.87.1
+Released 2016-12-13
+
+Announcements:
+ - Upgrades windshaft to [2.6.4](https://github.com/CartoDB/Windshaft/releases/tag/2.6.4).
+ - Upgrades request dependency.
+ - Regenerate npm-shrinkwrap.json: missing dependency updates.
+
+
+## 2.87.0
+Released 2016-12-12
+
+Enhancements:
+ - Upgrade turbo-carto dependency to version 0.19.0.
+
 ## 2.86.1
 Released 2016-12-02
 

docs/static_maps_api.md

@@ -146,9 +146,9 @@ It is important to note that generated images are cached from the live data refe
 
 ### Limits
 
-* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
-* Image resolution by default is set to 72 DPI
-* JPEG quality by default is 85%
+* While images can encompass an entirety of a map, the limit for pixel range is 8192 x 8192.
+* Image resolution is set to 72 DPI
+* JPEG quality is 85%
 * Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
 
 ## Examples

lib/cartodb/models/dataview/aggregation.js

@@ -19,25 +19,37 @@ var rankedCategoriesQueryTpl = dot.template([
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
     '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM ({{=it._query}}) _cdb_categories',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
     'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
     'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
 ].join('\n'));
@@ -45,7 +57,7 @@ var rankedAggregationQueryTpl = dot.template([
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
     '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
     'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
     'ORDER BY value DESC'
 ].join('\n'));
@@ -114,6 +126,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
     var _query = this.query;
 
     var aggregationSql;
+
     if (!!override.ownFilter) {
         aggregationSql = [
             "WITH",
@@ -125,9 +138,14 @@ Aggregation.prototype.sql = function(psql, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
@@ -150,9 +168,14 @@ Aggregation.prototype.sql = function(psql, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -18,25 +18,37 @@ var rankedCategoriesQueryTpl = dot.template([
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
     '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
     ')'
 ].join('\n'));
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
     '  FROM categories',
     ')'
 ].join('\n'));
 
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM ({{=it._query}}) _cdb_categories',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
 var rankedAggregationQueryTpl = dot.template([
     'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
     'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
     'GROUP BY nulls_count, min_val, max_val, count, categories_count'
 ].join('\n'));
@@ -44,7 +56,7 @@ var rankedAggregationQueryTpl = dot.template([
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
     '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
     'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
     'ORDER BY value DESC'
 ].join('\n'));
@@ -65,7 +77,7 @@ Aggregation.prototype.constructor = Aggregation;
 
 module.exports = Aggregation;
 
-Aggregation.prototype.sql = function(psql, filters, override, callback) {
+Aggregation.prototype.sql = function(psql, override, callback) {
     if (!callback) {
         callback = override;
         override = {};
@@ -85,9 +97,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })
@@ -110,9 +127,14 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) {
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
-                    _aggregation: this.getAggregationSql()
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
                 }),
-                categoriesSummaryQueryTpl({
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
                     _query: _query,
                     _column: this.column
                 })

lib/cartodb/models/dataview/overviews/base.js

@@ -55,20 +55,20 @@ BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
 };
 
 // Default behaviour
-BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
+BaseOverviewsDataview.prototype.defaultSql = function(psql, override, callback) {
     var query = this.query;
     var dataview = this.baseDataview;
     if ( SETTINGS.defaultOverviews ) {
         query = this.rewrittenQuery(query);
         dataview = new this.BaseDataview(query, this.queryOptions);
     }
-    return dataview.sql(psql, filters, override, callback);
+    return dataview.sql(psql, override, callback);
 };
 
 // default implementation that can be override in derived classes:
 
-BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
-    return this.defaultSql(psql, filters, override, callback);
+BaseOverviewsDataview.prototype.sql = function(psql, override, callback) {
+    return this.defaultSql(psql, override, callback);
 };
 
 BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {

lib/cartodb/models/dataview/overviews/formula.js

@@ -36,7 +36,7 @@ Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-Formula.prototype.sql = function(psql, filters, override, callback) {
+Formula.prototype.sql = function(psql, override, callback) {
     var formulaQueryTpl = formulaQueryTpls[this.operation];
 
     if ( formulaQueryTpl ) {
@@ -52,5 +52,5 @@ Formula.prototype.sql = function(psql, filters, override, callback) {
     }
 
     // default behaviour
-    return this.defaultSql(psql, filters, override, callback);
+    return this.defaultSql(psql, override, callback);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -8,7 +8,6 @@ dot.templateSettings.strip = false;
 var columnTypeQueryTpl = dot.template(
     'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
 );
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
 
 var BIN_MIN_NUMBER = 6;
 var BIN_MAX_NUMBER = 48;
@@ -149,7 +148,9 @@ Histogram.prototype.sql = function(psql, override, callback) {
     }
 
     if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
+        // overviews currently aggregate dates to NULL
+        // to avoid problem we don't use overviews for histograms of date columns
+        return this.defaultSql(psql, override, callback);
     }
 
     var _query = this.rewrittenQuery(this.query);

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.86.2",
+  "version": "2.87.4",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,8 +20,8 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.48.4",
-    "cartodb-psql": "0.10.1",
+    "camshaft": "0.48.5",
+    "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "0.13.1",
     "debug": "~2.2.0",
@@ -34,13 +34,14 @@
     "node-statsd": "~0.0.7",
     "queue-async": "~1.0.7",
     "redis-mpool": "~0.4.0",
-    "request": "~2.62.0",
+    "request": "~2.79.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.18.0",
+    "turbo-carto": "0.19.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.6.3",
-    "yargs": "~5.0.0"
+    "windshaft": "2.6.5",
+    "yargs": "~5.0.0",
+    "zipfile": "cartodb/node-zipfile#0.5.11-cdb1"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",

test/acceptance/dataviews/aggregation.js

@@ -3,7 +3,7 @@ require('../../support/test_helper');
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
-describe('aggregations', function() {
+describe('aggregations happy cases', function() {
 
     afterEach(function(done) {
         if (this.testClient) {
@@ -13,30 +13,36 @@ describe('aggregations', function() {
         }
     });
 
-    function aggregationOperationMapConfig(operation) {
-        return {
+    function aggregationOperationMapConfig(operation, query, column, aggregationColumn) {
+        column = column || 'adm0name';
+        aggregationColumn = aggregationColumn || 'pop_max';
+        query = query || 'select * from populated_places_simple_reduced';
+
+        var mapConfig = {
             version: '1.5.0',
             layers: [
                 {
                     type: 'mapnik',
                     options: {
-                        sql: 'select * from populated_places_simple_reduced',
+                        sql: query,
                         cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
                         cartocss_version: '2.0.1',
-                        widgets: {
-                            adm0name: {
-                                type: 'aggregation',
-                                options: {
-                                    column: 'adm0name',
-                                    aggregation: operation,
-                                    aggregationColumn: 'pop_max'
-                                }
-                            }
-                        }
+                        widgets: {}
                     }
                 }
             ]
         };
+
+        mapConfig.layers[0].options.widgets[column] = {
+            type: 'aggregation',
+            options: {
+                column: column,
+                aggregation: operation,
+                aggregationColumn: aggregationColumn
+            }
+        };
+
+        return mapConfig;
     }
 
     var operations = ['count', 'sum', 'avg', 'max', 'min'];
@@ -56,4 +62,49 @@ describe('aggregations', function() {
             });
         });
     });
+
+    var query = [
+        'select 1 as val, \'a\' as cat, ST_Transform(ST_SetSRID(ST_MakePoint(0,0),4326),3857) as the_geom_webmercator',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(0,1),4326),3857)',
+        'select null, \'b\', ST_Transform(ST_SetSRID(ST_MakePoint(1,0),4326),3857)',
+        'select null, null, ST_Transform(ST_SetSRID(ST_MakePoint(1,1),4326),3857)'
+    ].join(' UNION ALL ');
+
+    operations.forEach(function (operation) {
+        var not = operation === 'count' ? ' not ' : ' ';
+        var description = 'should' +
+            not +
+            'handle NULL values in category and aggregation columns using "' +
+            operation +
+            '" as aggregation operation';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query, 'cat', 'val'));
+            this.testClient.getDataview('cat', { own_filter: 0 }, function (err, aggregation) {
+                assert.ifError(err);
+
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+                assert.ok(aggregation.categories);
+                assert.equal(aggregation.categoriesCount, 3);
+                assert.equal(aggregation.count, 4);
+                assert.equal(aggregation.nulls, 1);
+
+                var hasNullCategory = false;
+                aggregation.categories.forEach(function (category) {
+                    if (category.category === null) {
+                        hasNullCategory = true;
+                    }
+                });
+
+                if (operation === 'count') {
+                    assert.ok(hasNullCategory, 'aggregation has not a category NULL');
+                } else {
+                    assert.ok(!hasNullCategory, 'aggregation has category NULL');
+                }
+
+                done();
+            });
+        });
+    });
 });