From 1d66e49910376ceeb66efb044df651c8a52c4316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 1 Jun 2017 20:07:46 +0200 Subject: [PATCH 01/44] WIP implemented date histogram --- lib/cartodb/backends/dataview.js | 7 ++ lib/cartodb/controllers/base.js | 1 + lib/cartodb/models/dataview/histogram.js | 96 ++++++++++++++++++++- test/acceptance/dataviews/histogram.js | 102 +++++++++++++++++++++-- test/support/test-client.js | 4 +- 5 files changed, 195 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e0515efe..84f0ccef 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,6 +20,8 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } +var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year']; + module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { @@ -30,6 +32,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param mapConfigProvider.getMapConfig(this); }, function runDataviewQuery(err, mapConfig) { + /* jshint maxcomplexity: 7 */ assert.ifError(err); var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); @@ -88,6 +91,10 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); + if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) { + overrideParams.aggregation = params.aggregation; + } + var dataview = dataviewFactory.getDataview(query, dataviewDefinition); dataview.getResult(pg, overrideParams, this); }, diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index ab6587c2..6ce49fa4 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -25,6 +25,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'start', // number 'end', // number 'column_type', // string + 'aggregation', //string // widgets search 'q' ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..6f3a2492 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -8,7 +8,7 @@ dot.templateSettings.strip = false; var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' ); -var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -71,7 +71,7 @@ var overrideBinsQueryTpl = dot.template([ var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -97,6 +97,52 @@ var histogramQueryTpl = dot.template([ 'ORDER BY bin' ].join('\n')); +var dateBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,', + ' max({{=it._column}}) AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var dateBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT', + ' bins_array,', + ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' FROM (', + ' SELECT', + ' ARRAY(', + ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' ) AS bins_array', + ' FROM basics', + ' ) _cdb_bins_array', + ')' +].join('\n')); + +var dateHistogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', + ' END AS bin,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', + ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); var TYPE = 'histogram'; @@ -118,6 +164,7 @@ function Histogram(query, options, queries) { this.queries = queries; this.column = options.column; this.bins = options.bins; + this.aggregation = options.aggregation; this._columnType = null; } @@ -163,7 +210,7 @@ Histogram.prototype.sql = function(psql, override, callback) { } if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); + return this._buildDateHistogramQuery(override, callback); } var _query = this.query; @@ -211,7 +258,6 @@ Histogram.prototype.sql = function(psql, override, callback) { } } - var histogramSql = [ "WITH", [ @@ -233,6 +279,48 @@ Histogram.prototype.sql = function(psql, override, callback) { return callback(null, histogramSql); }; +Histogram.prototype._buildDateHistogramQuery = function (override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + + var dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation + }); + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ + _query: _query, + _column: _column + }); + + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column + }); + + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c581380a 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -3,6 +3,15 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + describe('histogram-dataview', function() { afterEach(function(done) { @@ -13,15 +22,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -94,6 +94,90 @@ describe('histogram-dataview', function() { assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); + done(); + }); + }); + +}); + +describe('histogram-dataview for date column type', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "date-histogram-source" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month' + } + } + }, + [ + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval", + ") date" + ].join(' ') + } + } + ] + ); + + it('should create a date histogram aggregated in months', function (done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override aggregation in weeks', function (done) { + var params = { + aggregation: 'week' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + done(); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..1913d018 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } @@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), From 9c34428984565f94594903e2f9ccb20ec54d5b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 12:15:43 +0200 Subject: [PATCH 02/44] Allow override start and end params --- lib/cartodb/models/dataview/histogram.js | 52 ++++++++++++++++++++---- test/acceptance/dataviews/histogram.js | 23 +++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6f3a2492..dd3baf95 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -8,7 +8,7 @@ dot.templateSettings.strip = false; var columnTypeQueryTpl = dot.template( 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' ); -// var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -110,6 +110,28 @@ var dateBasicsQueryTpl = dot.template([ ')' ].join(' \n')); +var overrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', + ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join('\n')); + +var dateOverrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val,', + ' min({{=it._start}}) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(date_trunc(\'{{=it._aggregation}}\', TO_TIMESTAMP({{=it._start}})::timestamp)) AS start_date,', + ' max(TO_TIMESTAMP({{=it._end}})::timestamp) AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + var dateBinsQueryTpl = dot.template([ 'bins AS (', ' SELECT', @@ -181,6 +203,7 @@ var DATE_OIDS = { }; Histogram.prototype.sql = function(psql, override, callback) { + // jshint maxcomplexity: 7 if (!callback) { callback = override; override = {}; @@ -209,10 +232,13 @@ Histogram.prototype.sql = function(psql, override, callback) { return null; } - if (this._columnType === 'date') { + if (this._columnType === 'date' && this.aggregation !== undefined) { return this._buildDateHistogramQuery(override, callback); } + if (this._columnType === 'date') { + _column = columnCastTpl({ column: this.column}); + } var _query = this.query; var basicsQuery, binsQuery; @@ -284,11 +310,23 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var dateBasicsQuery = dateBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation - }); + var dateBasicsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end')) { + dateBasicsQuery = dateOverrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _start: getBinStart(override), + _end: getBinEnd(override) + }); + } else { + dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation + }); + } var dateBinsQuery = [ dateBinsQueryTpl({ diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c581380a..c50abebb 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -156,6 +156,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -174,6 +175,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -181,4 +183,25 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should override start and end', function (done) { + var params = { + start: 1180659600, // 2007-06-01 00:00:00 + end: 1193792400 // 2007-10-31 01:00:00 + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 5); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + }); From 6374d2e4b64929ce4394ce0bfef4e42a8c3bccc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 12:17:55 +0200 Subject: [PATCH 03/44] Fix typo --- test/acceptance/dataviews/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c50abebb..294c9106 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -186,7 +186,7 @@ describe('histogram-dataview for date column type', function() { it('should override start and end', function (done) { var params = { - start: 1180659600, // 2007-06-01 00:00:00 + start: 1180659600, // 2007-06-01 01:00:00 end: 1193792400 // 2007-10-31 01:00:00 }; From c1fac13d6bedf8140a5b6ed0eb3451f1540c70c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 12:45:34 +0200 Subject: [PATCH 04/44] Be able to accept timezone parameter --- lib/cartodb/backends/dataview.js | 2 +- lib/cartodb/controllers/base.js | 1 + lib/cartodb/models/dataview/histogram.js | 13 +++---------- test/acceptance/dataviews/histogram.js | 18 ++++++++++++++++++ test/support/test-client.js | 2 +- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 84f0ccef..dc43f864 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -80,7 +80,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } ); - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'timezone'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 6ce49fa4..91fcaba7 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -26,6 +26,7 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'end', // number 'column_type', // string 'aggregation', //string + 'timezone', // number // widgets search 'q' ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dd3baf95..649bd0f2 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -110,15 +110,6 @@ var dateBasicsQueryTpl = dot.template([ ')' ].join(' \n')); -var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', - ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', - ')' -].join('\n')); - var dateOverrideBasicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', @@ -156,13 +147,14 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', ' END AS bin,', + ' date_part(\'epoch\', start_date)::numeric AS bin_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', 'ORDER BY bin' ].join('\n')); @@ -187,6 +179,7 @@ function Histogram(query, options, queries) { this.column = options.column; this.bins = options.bins; this.aggregation = options.aggregation; + this.timezone = options.timezone; this._columnType = null; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 294c9106..06fc6b80 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -204,4 +204,22 @@ describe('histogram-dataview for date column type', function() { }); }); + it.skip('should aggregate respecting timezone', function (done) { + var params = { + timezone: -7200 // GMT -2h + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 5); + dataview.bins.forEach(function(bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index 1913d018..dc36929e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end', 'aggregation'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'timezone'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } From 8bc38a375a4b0189814e4563c060c2788f557035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 18:37:49 +0200 Subject: [PATCH 05/44] Support timezone aggregation --- lib/cartodb/models/dataview/histogram.js | 51 +++++++++++++++++++----- test/acceptance/dataviews/histogram.js | 11 ++--- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 649bd0f2..499f32cd 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -103,8 +103,10 @@ var dateBasicsQueryTpl = dot.template([ ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', - ' min(date_trunc(\'{{=it._aggregation}}\', {{=it._column}})) AS start_date,', - ' max({{=it._column}}) AS end_date,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' )) AS start_date,', + ' max({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -116,8 +118,11 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' max({{=it._end}}) AS max_val,', ' min({{=it._start}}) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', - ' min(date_trunc(\'{{=it._aggregation}}\', TO_TIMESTAMP({{=it._start}})::timestamp)) AS start_date,', - ' max(TO_TIMESTAMP({{=it._end}})::timestamp) AS end_date,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' )) AS start_date,', + ' max(TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -145,9 +150,16 @@ var dateHistogramQueryTpl = dot.template([ ' nulls_count,', ' CASE WHEN min_val = max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, bins_array), bins_number)) - 1', + ' ELSE GREATEST(1, LEAST(', + ' WIDTH_BUCKET({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\', bins_array),', + ' bins_number', + ' )) - 1', ' END AS bin,', - ' date_part(\'epoch\', start_date)::numeric AS bin_start,', + ' date_part(\'epoch\', start_date)::numeric AS bins_start,', + ' min(date_part(', + ' \'epoch\', ', + ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', + ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', @@ -302,6 +314,7 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _timezone = override && override.timezone ? override.timezone : this.timezone; var dateBasicsQuery; @@ -311,13 +324,15 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { _column: _column, _aggregation: _aggregation, _start: getBinStart(override), - _end: getBinEnd(override) + _end: getBinEnd(override), + _timezone: getTimezone(_timezone) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, - _aggregation: _aggregation + _aggregation: _aggregation, + _timezone: getTimezone(_timezone) }); } @@ -334,7 +349,9 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var dateHistogramQuery = dateHistogramQueryTpl({ _query: _query, - _column: _column + _column: _column, + _aggregation: _aggregation, + _timezone: getTimezone(_timezone) }); var histogramSql = [ @@ -371,7 +388,7 @@ Histogram.prototype.format = function(result, override) { binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'bins_start'); }); } @@ -414,6 +431,20 @@ function getWidth(override) { return width; } +function getTimezone(timezone) { + if (!timezone) { + return '0'; + } + + var timezoneInHours = Math.ceil(timezone / 3600); + + if (timezoneInHours >= 0) { + return '+' + timezoneInHours; + } else { + return '' + timezoneInHours; + } +} + Histogram.prototype.getType = function() { return TYPE; }; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 06fc6b80..f5fab192 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -130,7 +130,8 @@ describe('histogram-dataview for date column type', function() { type: 'histogram', options: { column: 'd', - aggregation: 'month' + aggregation: 'month', + timezone: -7200 } } }, @@ -195,7 +196,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 5); + assert.equal(dataview.bins.length, 6); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -204,9 +205,9 @@ describe('histogram-dataview for date column type', function() { }); }); - it.skip('should aggregate respecting timezone', function (done) { + it('should aggregate respecting timezone', function (done) { var params = { - timezone: -7200 // GMT -2h + timezone: 7200 // GMT +2h }; this.testClient = new TestClient(mapConfig, 1234); @@ -214,7 +215,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 5); + assert.equal(dataview.bins.length, 15); dataview.bins.forEach(function(bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); From 32eeb57fce7710e7a8ab936f9c52b7da0bb06fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 2 Jun 2017 19:00:26 +0200 Subject: [PATCH 06/44] Reduce complexity in function --- lib/cartodb/models/dataview/histogram.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 499f32cd..e14aa4d1 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -437,12 +437,7 @@ function getTimezone(timezone) { } var timezoneInHours = Math.ceil(timezone / 3600); - - if (timezoneInHours >= 0) { - return '+' + timezoneInHours; - } else { - return '' + timezoneInHours; - } + return '' + timezoneInHours; } Histogram.prototype.getType = function() { From 6ff3b33cde2874e5300c4727081676fb702f6cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:17:50 +0200 Subject: [PATCH 07/44] Removed bins_start as query output --- lib/cartodb/models/dataview/histogram.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index e14aa4d1..17921776 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -155,7 +155,6 @@ var dateHistogramQueryTpl = dot.template([ ' bins_number', ' )) - 1', ' END AS bin,', - ' date_part(\'epoch\', start_date)::numeric AS bins_start,', ' min(date_part(', ' \'epoch\', ', ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', @@ -385,7 +384,11 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; - binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; + binsStart = override.hasOwnProperty('start') ? + getBinStart(override) : + firstRow.hasOwnProperty('timestamp') ? + firstRow.timestamp : + firstRow.min; buckets = result.rows.map(function(row) { return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'bins_start'); From 71e9e62db07fa404052703f14decd7c5993b49c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:18:24 +0200 Subject: [PATCH 08/44] Improved histogram assertion with moment.js --- package.json | 1 + test/acceptance/dataviews/histogram.js | 19 +++++++++++++++-- yarn.lock | 28 +++++++++++++------------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 230ca8ff..fea8b67c 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "istanbul": "~0.4.3", "jshint": "~2.9.4", "mocha": "~3.4.1", + "moment": "~2.18.1", "nock": "~2.11.0", "redis": "~0.12.1", "semver": "~1.1.4", diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index f5fab192..2ca2ac6b 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -2,6 +2,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +var moment = require('moment'); function createMapConfig(layers, dataviews, analysis) { return { @@ -206,8 +207,10 @@ describe('histogram-dataview for date column type', function() { }); it('should aggregate respecting timezone', function (done) { + var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var params = { - timezone: 7200 // GMT +2h + timezone: TIMEZONE_CEST_IN_SECONDS }; this.testClient = new TestClient(mapConfig, 1234); @@ -216,7 +219,19 @@ describe('histogram-dataview for date column type', function() { assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin) { + + var initialTimestamp = '2007-02-01T00:00:00+02:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp).utcOffset(TIMEZONE_CEST_IN_MINUTES).add(index, 'month').format(); + console.log(binTimestampExpected); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); diff --git a/yarn.lock b/yarn.lock index 089b43ad..4cc18ef4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -150,7 +150,7 @@ boom@2.x.x: dependencies: hoek "2.x.x" -brace-expansion@^1.1.7: +brace-expansion@^1.0.0: version "1.1.7" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59" dependencies: @@ -209,7 +209,7 @@ camshaft@0.55.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -235,7 +235,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1292,7 +1292,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6: +moment@^2.10.6, moment@^2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1835,22 +1835,22 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@~4.3.3: - version "4.3.6" - resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" -semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - semver@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540" +semver@~4.3.3: + version "4.3.6" + resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" + semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" @@ -2107,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2115,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: From dc844f8131ab3033feedcf0e1f81156c082aa4da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:23:53 +0200 Subject: [PATCH 09/44] Remove console.log --- test/acceptance/dataviews/histogram.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 2ca2ac6b..62a4aac8 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -227,11 +227,10 @@ describe('histogram-dataview for date column type', function() { dataview.bins.forEach(function(bin, index) { var binTimestampExpected = moment.utc(initialTimestamp).utcOffset(TIMEZONE_CEST_IN_MINUTES).add(index, 'month').format(); - console.log(binTimestampExpected); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); - assert.equal(binTimestampFormatted, binTimestampExpected); + assert.equal(binTimestampFormatted, binTimestampExpected); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); From 869408b7b7057641814e734d7e22e4b1e1e6f068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:50:49 +0200 Subject: [PATCH 10/44] Use Eastern Daylight Time while testing --- test/acceptance/dataviews/histogram.js | 39 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 62a4aac8..b2c10ea5 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -132,7 +132,7 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd', aggregation: 'month', - timezone: -7200 + timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } } }, @@ -153,13 +153,35 @@ describe('histogram-dataview for date column type', function() { ); it('should create a date histogram aggregated in months', function (done) { + var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', {}, function(err, dataview) { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin) { + + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min && 'bin timestamp < bin min: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -222,13 +244,20 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00+02:00'; var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp).utcOffset(TIMEZONE_CEST_IN_MINUTES).add(index, 'month').format(); + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .add(index, 'month') + .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds).utcOffset(TIMEZONE_CEST_IN_MINUTES).format(); + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); assert.equal(binTimestampFormatted, binTimestampExpected); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); From 003227fb295f927edaef9151308cde328ad92520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 14:59:35 +0200 Subject: [PATCH 11/44] Fix assertion --- test/acceptance/dataviews/histogram.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index b2c10ea5..adcab0e1 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -181,7 +181,7 @@ describe('histogram-dataview for date column type', function() { .format(); assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min && 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -260,6 +260,7 @@ describe('histogram-dataview for date column type', function() { .format(); assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); From c86f92f8eb49cc885c7422cd2c3b5db5ad5a344b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 15:04:40 +0200 Subject: [PATCH 12/44] Improve test description --- test/acceptance/dataviews/histogram.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index adcab0e1..cc9e3dc1 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -152,7 +152,7 @@ describe('histogram-dataview for date column type', function() { ] ); - it('should create a date histogram aggregated in months', function (done) { + it('should create a date histogram aggregated in months (EDT)', function (done) { var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes this.testClient = new TestClient(mapConfig, 1234); @@ -228,7 +228,7 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should aggregate respecting timezone', function (done) { + it('should aggregate histogram overriding default timezone to CEST', function (done) { var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var params = { @@ -242,7 +242,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); assert.equal(dataview.bins.length, 15); - var initialTimestamp = '2007-02-01T00:00:00+02:00'; + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_CEST_IN_MINUTES) From d01787842f93f404d28ccb12c26aa374ccd084ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 15:23:04 +0200 Subject: [PATCH 13/44] Support UTC timezone override --- lib/cartodb/models/dataview/histogram.js | 3 +- test/acceptance/dataviews/histogram.js | 40 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 17921776..1328383d 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -313,7 +313,7 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _timezone = override && override.timezone ? override.timezone : this.timezone; + var _timezone = override && Number.isFinite(override.timezone) ? override.timezone : this.timezone; var dateBasicsQuery; @@ -438,7 +438,6 @@ function getTimezone(timezone) { if (!timezone) { return '0'; } - var timezoneInHours = Math.ceil(timezone / 3600); return '' + timezoneInHours; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index cc9e3dc1..5e4de9b9 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -267,4 +267,44 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should aggregate histogram overriding default timezone to UTC/GMT', function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); }); From 95a6ad3b86cea3b4f4a862b9a5d6249b389bba2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:04:42 +0200 Subject: [PATCH 14/44] Support quarter aggregation in histograms over date columns --- lib/cartodb/backends/dataview.js | 2 +- lib/cartodb/models/dataview/histogram.js | 4 +++ test/acceptance/dataviews/histogram.js | 42 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index dc43f864..ca451d38 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,7 +20,7 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } -var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'year']; +var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'quarter', 'year']; module.exports = DataviewBackend; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 1328383d..be4c7193 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -136,7 +136,11 @@ var dateBinsQueryTpl = dot.template([ ' FROM (', ' SELECT', ' ARRAY(', + ' {{?it._aggregation==="quarter"}}', + ' SELECT GENERATE_SERIES(start_date, end_date, \'3 month\'::interval)', + ' {{??}}', ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' {{?}}', ' ) AS bins_array', ' FROM basics', ' ) _cdb_bins_array', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 5e4de9b9..0efb56d6 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -307,4 +307,46 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should aggregate histogram using "quarter" aggregation', function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + }); From e82d688a187c6624213a26bbb74e27438945c3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:09:47 +0200 Subject: [PATCH 15/44] Fix typo --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index be4c7193..9db9e0e7 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -161,9 +161,9 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', - ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' min(date_part(\'epoch\', ))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', From 1ff095439014259ea89262319eed89909618a812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:25:47 +0200 Subject: [PATCH 16/44] Fix typo --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index be4c7193..f9162169 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -161,7 +161,7 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', d AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', From 9337cd948cf1b175bdbf038469574bf17e769d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 16:26:29 +0200 Subject: [PATCH 17/44] Fix typo --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 9db9e0e7..f9162169 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -163,7 +163,7 @@ var dateHistogramQueryTpl = dot.template([ ' \'epoch\', ', ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', - ' min(date_part(\'epoch\', ))::numeric AS min,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', From eeea51e10dece2e9d7c99b16b3064373d2661f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 5 Jun 2017 17:26:37 +0200 Subject: [PATCH 18/44] Removed unused column in group by statement --- lib/cartodb/models/dataview/histogram.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index f9162169..cbd9fe20 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -169,7 +169,7 @@ var dateHistogramQueryTpl = dot.template([ ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', 'ORDER BY bin' ].join('\n')); @@ -180,7 +180,8 @@ var TYPE = 'histogram'; type: 'histogram', options: { column: 'name', - bins: 10 // OPTIONAL + bins: 10, // OPTIONAL + aggregation: 'day' // OPTIONAL } } */ From d3bcf6f80d4d1d2802609a55116c796e4478fe9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 6 Jun 2017 09:15:49 +0200 Subject: [PATCH 19/44] Do not omit nonexistent property --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index cbd9fe20..1bf24df2 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -396,7 +396,7 @@ Histogram.prototype.format = function(result, override) { firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'bins_start'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); }); } From 42e2f9e4b1f360af341db7739895c23bab4fbf83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 7 Jun 2017 15:54:19 +0200 Subject: [PATCH 20/44] Update commented use cases --- lib/cartodb/models/dataview/histogram.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 1bf24df2..43b6ee86 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -176,13 +176,23 @@ var dateHistogramQueryTpl = dot.template([ var TYPE = 'histogram'; /** - { - type: 'histogram', - options: { - column: 'name', - bins: 10, // OPTIONAL - aggregation: 'day' // OPTIONAL - } +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} + +Time series: +{ + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) + timezone: -7200 // OPTIONAL (UTC offset in seconds) + } } */ function Histogram(query, options, queries) { From 35d4fb4d27ca48a04643df41bea0f5f57fc14e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 7 Jun 2017 16:11:09 +0200 Subject: [PATCH 21/44] Improve readability, using extract method pattern --- lib/cartodb/models/dataview/histogram.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 43b6ee86..01827b54 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -399,11 +399,7 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; - binsStart = override.hasOwnProperty('start') ? - getBinStart(override) : - firstRow.hasOwnProperty('timestamp') ? - firstRow.timestamp : - firstRow.min; + binsStart = populateBinStart(override, firstRow); buckets = result.rows.map(function(row) { return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); @@ -457,6 +453,20 @@ function getTimezone(timezone) { return '' + timezoneInHours; } +function populateBinStart(override, firstRow) { + var binStart; + + if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); + } else if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else { + binStart = firstRow.min; + } + + return binStart; +} + Histogram.prototype.getType = function() { return TYPE; }; From 2772fc62d2971b9ab23bea3a0c66acba08a27ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 18:38:44 +0200 Subject: [PATCH 22/44] Use a set/dict for checking the existence --- lib/cartodb/backends/dataview.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 8e991b30..55fe81a0 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,7 +20,15 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } -var DATE_AGGREGATIONS = ['minute', 'hour', 'day', 'week', 'month', 'quarter', 'year']; +var DATE_AGGREGATIONS = { + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; module.exports = DataviewBackend; @@ -108,7 +116,7 @@ function getOverrideParams(params, ownFilter) { {ownFilter: ownFilter} ); - if (params.aggregation && DATE_AGGREGATIONS.indexOf(params.aggregation) !== -1) { + if (params.aggregation && DATE_AGGREGATIONS.hasOwnProperty(params.aggregation)) { overrideParams.aggregation = params.aggregation; } From 29a6658e3dbf622e5d4ea8a2e20fbc4ce08d533b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 19:22:33 +0200 Subject: [PATCH 23/44] Migrate dataviews endpoints to use the allow-query-params --- lib/cartodb/controllers/base.js | 14 +------ lib/cartodb/controllers/layergroup.js | 56 +++++++++++++++++++++------ 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 91fcaba7..39ed62c3 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -17,18 +17,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'zoom', 'lon', 'lat', - // widgets & filters - 'filters', // json - 'own_filter', // 0, 1 - 'bbox', // w,s,e,n - 'bins', // number - 'start', // number - 'end', // number - 'column_type', // string - 'aggregation', //string - 'timezone', // number - // widgets search - 'q' + // analysis + 'filters' // json ]; function BaseController(authApi, pgConnection) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index e0c7a7a3..fcab20d2 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -79,19 +79,51 @@ LayergroupController.prototype.register = function(app) { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); + var allowedDataviewQueryParams = [ + 'filters', // json + 'own_filter', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'timezone', // number + 'q' // widgets search + ]; + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); app.get(app.base_url_mapconfig + '/:token/analysis/node/:nodeId', cors(), userMiddleware, From fe5c76d65b5aed1d21645b3f0b268cb24680c400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 8 Jun 2017 19:25:05 +0200 Subject: [PATCH 24/44] Remove jshint hook --- lib/cartodb/backends/dataview.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 55fe81a0..e54b3a83 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -40,7 +40,6 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param mapConfigProvider.getMapConfig(this); }, function runDataviewQuery(err, mapConfig) { - /* jshint maxcomplexity: 7 */ assert.ifError(err); var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); From 6bfedef7eb75ed78f470ea1a003b7262ce45afdb Mon Sep 17 00:00:00 2001 From: Ivan Malagon Date: Mon, 19 Jun 2017 12:47:08 +0200 Subject: [PATCH 25/44] Cast histogram width bucket to timestamp --- lib/cartodb/models/dataview/histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 01827b54..b7bb29a4 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -155,7 +155,7 @@ var dateHistogramQueryTpl = dot.template([ ' CASE WHEN min_val = max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\', bins_array),', + ' WIDTH_BUCKET({{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\', bins_array),', ' bins_number', ' )) - 1', ' END AS bin,', From 6f7cb75256e6d75a19979b160305a5b8a00e91d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 21 Jun 2017 20:19:02 +0200 Subject: [PATCH 26/44] Fix bad datetime conversion --- lib/cartodb/models/dataview/histogram.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b7bb29a4..dbaa002f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -155,7 +155,10 @@ var dateHistogramQueryTpl = dot.template([ ' CASE WHEN min_val = max_val', ' THEN 0', ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET({{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\', bins_array),', + ' WIDTH_BUCKET(', + ' {{=it._column}}::timestamp WITHOUT TIME ZONE AT TIME ZONE \'{{=it._timezone}}\',', + ' bins_array', + ' ),', ' bins_number', ' )) - 1', ' END AS bin,', From a6daca96283c8e304395d9068eb11a4bc4e0759a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 22 Jun 2017 18:04:23 +0200 Subject: [PATCH 27/44] Support date histograms using timestamp with and without timezones --- lib/cartodb/models/dataview/histogram.js | 12 +- test/acceptance/dataviews/histogram.js | 372 ++++++++++++----------- 2 files changed, 210 insertions(+), 174 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dbaa002f..3409e6fb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -137,9 +137,13 @@ var dateBinsQueryTpl = dot.template([ ' SELECT', ' ARRAY(', ' {{?it._aggregation==="quarter"}}', - ' SELECT GENERATE_SERIES(start_date, end_date, \'3 month\'::interval)', + ' SELECT GENERATE_SERIES(start_date::timestamptz, end_date::timestamptz, \'3 month\'::interval)', ' {{??}}', - ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' SELECT GENERATE_SERIES(', + ' start_date::timestamptz,', + ' end_date::timestamptz,', + ' \'1 {{=it._aggregation}}\'::interval', + ' )', ' {{?}}', ' ) AS bins_array', ' FROM basics', @@ -156,7 +160,7 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp WITHOUT TIME ZONE AT TIME ZONE \'{{=it._timezone}}\',', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\',', ' bins_array', ' ),', ' bins_number', @@ -164,7 +168,7 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}}::timestamptz', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 0efb56d6..f39db632 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -134,6 +134,17 @@ describe('histogram-dataview for date column type', function() { aggregation: 'month', timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } + }, + date_histogram_tz: { + source: { + id: 'date-histogram-source-tz' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } } }, [ @@ -148,205 +159,226 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "date-histogram-source-tz", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + ") date" + ].join(' ') + } } ] ); - it('should create a date histogram aggregated in months (EDT)', function (done) { - var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with timezone', + dataviewId: 'date_histogram_tz' + }, { + desc: 'supporting timestamp without timezone', + dataviewId: 'date_histogram' + }]; - this.testClient = new TestClient(mapConfig, 1234); + dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { + var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes - this.testClient.getDataview('date_histogram', {}, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); + this.testClient = new TestClient(mapConfig, 1234); - var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); + this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_EDT_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should override aggregation in weeks', function (done) { - var params = { - aggregation: 'week' - }; + it('should override aggregation in weeks ' + test.desc, function (done) { + var params = { + aggregation: 'week' + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 61); - dataview.bins.forEach(function(bin) { - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should override start and end', function (done) { - var params = { - start: 1180659600, // 2007-06-01 01:00:00 - end: 1193792400 // 2007-10-31 01:00:00 - }; + it('should override start and end ' + test.desc, function (done) { + var params = { + start: 1180659600, // 2007-06-01 01:00:00 + end: 1193792400 // 2007-10-31 01:00:00 + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); - dataview.bins.forEach(function(bin) { - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should aggregate histogram overriding default timezone to CEST', function (done) { - var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) - var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) - var params = { - timezone: TIMEZONE_CEST_IN_SECONDS - }; + it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { + var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + var params = { + timezone: TIMEZONE_CEST_IN_SECONDS + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); - var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_CEST_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); + }); - done(); + it('should aggregate histogram overriding default timezone to UTC/GMT ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); }); }); - - it('should aggregate histogram overriding default timezone to UTC/GMT', function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC - var params = { - timezone: TIMEZONE_UTC_IN_SECONDS - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); - - var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); - }); - - done(); - }); - }); - - it('should aggregate histogram using "quarter" aggregation', function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC - var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, - aggregation: 'quarter' - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); - - var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .add(index * 3, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); - }); - - done(); - }); - }); - }); From af42fba53bb0f964475a6883f49816de648808ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 14:28:23 +0200 Subject: [PATCH 28/44] Check that quarter aggreagtion uses filters properly in date histogram dataview --- test/acceptance/dataviews/histogram.js | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index f39db632..b26f0728 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -380,5 +380,49 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('bins_count should be equal to bins array length filtered by start and end ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870399 // 2008-06-30T23:59:59Z, last bin end + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 6); + assert.equal(dataview.bins_count, dataview.bins.length); + done(); + }); + }); + + it('bins_count should be greater than bins array length filtered by start and end ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 7); + assert.ok(dataview.bins_count > dataview.bins.length); + done(); + }); + }); }); }); From 01027b73da21d9fca87d054c90050599a86d649e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 27 Jun 2017 14:36:18 +0200 Subject: [PATCH 29/44] Fix jshint typo --- test/acceptance/dataviews/histogram.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index b26f0728..93b59b6e 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -381,9 +381,8 @@ describe('histogram-dataview for date column type', function() { }); }); - it('bins_count should be equal to bins array length filtered by start and end ' + test.desc, function (done) { + it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC var params = { timezone: TIMEZONE_UTC_IN_SECONDS, aggregation: 'quarter', @@ -403,9 +402,8 @@ describe('histogram-dataview for date column type', function() { }); }); - it('bins_count should be greater than bins array length filtered by start and end ' + test.desc, function (done) { + it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC var params = { timezone: TIMEZONE_UTC_IN_SECONDS, aggregation: 'quarter', From 7018af18b6d1d5623a919bcf0e30ad86a6425de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 28 Jun 2017 19:58:45 +0200 Subject: [PATCH 30/44] Support automatic aggregation for time-series histogram --- lib/cartodb/models/dataview/histogram.js | 99 +++++++++++++++++++++++- test/acceptance/dataviews/histogram.js | 22 ++++++ 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 3409e6fb..2070bab3 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -10,6 +10,47 @@ var columnTypeQueryTpl = dot.template( ); var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var dateIntervalQueryTpl = dot.template([ + 'WITH', + 'dates AS (', + ' SELECT', + ' MAX({{=it.column}}) AS _end,', + ' MIN({{=it.column}}) AS _start', + ' FROM ({{=it.query}}) _cdb_source', + '),', + 'interval_in_days AS (', + ' SELECT' , + ' DATE_PART(\'day\', _end - _start) AS days', + ' FROM dates', + '),', + 'interval_in_hours AS (', + ' SELECT', + ' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours', + ' FROM interval_in_days, dates', + '),', + 'interval_in_minutes AS (', + ' SELECT', + ' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes', + ' FROM interval_in_hours, dates', + '),', + 'interval_in_seconds AS (', + ' SELECT', + ' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds', + ' FROM interval_in_minutes, dates', + ')', + 'SELECT', + ' ROUND(days / 365) AS year,', + ' ROUND(days / 90) AS quarter,', + ' ROUND(days / 30) AS month,', + ' ROUND(days / 7) AS week,', + ' days AS day,', + ' hours AS hour,', + ' minutes AS minute,', + ' seconds AS second', + 'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds' +].join('\n')); + +var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -258,8 +299,8 @@ Histogram.prototype.sql = function(psql, override, callback) { return null; } - if (this._columnType === 'date' && this.aggregation !== undefined) { - return this._buildDateHistogramQuery(override, callback); + if (this._columnType === 'date') { + return this._buildDateHistogramQuery(psql, override, callback); } if (this._columnType === 'date') { @@ -331,12 +372,24 @@ Histogram.prototype.sql = function(psql, override, callback) { return callback(null, histogramSql); }; -Histogram.prototype._buildDateHistogramQuery = function (override, callback) { +Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _timezone = override && Number.isFinite(override.timezone) ? override.timezone : this.timezone; + if (!_aggregation) { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + var dateBasicsQuery; if (override && _.has(override, 'start') && _.has(override, 'end')) { @@ -390,10 +443,49 @@ Histogram.prototype._buildDateHistogramQuery = function (override, callback) { return callback(null, histogramSql); }; +Histogram.prototype.getAutomaticAggregation = function (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; + var aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); @@ -414,6 +506,7 @@ Histogram.prototype.format = function(result, override) { } return { + aggregation: aggregation, bin_width: width, bins_count: binsCount, bins_start: binsStart, diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 93b59b6e..857aaa07 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -145,6 +145,15 @@ describe('histogram-dataview for date column type', function() { aggregation: 'month', timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } + }, + date_histogram_automatic: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd' + } } }, [ @@ -423,4 +432,17 @@ describe('histogram-dataview for date column type', function() { }); }); }); + + it('automatic mode', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); }); From 6d2934b30baf4f43cb63b62f2e100d140170c5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 29 Jun 2017 16:53:52 +0200 Subject: [PATCH 31/44] Going red: add test to check automatic mode works with dates --- lib/cartodb/models/dataview/histogram.js | 2 + test/acceptance/dataviews/histogram.js | 78 +++++++++++++++++++++--- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 2070bab3..b8b9085a 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -449,6 +449,8 @@ Histogram.prototype.getAutomaticAggregation = function (psql, callback) { column: this.column }); + debug(dateIntervalQuery); + psql.query(dateIntervalQuery, function (err, result) { if (err) { return callback(err); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 857aaa07..c5927d52 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -116,7 +116,7 @@ describe('histogram-dataview for date column type', function() { "type": "cartodb", "options": { "source": { - "id": "date-histogram-source" + "id": "datetime-histogram-source" }, "cartocss": "#points { marker-width: 10; marker-fill: red; }", "cartocss_version": "2.3.0" @@ -124,9 +124,9 @@ describe('histogram-dataview for date column type', function() { } ], { - date_histogram: { + datetime_histogram: { source: { - id: 'date-histogram-source' + id: 'datetime-histogram-source' }, type: 'histogram', options: { @@ -135,9 +135,9 @@ describe('histogram-dataview for date column type', function() { timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, - date_histogram_tz: { + datetime_histogram_tz: { source: { - id: 'date-histogram-source-tz' + id: 'datetime-histogram-source-tz' }, type: 'histogram', options: { @@ -146,6 +146,25 @@ describe('histogram-dataview for date column type', function() { timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, + datetime_histogram_automatic: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd' + } + }, + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'year' + } + }, date_histogram_automatic: { source: { id: 'date-histogram-source' @@ -158,7 +177,7 @@ describe('histogram-dataview for date column type', function() { }, [ { - "id": "date-histogram-source", + "id": "datetime-histogram-source", "type": "source", "params": { "query": [ @@ -170,7 +189,7 @@ describe('histogram-dataview for date column type', function() { } }, { - "id": "date-histogram-source-tz", + "id": "datetime-histogram-source-tz", "type": "source", "params": { "query": [ @@ -180,16 +199,28 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date::date AS d", + "from generate_series(", + "'2007-02-15'::date, '2008-04-09'::date, '1 day'::interval", + ") date" + ].join(' ') + } } ] ); var dateHistogramsUseCases = [{ desc: 'supporting timestamp with timezone', - dataviewId: 'date_histogram_tz' + dataviewId: 'datetime_histogram_tz' }, { desc: 'supporting timestamp without timezone', - dataviewId: 'date_histogram' + dataviewId: 'datetime_histogram' }]; dateHistogramsUseCases.forEach(function (test) { @@ -433,7 +464,34 @@ describe('histogram-dataview for date column type', function() { }); }); - it('automatic mode', function (done) { + it('should find the best aggregation (automatic mode) to build the histogram', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should work with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'year'); + assert.equal(dataview.bins.length, 2); + assert.equal(dataview.bins_count, 2); + done(); + }); + }); + + + it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) { var params = {}; this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { From 468f641af8bf9896ed198f585d0d2b64c8f885ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 29 Jun 2017 16:57:27 +0200 Subject: [PATCH 32/44] Going green: automatic mode works with dates --- lib/cartodb/models/dataview/histogram.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index b8b9085a..4499dd6c 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -14,8 +14,8 @@ var dateIntervalQueryTpl = dot.template([ 'WITH', 'dates AS (', ' SELECT', - ' MAX({{=it.column}}) AS _end,', - ' MIN({{=it.column}}) AS _start', + ' MAX({{=it.column}}::timestamp) AS _end,', + ' MIN({{=it.column}}::timestamp) AS _start', ' FROM ({{=it.query}}) _cdb_source', '),', 'interval_in_days AS (', From 967dca95789595a136ba91592c3ab296629baa85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 12 Jul 2017 15:10:39 +0200 Subject: [PATCH 33/44] Improve readability --- lib/cartodb/models/dataview/histogram.js | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4499dd6c..8d624163 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -159,11 +159,15 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' max({{=it._end}}) AS max_val,', ' min({{=it._start}}) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', - ' min(date_trunc(', - ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', - ' )) AS start_date,', - ' max(TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', + ' min(', + ' date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' )', + ' ) AS start_date,', + ' max(', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' ) AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -177,15 +181,11 @@ var dateBinsQueryTpl = dot.template([ ' FROM (', ' SELECT', ' ARRAY(', - ' {{?it._aggregation==="quarter"}}', - ' SELECT GENERATE_SERIES(start_date::timestamptz, end_date::timestamptz, \'3 month\'::interval)', - ' {{??}}', ' SELECT GENERATE_SERIES(', ' start_date::timestamptz,', ' end_date::timestamptz,', - ' \'1 {{=it._aggregation}}\'::interval', + ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', ' )', - ' {{?}}', ' ) AS bins_array', ' FROM basics', ' ) _cdb_bins_array', @@ -207,10 +207,14 @@ var dateHistogramQueryTpl = dot.template([ ' bins_number', ' )) - 1', ' END AS bin,', - ' min(date_part(', - ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}}::timestamptz', - ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', + ' min(', + ' date_part(', + ' \'epoch\', ', + ' date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', + ' ) AT TIME ZONE \'{{=it._timezone}}\'', + ' )', + ' )::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', From b67f6053e826c3dd047c245af87d7cb4a7e59e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 12 Jul 2017 15:14:41 +0200 Subject: [PATCH 34/44] Should respect first timestamp as bin_start --- lib/cartodb/models/dataview/histogram.js | 6 +-- test/acceptance/dataviews/histogram.js | 48 +++++++++++++++++++++++- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 8d624163..212cf3bb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -562,10 +562,10 @@ function getTimezone(timezone) { function populateBinStart(override, firstRow) { var binStart; - if (override.hasOwnProperty('start')) { - binStart = getBinStart(override); - } else if (firstRow.hasOwnProperty('timestamp')) { + if (firstRow.hasOwnProperty('timestamp')) { binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); } else { binStart = firstRow.min; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c5927d52..e33cb2c0 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -173,6 +173,16 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd' } + }, + minute_histogram: { + source: { + id: 'minute-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'minute' + } } }, [ @@ -211,6 +221,18 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "minute-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", + ") date" + ].join(' ') + } } ] ); @@ -282,8 +304,8 @@ describe('histogram-dataview for date column type', function() { it('should override start and end ' + test.desc, function (done) { var params = { - start: 1180659600, // 2007-06-01 01:00:00 - end: 1193792400 // 2007-10-31 01:00:00 + start: 1180659600, // 2007-06-01 01:00:00 UTC => '2007-05-31T21:00:00-04:00' + end: 1193792400 // 2007-10-31 01:00:00 UTC }; this.testClient = new TestClient(mapConfig, 1234); @@ -300,6 +322,28 @@ describe('histogram-dataview for date column type', function() { }); }); + + it('should return same histogram ' + test.desc, function (done) { + var params = { + start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) + end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) { + assert.ok(!err, err); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) { + assert.ok(!err, err); + + assert.deepEqual(dataview, filteredDataview); + done(); + }); + }); + }); + + it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) From c8286233be6393d7ca18ec04c06bc13c5a7c97c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 12 Jul 2017 17:08:55 +0200 Subject: [PATCH 35/44] Do not apply timezone for minute an hour aggregations --- lib/cartodb/models/dataview/histogram.js | 13 ++- test/acceptance/dataviews/histogram.js | 133 +++++++++++++++++++++++ 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 212cf3bb..dd3b2353 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -403,14 +403,14 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _aggregation: _aggregation, _start: getBinStart(override), _end: getBinEnd(override), - _timezone: getTimezone(_timezone) + _timezone: getTimezone(_timezone, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone) + _timezone: getTimezone(_timezone, _aggregation) }); } @@ -429,7 +429,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone) + _timezone: getTimezone(_timezone, _aggregation) }); var histogramSql = [ @@ -551,10 +551,15 @@ function getWidth(override) { return width; } -function getTimezone(timezone) { +function getTimezone(timezone, aggregation) { if (!timezone) { return '0'; } + + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + var timezoneInHours = Math.ceil(timezone / 3600); return '' + timezoneInHours; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index e33cb2c0..790dbad4 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -547,4 +547,137 @@ describe('histogram-dataview for date column type', function() { done(); }); }); + + it('should not apply timezone for a histogram aggregated by minutes', function (done) { + var self = this; + var params = { + timezone: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + + self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { + assert.ifError(err); + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithTimezone) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithTimezone); + done(); + }); + }); + }); + + it('should filter by "start" & "end" for a histogram aggregated by minutes', function (done) { + var self = this; + var paramsWithFilter = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) + }; + + var paramsWithTimezone = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) + timezone: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, dataview) { + assert.ifError(err); + + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, filteredDataview) { + assert.ifError(err); + + assert.deepEqual(dataview, filteredDataview); + + self.testClient.getDataview('minute_histogram', paramsWithTimezone, + function (err, filteredWithTimezoneDataview) { + assert.ifError(err); + + assert.deepEqual(filteredDataview, filteredWithTimezoneDataview); + done(); + }); + }); + }); + }); + + + it('should return an histogram aggregated by days', function (done) { + var self = this; + var paramsWithDailyAgg = { + aggregation: 'day', + }; + + // data: from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggFixture = { + aggregation: 'day', + bin_width: 600, + bins_count: 2, + bins_start: 1171497600, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171497600, + min: 1171583400, + max: 1171583940, + avg: 1171583670, + freq: 10 + }, + { + bin: 1, + timestamp: 1171584000, + min: 1171584000, + max: 1171584600, + avg: 1171584300, + freq: 11 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAgg, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggFixture); + done(); + }); + }); + + it('should return a histogram aggregated by days with timezone', function (done) { + var self = this; + + var paramsWithDailyAggAndTimezone = { + aggregation: 'day', + timezone: '-3600' + }; + + // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggAndTimezoneFixture = { + aggregation: 'day', + bin_width: 1200, + bins_count: 1, + bins_start: 1171501200, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171501200, + min: 1171583400, + max: 1171584600, + avg: 1171584000, + freq: 21 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndTimezone, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggAndTimezoneFixture); + done(); + }); + }); }); From 21b8655f8568fbdfd8ca187bf22437a821102e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 13 Jul 2017 19:42:22 +0200 Subject: [PATCH 36/44] Return timezone applied or 0 if not present --- lib/cartodb/models/dataview/histogram.js | 18 +++++++++++++++++- test/acceptance/dataviews/histogram.js | 8 +++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dd3b2353..de5a66da 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -491,7 +491,8 @@ Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; - var aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var aggregation = getAggregation(override, this.aggregation); + var timezone = getTimezoneParam(override, this.timezone); var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); @@ -513,6 +514,7 @@ Histogram.prototype.format = function(result, override) { return { aggregation: aggregation, + timezone: timezone, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -522,6 +524,20 @@ Histogram.prototype.format = function(result, override) { }; }; +function getAggregation(override, aggregation) { + return override && override.aggregation ? override.aggregation : aggregation; +} + +function getTimezoneParam(override, timezone) { + if (override && override.timezone) { + return override.timezone; + } + if (timezone) { + return timezone; + } + return 0; +} + function getBinStart(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 790dbad4..233eed15 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -561,6 +561,8 @@ describe('histogram-dataview for date column type', function() { self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithTimezone) { assert.ifError(err); + assert.notEqual(dataview.timezone, dataviewWithTimezone.timezone); + dataview.timezone = dataviewWithTimezone.timezone; assert.deepEqual(dataview, dataviewWithTimezone); done(); }); @@ -593,7 +595,9 @@ describe('histogram-dataview for date column type', function() { function (err, filteredWithTimezoneDataview) { assert.ifError(err); - assert.deepEqual(filteredDataview, filteredWithTimezoneDataview); + assert.notEqual(filteredWithTimezoneDataview.timezone, filteredDataview.timezone); + filteredWithTimezoneDataview.timezone = filteredDataview.timezone; + assert.deepEqual(filteredWithTimezoneDataview, filteredDataview); done(); }); }); @@ -614,6 +618,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 600, bins_count: 2, bins_start: 1171497600, + timezone: 0, nulls: 0, bins: [{ @@ -660,6 +665,7 @@ describe('histogram-dataview for date column type', function() { bins_count: 1, bins_start: 1171501200, nulls: 0, + timezone: -3600, bins: [{ bin: 0, From 0fffafa1db87ad27e85aac675a39ef138cacedb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 14 Jul 2017 18:22:05 +0200 Subject: [PATCH 37/44] Add timestamp_start in histogram summary to help to build the histogram in frontend side --- lib/cartodb/models/dataview/histogram.js | 23 ++++++++++++++--------- test/acceptance/dataviews/histogram.js | 2 ++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index de5a66da..9d934b61 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -215,13 +215,14 @@ var dateHistogramQueryTpl = dot.template([ ' ) AT TIME ZONE \'{{=it._timezone}}\'', ' )', ' )::numeric AS timestamp,', + ' date_part(\'epoch\', start_date)::numeric AS timestamp_start,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', ' count(*) AS freq', 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', 'ORDER BY bin' ].join('\n')); @@ -403,14 +404,14 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _aggregation: _aggregation, _start: getBinStart(override), _end: getBinEnd(override), - _timezone: getTimezone(_timezone, _aggregation) + _timezone: parseTimezone(_timezone, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _timezone: parseTimezone(_timezone, _aggregation) }); } @@ -429,7 +430,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _timezone: parseTimezone(_timezone, _aggregation) }); var histogramSql = [ @@ -492,12 +493,13 @@ Histogram.prototype.format = function(result, override) { var buckets = []; var aggregation = getAggregation(override, this.aggregation); - var timezone = getTimezoneParam(override, this.timezone); + var timezone = getTimezone(override, this.timezone); var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); var nulls = 0; var avg; + var timestampStart; if (result.rows.length) { var firstRow = result.rows[0]; @@ -506,15 +508,16 @@ Histogram.prototype.format = function(result, override) { avg = firstRow.avg_val; nulls = firstRow.nulls_count; binsStart = populateBinStart(override, firstRow); - + timestampStart = firstRow.timestamp_start; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val', 'timestamp_start'); }); } return { aggregation: aggregation, timezone: timezone, + timestamp_start: timestampStart, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -528,13 +531,15 @@ function getAggregation(override, aggregation) { return override && override.aggregation ? override.aggregation : aggregation; } -function getTimezoneParam(override, timezone) { +function getTimezone(override, timezone) { if (override && override.timezone) { return override.timezone; } + if (timezone) { return timezone; } + return 0; } @@ -567,7 +572,7 @@ function getWidth(override) { return width; } -function getTimezone(timezone, aggregation) { +function parseTimezone(timezone, aggregation) { if (!timezone) { return '0'; } diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 233eed15..4efc9345 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -618,6 +618,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 600, bins_count: 2, bins_start: 1171497600, + timestamp_start: 1171497600, timezone: 0, nulls: 0, bins: @@ -664,6 +665,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 1200, bins_count: 1, bins_start: 1171501200, + timestamp_start: 1171497600, nulls: 0, timezone: -3600, bins: From 70b4d5b7fda6d4e4b362d32c64d2f3bb8a8b52e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Matall=C3=ADn?= Date: Fri, 14 Jul 2017 18:30:36 +0200 Subject: [PATCH 38/44] replace timezone => offset --- lib/cartodb/backends/dataview.js | 2 +- lib/cartodb/controllers/layergroup.js | 2 +- lib/cartodb/models/dataview/histogram.js | 46 +++++----- test/acceptance/dataviews/histogram.js | 106 +++++++++++------------ test/support/test-client.js | 2 +- 5 files changed, 79 insertions(+), 79 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index e54b3a83..5fe1745a 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -104,7 +104,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { } function getOverrideParams(params, ownFilter) { - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'timezone'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index fcab20d2..9644aa06 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -89,7 +89,7 @@ LayergroupController.prototype.register = function(app) { 'column_type', // string 'bins', // number 'aggregation', //string - 'timezone', // number + 'offset', // number 'q' // widgets search ]; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index de5a66da..a0814095 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -145,9 +145,9 @@ var dateBasicsQueryTpl = dot.template([ ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'', ' )) AS start_date,', - ' max({{=it._column}} AT TIME ZONE \'{{=it._timezone}}\') AS end_date,', + ' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', ')' @@ -162,11 +162,11 @@ var dateOverrideBasicsQueryTpl = dot.template([ ' min(', ' date_trunc(', ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' )', ' ) AS start_date,', ' max(', - ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._timezone}}\'', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', ' ) AS end_date,', ' count(1) AS total_rows', ' FROM ({{=it._query}}) _cdb_basics', @@ -201,7 +201,7 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\',', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', ' bins_array', ' ),', ' bins_number', @@ -212,7 +212,7 @@ var dateHistogramQueryTpl = dot.template([ ' \'epoch\', ', ' date_trunc(', ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', - ' ) AT TIME ZONE \'{{=it._timezone}}\'', + ' ) AT TIME ZONE \'{{=it._offset}}\'', ' )', ' )::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', @@ -243,7 +243,7 @@ Time series: options: { column: 'date', // column data type: date aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) - timezone: -7200 // OPTIONAL (UTC offset in seconds) + offset: -7200 // OPTIONAL (UTC offset in seconds) } } */ @@ -257,7 +257,7 @@ function Histogram(query, options, queries) { this.column = options.column; this.bins = options.bins; this.aggregation = options.aggregation; - this.timezone = options.timezone; + this.offset = options.offset; this._columnType = null; } @@ -380,7 +380,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _timezone = override && Number.isFinite(override.timezone) ? override.timezone : this.timezone; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; if (!_aggregation) { this.getAutomaticAggregation(psql, function (err, aggregation) { @@ -403,14 +403,14 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _aggregation: _aggregation, _start: getBinStart(override), _end: getBinEnd(override), - _timezone: getTimezone(_timezone, _aggregation) + _offset: getOffset(_offset, _aggregation) }); } else { dateBasicsQuery = dateBasicsQueryTpl({ _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _offset: getOffset(_offset, _aggregation) }); } @@ -429,7 +429,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac _query: _query, _column: _column, _aggregation: _aggregation, - _timezone: getTimezone(_timezone, _aggregation) + _offset: getOffset(_offset, _aggregation) }); var histogramSql = [ @@ -492,7 +492,7 @@ Histogram.prototype.format = function(result, override) { var buckets = []; var aggregation = getAggregation(override, this.aggregation); - var timezone = getTimezoneParam(override, this.timezone); + var offset = getOffsetParam(override, this.offset); var binsCount = getBinsCount(override); var width = getWidth(override); var binsStart = getBinStart(override); @@ -514,7 +514,7 @@ Histogram.prototype.format = function(result, override) { return { aggregation: aggregation, - timezone: timezone, + offset: offset, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -528,12 +528,12 @@ function getAggregation(override, aggregation) { return override && override.aggregation ? override.aggregation : aggregation; } -function getTimezoneParam(override, timezone) { - if (override && override.timezone) { - return override.timezone; +function getOffsetParam(override, offset) { + if (override && override.offset) { + return override.offset; } - if (timezone) { - return timezone; + if (offset) { + return offset; } return 0; } @@ -567,8 +567,8 @@ function getWidth(override) { return width; } -function getTimezone(timezone, aggregation) { - if (!timezone) { +function getOffset(offset, aggregation) { + if (!offset) { return '0'; } @@ -576,8 +576,8 @@ function getTimezone(timezone, aggregation) { return '0'; } - var timezoneInHours = Math.ceil(timezone / 3600); - return '' + timezoneInHours; + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; } function populateBinStart(override, firstRow) { diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 233eed15..520c0fa7 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -132,7 +132,7 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd', aggregation: 'month', - timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, datetime_histogram_tz: { @@ -143,7 +143,7 @@ describe('histogram-dataview for date column type', function() { options: { column: 'd', aggregation: 'month', - timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } }, datetime_histogram_automatic: { @@ -238,16 +238,16 @@ describe('histogram-dataview for date column type', function() { ); var dateHistogramsUseCases = [{ - desc: 'supporting timestamp with timezone', + desc: 'supporting timestamp with offset', dataviewId: 'datetime_histogram_tz' }, { - desc: 'supporting timestamp without timezone', + desc: 'supporting timestamp without offset', dataviewId: 'datetime_histogram' }]; dateHistogramsUseCases.forEach(function (test) { it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { - var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes this.testClient = new TestClient(mapConfig, 1234); @@ -260,18 +260,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .utcOffset(OFFSET_EDT_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function(bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .utcOffset(OFFSET_EDT_IN_MINUTES) .add(index, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .utcOffset(OFFSET_EDT_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -344,11 +344,11 @@ describe('histogram-dataview for date column type', function() { }); - it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { - var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) - var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { + var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) var params = { - timezone: TIMEZONE_CEST_IN_SECONDS + offset: OFFSET_CEST_IN_SECONDS }; this.testClient = new TestClient(mapConfig, 1234); @@ -361,18 +361,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .utcOffset(OFFSET_CEST_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function (bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .utcOffset(OFFSET_CEST_IN_MINUTES) .add(index, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .utcOffset(OFFSET_CEST_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -384,11 +384,11 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should aggregate histogram overriding default timezone to UTC/GMT ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + it('should aggregate histogram overriding default offset to UTC/GMT ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS + offset: OFFSET_UTC_IN_SECONDS }; this.testClient = new TestClient(mapConfig, 1234); @@ -401,18 +401,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function (bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .add(index, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -425,10 +425,10 @@ describe('histogram-dataview for date column type', function() { }); it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, + offset: OFFSET_UTC_IN_SECONDS, aggregation: 'quarter' }; @@ -442,18 +442,18 @@ describe('histogram-dataview for date column type', function() { var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight var binsStartInMilliseconds = dataview.bins_start * 1000; var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binsStartFormatted, initialTimestamp); dataview.bins.forEach(function (bin, index) { var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .add(index * 3, 'month') .format(); var binsTimestampInMilliseconds = bin.timestamp * 1000; var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .utcOffset(OFFSET_UTC_IN_MINUTES) .format(); assert.equal(binTimestampFormatted, binTimestampExpected); @@ -466,9 +466,9 @@ describe('histogram-dataview for date column type', function() { }); it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, + offset: OFFSET_UTC_IN_SECONDS, aggregation: 'quarter', start: 1167609600, // 2007-01-01T00:00:00Z, first bin start end: 1214870399 // 2008-06-30T23:59:59Z, last bin end @@ -487,9 +487,9 @@ describe('histogram-dataview for date column type', function() { }); it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, + offset: OFFSET_UTC_IN_SECONDS, aggregation: 'quarter', start: 1167609600, // 2007-01-01T00:00:00Z, first bin start end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last @@ -548,22 +548,22 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should not apply timezone for a histogram aggregated by minutes', function (done) { + it('should not apply offset for a histogram aggregated by minutes', function (done) { var self = this; var params = { - timezone: '-3600' + offset: '-3600' }; self.testClient = new TestClient(mapConfig, 1234); self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { assert.ifError(err); - self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithTimezone) { + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithOffset) { assert.ifError(err); - assert.notEqual(dataview.timezone, dataviewWithTimezone.timezone); - dataview.timezone = dataviewWithTimezone.timezone; - assert.deepEqual(dataview, dataviewWithTimezone); + assert.notEqual(dataview.offset, dataviewWithOffset.offset); + dataview.offset = dataviewWithOffset.offset; + assert.deepEqual(dataview, dataviewWithOffset); done(); }); }); @@ -576,10 +576,10 @@ describe('histogram-dataview for date column type', function() { end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) }; - var paramsWithTimezone = { + var paramsWithOffset = { start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) - timezone: '-3600' + offset: '-3600' }; self.testClient = new TestClient(mapConfig, 1234); @@ -591,13 +591,13 @@ describe('histogram-dataview for date column type', function() { assert.deepEqual(dataview, filteredDataview); - self.testClient.getDataview('minute_histogram', paramsWithTimezone, - function (err, filteredWithTimezoneDataview) { + self.testClient.getDataview('minute_histogram', paramsWithOffset, + function (err, filteredWithOffsetDataview) { assert.ifError(err); - assert.notEqual(filteredWithTimezoneDataview.timezone, filteredDataview.timezone); - filteredWithTimezoneDataview.timezone = filteredDataview.timezone; - assert.deepEqual(filteredWithTimezoneDataview, filteredDataview); + assert.notEqual(filteredWithOffsetDataview.offset, filteredDataview.offset); + filteredWithOffsetDataview.offset = filteredDataview.offset; + assert.deepEqual(filteredWithOffsetDataview, filteredDataview); done(); }); }); @@ -618,7 +618,7 @@ describe('histogram-dataview for date column type', function() { bin_width: 600, bins_count: 2, bins_start: 1171497600, - timezone: 0, + offset: 0, nulls: 0, bins: [{ @@ -649,23 +649,23 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should return a histogram aggregated by days with timezone', function (done) { + it('should return a histogram aggregated by days with offset', function (done) { var self = this; - var paramsWithDailyAggAndTimezone = { + var paramsWithDailyAggAndOffset = { aggregation: 'day', - timezone: '-3600' + offset: '-3600' }; // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 - var dataviewWithDailyAggAndTimezoneFixture = { + var dataviewWithDailyAggAndOffsetFixture = { aggregation: 'day', bin_width: 1200, bins_count: 1, bins_start: 1171501200, nulls: 0, - timezone: -3600, + offset: -3600, bins: [{ bin: 0, @@ -679,10 +679,10 @@ describe('histogram-dataview for date column type', function() { }; self.testClient = new TestClient(mapConfig, 1234); - self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndTimezone, function (err, dataview) { + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndOffset, function (err, dataview) { assert.ifError(err); - assert.deepEqual(dataview, dataviewWithDailyAggAndTimezoneFixture); + assert.deepEqual(dataview, dataviewWithDailyAggAndOffsetFixture); done(); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index dc36929e..76219990 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -369,7 +369,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end', 'aggregation', 'timezone'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } From 90345082441fc88e6cad11c52672430638ab3320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Aug 2017 17:15:45 +0200 Subject: [PATCH 39/44] Support automattic aggregation only when aggregation para is set to 'auto' --- lib/cartodb/backends/dataview.js | 1 + lib/cartodb/models/dataview/histogram.js | 10 ++++++++-- test/acceptance/dataviews/histogram.js | 6 ++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 5fe1745a..0c9a30fe 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -21,6 +21,7 @@ function DataviewBackend(analysisBackend) { } var DATE_AGGREGATIONS = { + 'auto': true, 'minute': true, 'hour': true, 'day': true, diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 7f2d13d7..436a0368 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,6 +5,8 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); + var dateIntervalQueryTpl = dot.template([ 'WITH', 'dates AS (', @@ -331,10 +333,14 @@ Histogram.prototype._buildQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; - if (this._columnType === 'date') { + if (this._columnType === 'date' && this.aggregation !== undefined) { return this._buildDateHistogramQuery(psql, override, callback); } + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } + filteredQuery = filteredQueryTpl({ _isFloatColumn: this._columnType === 'float', _query: _query, @@ -434,7 +440,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - if (!_aggregation) { + if (_aggregation === 'auto') { this.getAutomaticAggregation(psql, function (err, aggregation) { if (err || aggregation === 'none') { this.aggregation = 'day'; diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 58084aa4..4d48ab33 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -160,7 +160,8 @@ describe('histogram-dataview for date column type', function() { }, type: 'histogram', options: { - column: 'd' + column: 'd', + aggregation: 'auto' } }, date_histogram: { @@ -179,7 +180,8 @@ describe('histogram-dataview for date column type', function() { }, type: 'histogram', options: { - column: 'd' + column: 'd', + aggregation: 'auto' } }, minute_histogram: { From 0a57e86cb8d34a5e48e5e1a690e2fa06d58ad976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Aug 2017 12:06:10 +0200 Subject: [PATCH 40/44] Do not build data histogram infinitely when overriding aggregation with auto mode --- lib/cartodb/models/dataview/histogram.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 436a0368..4c01cc46 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -447,6 +447,7 @@ Histogram.prototype._buildDateHistogramQuery = function (psql, override, callbac } else { this.aggregation = aggregation; } + override.aggregation = this.aggregation; this._buildDateHistogramQuery(psql, override, callback); }.bind(this)); return null; From 7fed91900d952c18ee0588dacbc1c92a8eb8aec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 12:19:34 +0200 Subject: [PATCH 41/44] Handle error --- lib/cartodb/models/dataview/base.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index f8eb1027..280487b4 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -8,6 +8,10 @@ module.exports = BaseDataview; BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { + if (err) { + return callback(err); + } + psql.query(query, function(err, result) { if (err) { From 90ded34af78df9c763836530aec637fd0e3c74dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 12:22:30 +0200 Subject: [PATCH 42/44] Do not fail if layergroup is undefined --- test/support/test-client.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index 899c239f..3b9a003d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -404,8 +404,11 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }, function finish(err, dataview) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + return callback(err, dataview); } ); From b56d2ec30b584c0751ac469b700426524c5a7b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 12:24:05 +0200 Subject: [PATCH 43/44] Validate aggregation value --- lib/cartodb/backends/dataview.js | 14 +-- lib/cartodb/models/dataview/histogram.js | 17 ++++ test/acceptance/dataviews/histogram.js | 121 +++++++++++++++++++++++ yarn.lock | 12 +-- 4 files changed, 146 insertions(+), 18 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 0c9a30fe..29dcd903 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -20,17 +20,6 @@ function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; } -var DATE_AGGREGATIONS = { - 'auto': true, - 'minute': true, - 'hour': true, - 'day': true, - 'week': true, - 'month': true, - 'quarter': true, - 'year': true -}; - module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { @@ -116,7 +105,8 @@ function getOverrideParams(params, ownFilter) { {ownFilter: ownFilter} ); - if (params.aggregation && DATE_AGGREGATIONS.hasOwnProperty(params.aggregation)) { + // validation will be delegated to the proper dataview + if (params.aggregation !== undefined) { overrideParams.aggregation = params.aggregation; } diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 4c01cc46..13a9d998 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -434,12 +434,29 @@ Histogram.prototype._shouldOverrideBins = function (override) { return override && _.has(override, 'bins'); }; +var DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { var _column = this.column; var _query = this.query; var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + if (_aggregation === 'auto') { this.getAutomaticAggregation(psql, function (err, aggregation) { if (err || aggregation === 'none') { diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 4d48ab33..51808d67 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -766,3 +766,124 @@ describe('histogram-dataview: special float valuer', function() { }); }); }); + +describe('histogram-dates: aggregation input value', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'day' + } + }, + bad_agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'wadus' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'2007-02-15 01:00:00\'::timestamp,', + '\'2008-04-09 01:00:00\'::timestamp,', + ' \'1 day\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should fail when aggregation values is not valid while instantiating the map', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + response: { + status: 400 + } + }; + + this.testClient.getDataview('bad_agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); + + it('should fail when aggregation values is not valid while fetching dataview result', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + aggregation: 'wadus', + response: { + status: 400 + } + }; + + this.testClient.getDataview('agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index a835b0e0..49cbc74e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1288,7 +1288,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@^2.18.1: +moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -2103,7 +2103,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb3: version "2.3.1-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" dependencies: @@ -2111,7 +2111,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: From 431ca9c56ff9ab0470137ee6cb1b599f83df686f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Aug 2017 15:15:37 +0200 Subject: [PATCH 44/44] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index a3228af1..cb84fe39 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.9 Released 2017-mm-dd + - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698 + ## 3.9.8 Released 2017-07-21