From a6daca96283c8e304395d9068eb11a4bc4e0759a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 22 Jun 2017 18:04:23 +0200 Subject: [PATCH] Support date histograms using timestamp with and without timezones --- lib/cartodb/models/dataview/histogram.js | 12 +- test/acceptance/dataviews/histogram.js | 372 ++++++++++++----------- 2 files changed, 210 insertions(+), 174 deletions(-) diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index dbaa002f..3409e6fb 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -137,9 +137,13 @@ var dateBinsQueryTpl = dot.template([ ' SELECT', ' ARRAY(', ' {{?it._aggregation==="quarter"}}', - ' SELECT GENERATE_SERIES(start_date, end_date, \'3 month\'::interval)', + ' SELECT GENERATE_SERIES(start_date::timestamptz, end_date::timestamptz, \'3 month\'::interval)', ' {{??}}', - ' SELECT GENERATE_SERIES(start_date, end_date, \'1 {{=it._aggregation}}\'::interval)', + ' SELECT GENERATE_SERIES(', + ' start_date::timestamptz,', + ' end_date::timestamptz,', + ' \'1 {{=it._aggregation}}\'::interval', + ' )', ' {{?}}', ' ) AS bins_array', ' FROM basics', @@ -156,7 +160,7 @@ var dateHistogramQueryTpl = dot.template([ ' THEN 0', ' ELSE GREATEST(1, LEAST(', ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp WITHOUT TIME ZONE AT TIME ZONE \'{{=it._timezone}}\',', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._timezone}}\',', ' bins_array', ' ),', ' bins_number', @@ -164,7 +168,7 @@ var dateHistogramQueryTpl = dot.template([ ' END AS bin,', ' min(date_part(', ' \'epoch\', ', - ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._timezone}}\'', + ' date_trunc(\'{{=it._aggregation}}\', {{=it._column}}::timestamptz', ' ) AT TIME ZONE \'{{=it._timezone}}\'))::numeric AS timestamp,', ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 0efb56d6..f39db632 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -134,6 +134,17 @@ describe('histogram-dataview for date column type', function() { aggregation: 'month', timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds } + }, + date_histogram_tz: { + source: { + id: 'date-histogram-source-tz' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + timezone: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } } }, [ @@ -148,205 +159,226 @@ describe('histogram-dataview for date column type', function() { ") date" ].join(' ') } + }, + { + "id": "date-histogram-source-tz", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + ") date" + ].join(' ') + } } ] ); - it('should create a date histogram aggregated in months (EDT)', function (done) { - var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with timezone', + dataviewId: 'date_histogram_tz' + }, { + desc: 'supporting timestamp without timezone', + dataviewId: 'date_histogram' + }]; - this.testClient = new TestClient(mapConfig, 1234); + dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { + var TIMEZONE_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes - this.testClient.getDataview('date_histogram', {}, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); + this.testClient = new TestClient(mapConfig, 1234); - var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); + this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_EDT_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_EDT_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should override aggregation in weeks', function (done) { - var params = { - aggregation: 'week' - }; + it('should override aggregation in weeks ' + test.desc, function (done) { + var params = { + aggregation: 'week' + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 61); - dataview.bins.forEach(function(bin) { - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should override start and end', function (done) { - var params = { - start: 1180659600, // 2007-06-01 01:00:00 - end: 1193792400 // 2007-10-31 01:00:00 - }; + it('should override start and end ' + test.desc, function (done) { + var params = { + start: 1180659600, // 2007-06-01 01:00:00 + end: 1193792400 // 2007-10-31 01:00:00 + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); - dataview.bins.forEach(function(bin) { - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); - - done(); }); - }); - it('should aggregate histogram overriding default timezone to CEST', function (done) { - var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) - var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) - var params = { - timezone: TIMEZONE_CEST_IN_SECONDS - }; + it('should aggregate histogram overriding default timezone to CEST ' + test.desc, function (done) { + var TIMEZONE_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var TIMEZONE_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + var params = { + timezone: TIMEZONE_CEST_IN_SECONDS + }; - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); - var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_CEST_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) .utcOffset(TIMEZONE_CEST_IN_MINUTES) .format(); + assert.equal(binsStartFormatted, initialTimestamp); - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_CEST_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); }); + }); - done(); + it('should aggregate histogram overriding default timezone to UTC/GMT ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { + var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC + var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + timezone: TIMEZONE_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(TIMEZONE_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); }); }); - - it('should aggregate histogram overriding default timezone to UTC/GMT', function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC - var params = { - timezone: TIMEZONE_UTC_IN_SECONDS - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 15); - - var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .add(index, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); - }); - - done(); - }); - }); - - it('should aggregate histogram using "quarter" aggregation', function (done) { - var TIMEZONE_UTC_IN_SECONDS = 0 * 3600; // UTC - var TIMEZONE_UTC_IN_MINUTES = 0 * 60; // UTC - var params = { - timezone: TIMEZONE_UTC_IN_SECONDS, - aggregation: 'quarter' - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview('date_histogram', params, function(err, dataview) { - assert.ok(!err, err); - assert.equal(dataview.type, 'histogram'); - assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); - - var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight - var binsStartInMilliseconds = dataview.bins_start * 1000; - var binsStartFormatted = moment.utc(binsStartInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - assert.equal(binsStartFormatted, initialTimestamp); - - dataview.bins.forEach(function(bin, index) { - var binTimestampExpected = moment.utc(initialTimestamp) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .add(index * 3, 'month') - .format(); - var binsTimestampInMilliseconds = bin.timestamp * 1000; - var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) - .utcOffset(TIMEZONE_UTC_IN_MINUTES) - .format(); - - assert.equal(binTimestampFormatted, binTimestampExpected); - assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); - assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); - }); - - done(); - }); - }); - });