diff --git a/NEWS.md b/NEWS.md index a3228af1..cb84fe39 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 3.9.9 Released 2017-mm-dd + - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698 + ## 3.9.8 Released 2017-07-21 diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index a77ba585..29dcd903 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -94,7 +94,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { } function getOverrideParams(params, ownFilter) { - return _.reduce(_.pick(params, 'start', 'end', 'bins'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); @@ -104,6 +104,13 @@ function getOverrideParams(params, ownFilter) { }, {ownFilter: ownFilter} ); + + // validation will be delegated to the proper dataview + if (params.aggregation !== undefined) { + overrideParams.aggregation = params.aggregation; + } + + return overrideParams; } DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 7cdd0713..5aab7cb3 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -17,16 +17,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'zoom', 'lon', 'lat', - // widgets & filters - 'filters', // json - 'own_filter', // 0, 1 - 'bbox', // w,s,e,n - 'bins', // number - 'start', // number - 'end', // number - 'column_type', // string - // widgets search - 'q' + // analysis + 'filters' // json ]; function BaseController(authApi, pgConnection) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index e0c7a7a3..9644aa06 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -79,19 +79,51 @@ LayergroupController.prototype.register = function(app) { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); + var allowedDataviewQueryParams = [ + 'filters', // json + 'own_filter', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'offset', // number + 'q' // widgets search + ]; + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); app.get(app.base_url_mapconfig + '/:token/analysis/node/:nodeId', cors(), userMiddleware, diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index f8eb1027..280487b4 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -8,6 +8,10 @@ module.exports = BaseDataview; BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { + if (err) { + return callback(err); + } + psql.query(query, function(err, result) { if (err) { diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 73cc8724..13a9d998 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -7,6 +7,47 @@ dot.templateSettings.strip = false; var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var dateIntervalQueryTpl = dot.template([ + 'WITH', + 'dates AS (', + ' SELECT', + ' MAX({{=it.column}}::timestamp) AS _end,', + ' MIN({{=it.column}}::timestamp) AS _start', + ' FROM ({{=it.query}}) _cdb_source', + '),', + 'interval_in_days AS (', + ' SELECT' , + ' DATE_PART(\'day\', _end - _start) AS days', + ' FROM dates', + '),', + 'interval_in_hours AS (', + ' SELECT', + ' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours', + ' FROM interval_in_days, dates', + '),', + 'interval_in_minutes AS (', + ' SELECT', + ' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes', + ' FROM interval_in_hours, dates', + '),', + 'interval_in_seconds AS (', + ' SELECT', + ' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds', + ' FROM interval_in_minutes, dates', + ')', + 'SELECT', + ' ROUND(days / 365) AS year,', + ' ROUND(days / 90) AS quarter,', + ' ROUND(days / 30) AS month,', + ' ROUND(days / 7) AS week,', + ' days AS day,', + ' hours AS hour,', + ' minutes AS minute,', + ' seconds AS second', + 'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds' +].join('\n')); + +var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; @@ -83,7 +124,7 @@ var overrideBinsQueryTpl = dot.template([ var nullsQueryTpl = dot.template([ 'nulls AS (', ' SELECT', - ' count(*) AS nulls_count', + ' count(*) AS nulls_count', ' FROM ({{=it._query}}) _cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' @@ -132,16 +173,114 @@ var histogramQueryTpl = dot.template([ 'ORDER BY bin' ].join('\n')); +var dateBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max(date_part(\'epoch\', {{=it._column}})) AS max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'', + ' )) AS start_date,', + ' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var dateOverrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val,', + ' min({{=it._start}}) AS min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,', + ' min(', + ' date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' )', + ' ) AS start_date,', + ' max(', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' ) AS end_date,', + ' count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var dateBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT', + ' bins_array,', + ' ARRAY_LENGTH(bins_array, 1) AS bins_number', + ' FROM (', + ' SELECT', + ' ARRAY(', + ' SELECT GENERATE_SERIES(', + ' start_date::timestamptz,', + ' end_date::timestamptz,', + ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', + ' )', + ' ) AS bins_array', + ' FROM basics', + ' ) _cdb_bins_array', + ')' +].join('\n')); + +var dateHistogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(', + ' WIDTH_BUCKET(', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', + ' bins_array', + ' ),', + ' bins_number', + ' )) - 1', + ' END AS bin,', + ' min(', + ' date_part(', + ' \'epoch\', ', + ' date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz', + ' ) AT TIME ZONE \'{{=it._offset}}\'', + ' )', + ' )::numeric AS timestamp,', + ' date_part(\'epoch\', start_date)::numeric AS timestamp_start,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', + ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls', + 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date', + 'ORDER BY bin' +].join('\n')); var TYPE = 'histogram'; /** - { - type: 'histogram', - options: { - column: 'name', - bins: 10 // OPTIONAL - } +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} + +Time series: +{ + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) + offset: -7200 // OPTIONAL (UTC offset in seconds) + } } */ function Histogram(query, options, queries) { @@ -153,6 +292,8 @@ function Histogram(query, options, queries) { this.queries = queries; this.column = options.column; this.bins = options.bins; + this.aggregation = options.aggregation; + this.offset = options.offset; this._columnType = null; } @@ -184,16 +325,18 @@ Histogram.prototype.sql = function(psql, override, callback) { return null; } - var histogramSql = this._buildQuery(override); - - return callback(null, histogramSql); + this._buildQuery(psql, override, callback); }; -Histogram.prototype._buildQuery = function (override) { +Histogram.prototype._buildQuery = function (psql, override, callback) { var filteredQuery, basicsQuery, binsQuery; var _column = this.column; var _query = this.query; + if (this._columnType === 'date' && this.aggregation !== undefined) { + return this._buildDateHistogramQuery(psql, override, callback); + } + if (this._columnType === 'date') { _column = columnCastTpl({column: _column}); } @@ -280,7 +423,7 @@ Histogram.prototype._buildQuery = function (override) { debug(histogramSql); - return histogramSql; + return callback(null, histogramSql); }; Histogram.prototype._shouldOverride = function (override) { @@ -291,6 +434,135 @@ Histogram.prototype._shouldOverrideBins = function (override) { return override && _.has(override, 'bins'); }; +var DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + +Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (_aggregation === 'auto') { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + + var dateBasicsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end')) { + dateBasicsQuery = dateOverrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _start: getBinStart(override), + _end: getBinEnd(override), + _offset: parseOffset(_offset, _aggregation) + }); + } else { + dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + } + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ + _query: _query, + _column: _column + }); + + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + +Histogram.prototype.getAutomaticAggregation = function (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + debug(dateIntervalQuery); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; @@ -302,6 +574,9 @@ Histogram.prototype.format = function(result, override) { var infinities = 0; var nans = 0; var avg; + var timestampStart; + var aggregation; + var offset; if (result.rows.length) { var firstRow = result.rows[0]; @@ -309,16 +584,34 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; + timestampStart = firstRow.timestamp_start; infinities = firstRow.infinities_count; nans = firstRow.nans_count; - binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; + binsStart = populateBinStart(override, firstRow); + + if (timestampStart) { + aggregation = getAggregation(override, this.aggregation); + offset = getOffset(override, this.offset); + } buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val'); + return _.omit( + row, + 'bins_number', + 'bin_width', + 'nulls_count', + 'infinities_count', + 'nans_count', + 'avg_val', + 'timestamp_start' + ); }); } return { + aggregation: aggregation, + offset: offset, + timestamp_start: timestampStart, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -330,6 +623,21 @@ Histogram.prototype.format = function(result, override) { }; }; +function getAggregation(override, aggregation) { + return override && override.aggregation ? override.aggregation : aggregation; +} + +function getOffset(override, offset) { + if (override && override.offset) { + return override.offset; + } + if (offset) { + return offset; + } + + return 0; +} + function getBinStart(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); @@ -359,6 +667,32 @@ function getWidth(override) { return width; } +function parseOffset(offset, aggregation) { + if (!offset) { + return '0'; + } + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; +} + +function populateBinStart(override, firstRow) { + var binStart; + + if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; +} + Histogram.prototype.getType = function() { return TYPE; }; diff --git a/package.json b/package.json index 08dc20cf..d6e95c12 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "istanbul": "~0.4.3", "jshint": "~2.9.4", "mocha": "~3.4.1", + "moment": "~2.18.1", "nock": "~2.11.0", "redis": "~0.12.1", "semver": "~1.1.4", diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c75e9baf..51808d67 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -2,6 +2,16 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +var moment = require('moment'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} function createMapConfig(layers, dataviews, analysis) { return { @@ -96,6 +106,598 @@ describe('histogram-dataview', function() { done(); }); }); + +}); + +describe('histogram-dataview for date column type', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "datetime-histogram-source" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + datetime_histogram: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } + }, + datetime_histogram_tz: { + source: { + id: 'datetime-histogram-source-tz' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } + }, + datetime_histogram_automatic: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'year' + } + }, + date_histogram_automatic: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + minute_histogram: { + source: { + id: 'minute-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'minute' + } + } + }, + [ + { + "id": "datetime-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "datetime-histogram-source-tz", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date::date AS d", + "from generate_series(", + "'2007-02-15'::date, '2008-04-09'::date, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "minute-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", + ") date" + ].join(' ') + } + } + ] + ); + + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'datetime_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'datetime_histogram' + }]; + + dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { + var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + + this.testClient = new TestClient(mapConfig, 1234); + + this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override aggregation in weeks ' + test.desc, function (done) { + var params = { + aggregation: 'week' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override start and end ' + test.desc, function (done) { + var params = { + start: 1180659600, // 2007-06-01 01:00:00 UTC => '2007-05-31T21:00:00-04:00' + end: 1193792400 // 2007-10-31 01:00:00 UTC + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + + it('should return same histogram ' + test.desc, function (done) { + var params = { + start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) + end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) { + assert.ok(!err, err); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) { + assert.ok(!err, err); + + assert.deepEqual(dataview, filteredDataview); + done(); + }); + }); + }); + + + it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { + var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + var params = { + offset: OFFSET_CEST_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram overriding default offset to UTC/GMT ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870399 // 2008-06-30T23:59:59Z, last bin end + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 6); + assert.equal(dataview.bins_count, dataview.bins.length); + done(); + }); + }); + + it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 7); + assert.ok(dataview.bins_count > dataview.bins.length); + done(); + }); + }); + }); + + it('should find the best aggregation (automatic mode) to build the histogram', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should work with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'year'); + assert.equal(dataview.bins.length, 2); + assert.equal(dataview.bins_count, 2); + done(); + }); + }); + + + it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should not apply offset for a histogram aggregated by minutes', function (done) { + var self = this; + var params = { + offset: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + + self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { + assert.ifError(err); + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithOffset) { + assert.ifError(err); + + assert.notEqual(dataview.offset, dataviewWithOffset.offset); + dataview.offset = dataviewWithOffset.offset; + assert.deepEqual(dataview, dataviewWithOffset); + done(); + }); + }); + }); + + it('should filter by "start" & "end" for a histogram aggregated by minutes', function (done) { + var self = this; + var paramsWithFilter = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) + }; + + var paramsWithOffset = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) + offset: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, dataview) { + assert.ifError(err); + + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, filteredDataview) { + assert.ifError(err); + + assert.deepEqual(dataview, filteredDataview); + + self.testClient.getDataview('minute_histogram', paramsWithOffset, + function (err, filteredWithOffsetDataview) { + assert.ifError(err); + + assert.notEqual(filteredWithOffsetDataview.offset, filteredDataview.offset); + filteredWithOffsetDataview.offset = filteredDataview.offset; + assert.deepEqual(filteredWithOffsetDataview, filteredDataview); + done(); + }); + }); + }); + }); + + + it('should return an histogram aggregated by days', function (done) { + var self = this; + var paramsWithDailyAgg = { + aggregation: 'day', + }; + + // data: from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggFixture = { + aggregation: 'day', + bin_width: 600, + bins_count: 2, + bins_start: 1171497600, + timestamp_start: 1171497600, + offset: 0, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171497600, + min: 1171583400, + max: 1171583940, + avg: 1171583670, + freq: 10 + }, + { + bin: 1, + timestamp: 1171584000, + min: 1171584000, + max: 1171584600, + avg: 1171584300, + freq: 11 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAgg, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggFixture); + done(); + }); + }); + + it('should return a histogram aggregated by days with offset', function (done) { + var self = this; + + var paramsWithDailyAggAndOffset = { + aggregation: 'day', + offset: '-3600' + }; + + // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggAndOffsetFixture = { + aggregation: 'day', + bin_width: 1200, + bins_count: 1, + bins_start: 1171501200, + timestamp_start: 1171497600, + nulls: 0, + offset: -3600, + bins: + [{ + bin: 0, + timestamp: 1171501200, + min: 1171583400, + max: 1171584600, + avg: 1171584000, + freq: 21 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndOffset, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggAndOffsetFixture); + done(); + }); + }); }); @@ -164,3 +766,124 @@ describe('histogram-dataview: special float valuer', function() { }); }); }); + +describe('histogram-dates: aggregation input value', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'day' + } + }, + bad_agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'wadus' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'2007-02-15 01:00:00\'::timestamp,', + '\'2008-04-09 01:00:00\'::timestamp,', + ' \'1 day\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should fail when aggregation values is not valid while instantiating the map', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + response: { + status: 400 + } + }; + + this.testClient.getDataview('bad_agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); + + it('should fail when aggregation values is not valid while fetching dataview result', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + aggregation: 'wadus', + response: { + status: 400 + } + }; + + this.testClient.getDataview('agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); +}); diff --git a/test/support/test-client.js b/test/support/test-client.js index 5d0eddff..3b9a003d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -374,7 +374,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } @@ -404,8 +404,11 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }, function finish(err, dataview) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; + if (layergroupId) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + return callback(err, dataview); } ); diff --git a/yarn.lock b/yarn.lock index 02d88ef6..49cbc74e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1288,7 +1288,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6: +moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -2103,7 +2103,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb3: version "2.3.1-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23" dependencies: @@ -2111,7 +2111,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: