cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #698 from CartoDB/691-date-histogram

Browse Source 
Support histograms with date/time expressions for aggregations
This commit is contained in:
Daniel
2017-08-03 15:16:13 +02:00
committed by GitHub
parent 52da3bfa55 431ca9c56f
commit a98c884e1a
10 changed files with 1144 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS.md
+2
View File
@@ -3,6 +3,8 @@
## 3.9.9
Released 2017-mm-dd
- Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698
## 3.9.8
Released 2017-07-21
lib/cartodb/backends/dataview.js
+8 -1
View File
@@ -94,7 +94,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
}
function getOverrideParams(params, ownFilter) {
return _.reduce(_.pick(params, 'start', 'end', 'bins'),
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
@@ -104,6 +104,13 @@ function getOverrideParams(params, ownFilter) {
},
{ownFilter: ownFilter}
);
// validation will be delegated to the proper dataview
if (params.aggregation !== undefined) {
overrideParams.aggregation = params.aggregation;
}
return overrideParams;
}
DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
lib/cartodb/controllers/base.js
+2 -10
View File
@@ -17,16 +17,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
'zoom',
'lon',
'lat',
// widgets & filters
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'bins', // number
'start', // number
'end', // number
'column_type', // string
// widgets search
'q'
// analysis
'filters' // json
];
function BaseController(authApi, pgConnection) {
lib/cartodb/controllers/layergroup.js
+44 -12
View File
@@ -79,19 +79,51 @@ LayergroupController.prototype.register = function(app) {
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(app.base_url_mapconfig +
'/:token/dataview/:dataviewName', cors(), userMiddleware,
this.dataview.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:dataviewName', cors(), userMiddleware,
this.dataview.bind(this));
app.get(app.base_url_mapconfig +
'/:token/dataview/:dataviewName/search', cors(), userMiddleware,
this.dataviewSearch.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware,
this.dataviewSearch.bind(this));
var allowedDataviewQueryParams = [
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q' // widgets search
];
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataview.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataview.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataviewSearch.bind(this)
);
app.get(
app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
cors(),
userMiddleware,
allowQueryParams(allowedDataviewQueryParams),
this.dataviewSearch.bind(this)
);
app.get(app.base_url_mapconfig +
'/:token/analysis/node/:nodeId', cors(), userMiddleware,
lib/cartodb/models/dataview/base.js
+4
View File
@@ -8,6 +8,10 @@ module.exports = BaseDataview;
BaseDataview.prototype.getResult = function(psql, override, callback) {
var self = this;
this.sql(psql, override, function(err, query) {
if (err) {
return callback(err);
}
psql.query(query, function(err, result) {
if (err) {
lib/cartodb/models/dataview/histogram.js
+348 -14
View File
@@ -7,6 +7,47 @@ dot.templateSettings.strip = false;
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var dateIntervalQueryTpl = dot.template([
'WITH',
'dates AS (',
' SELECT',
' MAX({{=it.column}}::timestamp) AS _end,',
' MIN({{=it.column}}::timestamp) AS _start',
' FROM ({{=it.query}}) _cdb_source',
'),',
'interval_in_days AS (',
' SELECT' ,
' DATE_PART(\'day\', _end - _start) AS days',
' FROM dates',
'),',
'interval_in_hours AS (',
' SELECT',
' days * 24 + DATE_PART(\'hour\', _end - _start) AS hours',
' FROM interval_in_days, dates',
'),',
'interval_in_minutes AS (',
' SELECT',
' hours * 60 + DATE_PART(\'minute\', _end - _start) AS minutes',
' FROM interval_in_hours, dates',
'),',
'interval_in_seconds AS (',
' SELECT',
' minutes * 60 + DATE_PART(\'second\', _end - _start) AS seconds',
' FROM interval_in_minutes, dates',
')',
'SELECT',
' ROUND(days / 365) AS year,',
' ROUND(days / 90) AS quarter,',
' ROUND(days / 30) AS month,',
' ROUND(days / 7) AS week,',
' days AS day,',
' hours AS hour,',
' minutes AS minute,',
' seconds AS second',
'FROM interval_in_days, interval_in_hours, interval_in_minutes, interval_in_seconds'
].join('\n'));
var MAX_INTERVAL_VALUE = 366;
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
@@ -83,7 +124,7 @@ var overrideBinsQueryTpl = dot.template([
var nullsQueryTpl = dot.template([
'nulls AS (',
' SELECT',
' count(*) AS nulls_count',
' count(*) AS nulls_count',
' FROM ({{=it._query}}) _cdb_histogram_nulls',
' WHERE {{=it._column}} IS NULL',
')'
@@ -132,16 +173,114 @@ var histogramQueryTpl = dot.template([
'ORDER BY bin'
].join('\n'));
var dateBasicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max(date_part(\'epoch\', {{=it._column}})) AS max_val,',
' min(date_part(\'epoch\', {{=it._column}})) AS min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,',
' min(date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}} AT TIME ZONE \'{{=it._offset}}\'',
' )) AS start_date,',
' max({{=it._column}} AT TIME ZONE \'{{=it._offset}}\') AS end_date,',
' count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
')'
].join(' \n'));
var dateOverrideBasicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max({{=it._end}}) AS max_val,',
' min({{=it._start}}) AS min_val,',
' avg(date_part(\'epoch\', {{=it._column}})) AS avg_val,',
' min(',
' date_trunc(',
' \'{{=it._aggregation}}\',',
' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' )',
' ) AS start_date,',
' max(',
' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
' ) AS end_date,',
' count(1) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
')'
].join(' \n'));
var dateBinsQueryTpl = dot.template([
'bins AS (',
' SELECT',
' bins_array,',
' ARRAY_LENGTH(bins_array, 1) AS bins_number',
' FROM (',
' SELECT',
' ARRAY(',
' SELECT GENERATE_SERIES(',
' start_date::timestamptz,',
' end_date::timestamptz,',
' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval',
' )',
' ) AS bins_array',
' FROM basics',
' ) _cdb_bins_array',
')'
].join('\n'));
var dateHistogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' CASE WHEN min_val = max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(',
' WIDTH_BUCKET(',
' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',',
' bins_array',
' ),',
' bins_number',
' )) - 1',
' END AS bin,',
' min(',
' date_part(',
' \'epoch\', ',
' date_trunc(',
' \'{{=it._aggregation}}\', {{=it._column}}::timestamptz',
' ) AT TIME ZONE \'{{=it._offset}}\'',
' )',
' )::numeric AS timestamp,',
' date_part(\'epoch\', start_date)::numeric AS timestamp_start,',
' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
' count(*) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, bins, nulls',
'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val, start_date',
'ORDER BY bin'
].join('\n'));
var TYPE = 'histogram';
/**
{
type: 'histogram',
options: {
column: 'name',
bins: 10 // OPTIONAL
}
Numeric histogram:
{
type: 'histogram',
options: {
column: 'name', // column data type: numeric
bins: 10 // OPTIONAL
}
}
Time series:
{
type: 'histogram',
options: {
column: 'date', // column data type: date
aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric)
offset: -7200 // OPTIONAL (UTC offset in seconds)
}
}
*/
function Histogram(query, options, queries) {
@@ -153,6 +292,8 @@ function Histogram(query, options, queries) {
this.queries = queries;
this.column = options.column;
this.bins = options.bins;
this.aggregation = options.aggregation;
this.offset = options.offset;
this._columnType = null;
}
@@ -184,16 +325,18 @@ Histogram.prototype.sql = function(psql, override, callback) {
return null;
}
var histogramSql = this._buildQuery(override);
return callback(null, histogramSql);
this._buildQuery(psql, override, callback);
};
Histogram.prototype._buildQuery = function (override) {
Histogram.prototype._buildQuery = function (psql, override, callback) {
var filteredQuery, basicsQuery, binsQuery;
var _column = this.column;
var _query = this.query;
if (this._columnType === 'date' && this.aggregation !== undefined) {
return this._buildDateHistogramQuery(psql, override, callback);
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
}
@@ -280,7 +423,7 @@ Histogram.prototype._buildQuery = function (override) {
debug(histogramSql);
return histogramSql;
return callback(null, histogramSql);
};
Histogram.prototype._shouldOverride = function (override) {
@@ -291,6 +434,135 @@ Histogram.prototype._shouldOverrideBins = function (override) {
return override && _.has(override, 'bins');
};
var DATE_AGGREGATIONS = {
'auto': true,
'minute': true,
'hour': true,
'day': true,
'week': true,
'month': true,
'quarter': true,
'year': true
};
Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
var _column = this.column;
var _query = this.query;
var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset;
if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) {
return callback(new Error('Invalid aggregation value. Valid ones: ' +
Object.keys(DATE_AGGREGATIONS).join(', ')
));
}
if (_aggregation === 'auto') {
this.getAutomaticAggregation(psql, function (err, aggregation) {
if (err || aggregation === 'none') {
this.aggregation = 'day';
} else {
this.aggregation = aggregation;
}
override.aggregation = this.aggregation;
this._buildDateHistogramQuery(psql, override, callback);
}.bind(this));
return null;
}
var dateBasicsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end')) {
dateBasicsQuery = dateOverrideBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_start: getBinStart(override),
_end: getBinEnd(override),
_offset: parseOffset(_offset, _aggregation)
});
} else {
dateBasicsQuery = dateBasicsQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
}
var dateBinsQuery = [
dateBinsQueryTpl({
_aggregation: _aggregation
})
].join(',\n');
var nullsQuery = nullsQueryTpl({
_query: _query,
_column: _column
});
var dateHistogramQuery = dateHistogramQueryTpl({
_query: _query,
_column: _column,
_aggregation: _aggregation,
_offset: parseOffset(_offset, _aggregation)
});
var histogramSql = [
"WITH",
[
dateBasicsQuery,
dateBinsQuery,
nullsQuery
].join(',\n'),
dateHistogramQuery
].join('\n');
debug(histogramSql);
return callback(null, histogramSql);
};
Histogram.prototype.getAutomaticAggregation = function (psql, callback) {
var dateIntervalQuery = dateIntervalQueryTpl({
query: this.query,
column: this.column
});
debug(dateIntervalQuery);
psql.query(dateIntervalQuery, function (err, result) {
if (err) {
return callback(err);
}
var aggegations = result.rows[0];
var aggregation = Object.keys(aggegations)
.map(function (key) {
return {
name: key,
value: aggegations[key]
};
})
.reduce(function (closer, current) {
if (current.value > MAX_INTERVAL_VALUE) {
return closer;
}
var closerDiff = MAX_INTERVAL_VALUE - closer.value;
var currentDiff = MAX_INTERVAL_VALUE - current.value;
if (Number.isFinite(current.value) && closerDiff > currentDiff) {
return current;
}
return closer;
}, { name: 'none', value: -1 });
callback(null, aggregation.name);
});
};
Histogram.prototype.format = function(result, override) {
override = override || {};
var buckets = [];
@@ -302,6 +574,9 @@ Histogram.prototype.format = function(result, override) {
var infinities = 0;
var nans = 0;
var avg;
var timestampStart;
var aggregation;
var offset;
if (result.rows.length) {
var firstRow = result.rows[0];
@@ -309,16 +584,34 @@ Histogram.prototype.format = function(result, override) {
width = firstRow.bin_width || width;
avg = firstRow.avg_val;
nulls = firstRow.nulls_count;
timestampStart = firstRow.timestamp_start;
infinities = firstRow.infinities_count;
nans = firstRow.nans_count;
binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
binsStart = populateBinStart(override, firstRow);
if (timestampStart) {
aggregation = getAggregation(override, this.aggregation);
offset = getOffset(override, this.offset);
}
buckets = result.rows.map(function(row) {
return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val');
return _.omit(
row,
'bins_number',
'bin_width',
'nulls_count',
'infinities_count',
'nans_count',
'avg_val',
'timestamp_start'
);
});
}
return {
aggregation: aggregation,
offset: offset,
timestamp_start: timestampStart,
bin_width: width,
bins_count: binsCount,
bins_start: binsStart,
@@ -330,6 +623,21 @@ Histogram.prototype.format = function(result, override) {
};
};
function getAggregation(override, aggregation) {
return override && override.aggregation ? override.aggregation : aggregation;
}
function getOffset(override, offset) {
if (override && override.offset) {
return override.offset;
}
if (offset) {
return offset;
}
return 0;
}
function getBinStart(override) {
if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
return Math.min(override.start, override.end);
@@ -359,6 +667,32 @@ function getWidth(override) {
return width;
}
function parseOffset(offset, aggregation) {
if (!offset) {
return '0';
}
if (aggregation === 'hour' || aggregation === 'minute') {
return '0';
}
var offsetInHours = Math.ceil(offset / 3600);
return '' + offsetInHours;
}
function populateBinStart(override, firstRow) {
var binStart;
if (firstRow.hasOwnProperty('timestamp')) {
binStart = firstRow.timestamp;
} else if (override.hasOwnProperty('start')) {
binStart = getBinStart(override);
} else {
binStart = firstRow.min;
}
return binStart;
}
Histogram.prototype.getType = function() {
return TYPE;
};
package.json
+1
View File
@@ -47,6 +47,7 @@
"istanbul": "~0.4.3",
"jshint": "~2.9.4",
"mocha": "~3.4.1",
"moment": "~2.18.1",
"nock": "~2.11.0",
"redis": "~0.12.1",
"semver": "~1.1.4",
test/acceptance/dataviews/histogram.js
+723
View File
@@ -2,6 +2,16 @@ require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
var moment = require('moment');
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
function createMapConfig(layers, dataviews, analysis) {
return {
@@ -96,6 +106,598 @@ describe('histogram-dataview', function() {
done();
});
});
});
describe('histogram-dataview for date column type', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "datetime-histogram-source"
},
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0"
}
}
],
{
datetime_histogram: {
source: {
id: 'datetime-histogram-source'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'month',
offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds
}
},
datetime_histogram_tz: {
source: {
id: 'datetime-histogram-source-tz'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'month',
offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds
}
},
datetime_histogram_automatic: {
source: {
id: 'datetime-histogram-source'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
date_histogram: {
source: {
id: 'date-histogram-source'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'year'
}
},
date_histogram_automatic: {
source: {
id: 'date-histogram-source'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'auto'
}
},
minute_histogram: {
source: {
id: 'minute-histogram-source'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'minute'
}
}
},
[
{
"id": "datetime-histogram-source",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval",
") date"
].join(' ')
}
},
{
"id": "datetime-histogram-source-tz",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval",
") date"
].join(' ')
}
},
{
"id": "date-histogram-source",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date::date AS d",
"from generate_series(",
"'2007-02-15'::date, '2008-04-09'::date, '1 day'::interval",
") date"
].join(' ')
}
},
{
"id": "minute-histogram-source",
"type": "source",
"params": {
"query": [
"select null::geometry the_geom_webmercator, date AS d",
"from generate_series(",
"'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval",
") date"
].join(' ')
}
}
]
);
var dateHistogramsUseCases = [{
desc: 'supporting timestamp with offset',
dataviewId: 'datetime_histogram_tz'
}, {
desc: 'supporting timestamp without offset',
dataviewId: 'datetime_histogram'
}];
dateHistogramsUseCases.forEach(function (test) {
it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) {
var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 15);
var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_EDT_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function(bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_EDT_IN_MINUTES)
.add(index, 'month')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_EDT_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should override aggregation in weeks ' + test.desc, function (done) {
var params = {
aggregation: 'week'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 61);
dataview.bins.forEach(function (bin) {
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should override start and end ' + test.desc, function (done) {
var params = {
start: 1180659600, // 2007-06-01 01:00:00 UTC => '2007-05-31T21:00:00-04:00'
end: 1193792400 // 2007-10-31 01:00:00 UTC
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 6);
dataview.bins.forEach(function (bin) {
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should return same histogram ' + test.desc, function (done) {
var params = {
start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) {
assert.ok(!err, err);
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) {
assert.ok(!err, err);
assert.deepEqual(dataview, filteredDataview);
done();
});
});
});
it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) {
var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time)
var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time)
var params = {
offset: OFFSET_CEST_IN_SECONDS
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 15);
var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_CEST_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_CEST_IN_MINUTES)
.add(index, 'month')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_CEST_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should aggregate histogram overriding default offset to UTC/GMT ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 15);
var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.add(index, 'month')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'quarter'
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ok(!err, err);
assert.equal(dataview.type, 'histogram');
assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
assert.equal(dataview.bins.length, 6);
var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight
var binsStartInMilliseconds = dataview.bins_start * 1000;
var binsStartFormatted = moment.utc(binsStartInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binsStartFormatted, initialTimestamp);
dataview.bins.forEach(function (bin, index) {
var binTimestampExpected = moment.utc(initialTimestamp)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.add(index * 3, 'month')
.format();
var binsTimestampInMilliseconds = bin.timestamp * 1000;
var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
.utcOffset(OFFSET_UTC_IN_MINUTES)
.format();
assert.equal(binTimestampFormatted, binTimestampExpected);
assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
});
done();
});
});
it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'quarter',
start: 1167609600, // 2007-01-01T00:00:00Z, first bin start
end: 1214870399 // 2008-06-30T23:59:59Z, last bin end
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.bins.length, 6);
assert.equal(dataview.bins_count, 6);
assert.equal(dataview.bins_count, dataview.bins.length);
done();
});
});
it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) {
var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
var params = {
offset: OFFSET_UTC_IN_SECONDS,
aggregation: 'quarter',
start: 1167609600, // 2007-01-01T00:00:00Z, first bin start
end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last
};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.bins.length, 6);
assert.equal(dataview.bins_count, 7);
assert.ok(dataview.bins_count > dataview.bins.length);
done();
});
});
});
it('should find the best aggregation (automatic mode) to build the histogram', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'week');
assert.equal(dataview.bins.length, 61);
assert.equal(dataview.bins_count, 61);
done();
});
});
it('should work with dates', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('date_histogram', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'year');
assert.equal(dataview.bins.length, 2);
assert.equal(dataview.bins_count, 2);
done();
});
});
it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) {
var params = {};
this.testClient = new TestClient(mapConfig, 1234);
this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) {
assert.ifError(err);
assert.equal(dataview.type, 'histogram');
assert.equal(dataview.aggregation, 'week');
assert.equal(dataview.bins.length, 61);
assert.equal(dataview.bins_count, 61);
done();
});
});
it('should not apply offset for a histogram aggregated by minutes', function (done) {
var self = this;
var params = {
offset: '-3600'
};
self.testClient = new TestClient(mapConfig, 1234);
self.testClient.getDataview('minute_histogram', {}, function (err, dataview) {
assert.ifError(err);
self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithOffset) {
assert.ifError(err);
assert.notEqual(dataview.offset, dataviewWithOffset.offset);
dataview.offset = dataviewWithOffset.offset;
assert.deepEqual(dataview, dataviewWithOffset);
done();
});
});
});
it('should filter by "start" & "end" for a histogram aggregated by minutes', function (done) {
var self = this;
var paramsWithFilter = {
start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum)
end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum)
};
var paramsWithOffset = {
start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum)
end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum)
offset: '-3600'
};
self.testClient = new TestClient(mapConfig, 1234);
self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, dataview) {
assert.ifError(err);
self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, filteredDataview) {
assert.ifError(err);
assert.deepEqual(dataview, filteredDataview);
self.testClient.getDataview('minute_histogram', paramsWithOffset,
function (err, filteredWithOffsetDataview) {
assert.ifError(err);
assert.notEqual(filteredWithOffsetDataview.offset, filteredDataview.offset);
filteredWithOffsetDataview.offset = filteredDataview.offset;
assert.deepEqual(filteredWithOffsetDataview, filteredDataview);
done();
});
});
});
});
it('should return an histogram aggregated by days', function (done) {
var self = this;
var paramsWithDailyAgg = {
aggregation: 'day',
};
// data: from 2007-02-15 23:50:00 to 2007-02-16 00:10:00
var dataviewWithDailyAggFixture = {
aggregation: 'day',
bin_width: 600,
bins_count: 2,
bins_start: 1171497600,
timestamp_start: 1171497600,
offset: 0,
nulls: 0,
bins:
[{
bin: 0,
timestamp: 1171497600,
min: 1171583400,
max: 1171583940,
avg: 1171583670,
freq: 10
},
{
bin: 1,
timestamp: 1171584000,
min: 1171584000,
max: 1171584600,
avg: 1171584300,
freq: 11
}],
type: 'histogram'
};
self.testClient = new TestClient(mapConfig, 1234);
self.testClient.getDataview('minute_histogram', paramsWithDailyAgg, function (err, dataview) {
assert.ifError(err);
assert.deepEqual(dataview, dataviewWithDailyAggFixture);
done();
});
});
it('should return a histogram aggregated by days with offset', function (done) {
var self = this;
var paramsWithDailyAggAndOffset = {
aggregation: 'day',
offset: '-3600'
};
// data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00
var dataviewWithDailyAggAndOffsetFixture = {
aggregation: 'day',
bin_width: 1200,
bins_count: 1,
bins_start: 1171501200,
timestamp_start: 1171497600,
nulls: 0,
offset: -3600,
bins:
[{
bin: 0,
timestamp: 1171501200,
min: 1171583400,
max: 1171584600,
avg: 1171584000,
freq: 21
}],
type: 'histogram'
};
self.testClient = new TestClient(mapConfig, 1234);
self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndOffset, function (err, dataview) {
assert.ifError(err);
assert.deepEqual(dataview, dataviewWithDailyAggAndOffsetFixture);
done();
});
});
});
@@ -164,3 +766,124 @@ describe('histogram-dataview: special float valuer', function() {
});
});
});
describe('histogram-dates: aggregation input value', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
var mapConfig = createMapConfig(
[
{
type: "cartodb",
options: {
source: {
id: "a0"
},
cartocss: "#points { marker-width: 10; marker-fill: red; }",
cartocss_version: "2.3.0"
}
}
],
{
agg_value_histogram: {
source: {
id: 'a0'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'day'
}
},
bad_agg_value_histogram: {
source: {
id: 'a0'
},
type: 'histogram',
options: {
column: 'd',
aggregation: 'wadus'
}
}
},
[
{
id: 'a0',
type: 'source',
params: {
query: [
'select null::geometry the_geom_webmercator, date AS d',
'from generate_series(',
'\'2007-02-15 01:00:00\'::timestamp,',
'\'2008-04-09 01:00:00\'::timestamp,',
' \'1 day\'::interval',
') date'
].join(' ')
}
}
]
);
it('should fail when aggregation values is not valid while instantiating the map', function(done) {
this.testClient = new TestClient(mapConfig, 1234);
const override = {
response: {
status: 400
}
};
this.testClient.getDataview('bad_agg_value_histogram', override, function(err, dataviewError) {
assert.ifError(err);
assert.deepEqual(dataviewError, {
errors: [
'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year'
],
errors_with_context: [{
type: 'unknown',
message: [
'Invalid aggregation value. ',
'Valid ones: auto, minute, hour, day, week, month, quarter, year'
].join('')
}]
});
done();
});
});
it('should fail when aggregation values is not valid while fetching dataview result', function(done) {
this.testClient = new TestClient(mapConfig, 1234);
const override = {
aggregation: 'wadus',
response: {
status: 400
}
};
this.testClient.getDataview('agg_value_histogram', override, function(err, dataviewError) {
assert.ifError(err);
assert.deepEqual(dataviewError, {
errors: [
'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year'
],
errors_with_context: [{
type: 'unknown',
message: [
'Invalid aggregation value. ',
'Valid ones: auto, minute, hour, day, week, month, quarter, year'
].join('')
}]
});
done();
});
});
});
test/support/test-client.js
+6 -3
View File
@@ -374,7 +374,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
};
['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) {
if (params.hasOwnProperty(extraParam)) {
urlParams[extraParam] = params[extraParam];
}
@@ -404,8 +404,11 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
);
},
function finish(err, dataview) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
if (layergroupId) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(err, dataview);
}
);
yarn.lock
+6 -6
View File
@@ -2,7 +2,7 @@
# yarn lockfile v1
"abaculus@github:cartodb/abaculus#2.0.3-cdb1":
abaculus@cartodb/abaculus#2.0.3-cdb1:
version "2.0.3-cdb1"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf"
dependencies:
@@ -205,7 +205,7 @@ camshaft@0.55.6:
dot "^1.0.3"
request "^2.69.0"
"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
canvas@cartodb/node-canvas#1.6.2-cdb2:
version "1.6.2-cdb2"
resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
dependencies:
@@ -231,7 +231,7 @@ carto@CartoDB/carto#0.15.1-cdb1:
optimist "~0.6.0"
underscore "~1.6.0"
"carto@github:cartodb/carto#0.15.1-cdb3":
carto@cartodb/carto#0.15.1-cdb3:
version "0.15.1-cdb3"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
dependencies:
@@ -1288,7 +1288,7 @@ mocha@~3.4.1:
mkdirp "0.5.1"
supports-color "3.1.2"
moment@^2.10.6:
moment@^2.10.6, moment@~2.18.1:
version "2.18.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
@@ -2103,7 +2103,7 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb3":
tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb3:
version "2.3.1-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/bde83c8dcf4ada40c7c0eb1b477f212e75399d23"
dependencies:
@@ -2111,7 +2111,7 @@ through@2:
mapnik-pool "~0.1.3"
sphericalmercator "1.0.x"
"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2":
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2:
version "0.6.18-cdb2"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858"
dependencies: