diff --git a/NEWS.md b/NEWS.md index a3228af1..bf1dd7e4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,37 @@ # Changelog -## 3.9.9 +## 3.11.1 Released 2017-mm-dd +## 3.11.0 +Released 2017-08-08 + +Announcements: + - Allow to override with any aggregation for histograms instantiated w/o aggregation. + +Bug fixes: + - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series. + - Support timestamp with timezones to calculate the number of bins in time-series. + - Fixed issue related to name collision while building time-series query. + + +## 3.10.1 +Released 2017-08-04 + +Bug fixes: + - Exclude Infinities & NaNs from ramps #719. + - Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720. + + +## 3.10.0 +Released 2017-08-03 + +Announcements: + - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698. + - Support special numeric values (±Infinity, NaN) for json responses #706 + + ## 3.9.8 Released 2017-07-21 diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index a77ba585..29dcd903 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -94,7 +94,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { } function getOverrideParams(params, ownFilter) { - return _.reduce(_.pick(params, 'start', 'end', 'bins'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); @@ -104,6 +104,13 @@ function getOverrideParams(params, ownFilter) { }, {ownFilter: ownFilter} ); + + // validation will be delegated to the proper dataview + if (params.aggregation !== undefined) { + overrideParams.aggregation = params.aggregation; + } + + return overrideParams; } DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { diff --git a/lib/cartodb/backends/turbo-carto-postgres-datasource.js b/lib/cartodb/backends/turbo-carto-postgres-datasource.js index 8c0a93c7..65109325 100644 --- a/lib/cartodb/backends/turbo-carto-postgres-datasource.js +++ b/lib/cartodb/backends/turbo-carto-postgres-datasource.js @@ -10,7 +10,13 @@ function createTemplate(method) { 'max({{=it._column}}) max_val,', 'avg({{=it._column}}) avg_val,', method, - 'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL' + 'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL', + 'AND', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float' ].join('\n')); } diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index e3d2a407..98c5ee2f 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -17,16 +17,8 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'zoom', 'lon', 'lat', - // widgets & filters - 'filters', // json - 'own_filter', // 0, 1 - 'bbox', // w,s,e,n - 'bins', // number - 'start', // number - 'end', // number - 'column_type', // string - // widgets search - 'q' + // analysis + 'filters' // json ]; function BaseController(authApi, pgConnection) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index e0c7a7a3..9644aa06 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -79,19 +79,51 @@ LayergroupController.prototype.register = function(app) { // Undocumented/non-supported API endpoint methods. // Use at your own peril. - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName', cors(), userMiddleware, - this.dataview.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/dataview/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); - app.get(app.base_url_mapconfig + - '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware, - this.dataviewSearch.bind(this)); + var allowedDataviewQueryParams = [ + 'filters', // json + 'own_filter', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'offset', // number + 'q' // widgets search + ]; + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataview.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/dataview/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); + + app.get( + app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search', + cors(), + userMiddleware, + allowQueryParams(allowedDataviewQueryParams), + this.dataviewSearch.bind(this) + ); app.get(app.base_url_mapconfig + '/:token/analysis/node/:nodeId', cors(), userMiddleware, diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index 90d8a299..29069d37 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -8,6 +8,10 @@ module.exports = BaseDataview; BaseDataview.prototype.getResult = function(psql, override, callback) { var self = this; this.sql(psql, override, function(err, query) { + if (err) { + return callback(err); + } + psql.query(query, function(err, result) { if (err) { return callback(err, result); diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 73cc8724..c27c696f 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -7,13 +7,54 @@ dot.templateSettings.strip = false; var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +var dateIntervalQueryTpl = dot.template([ + 'WITH', + '__cdb_dates AS (', + ' SELECT', + ' MAX({{=it.column}}::timestamp) AS __cdb_end,', + ' MIN({{=it.column}}::timestamp) AS __cdb_start', + ' FROM ({{=it.query}}) __cdb_source', + '),', + '__cdb_interval_in_days AS (', + ' SELECT' , + ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', + ' FROM __cdb_dates', + '),', + '__cdb_interval_in_hours AS (', + ' SELECT', + ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', + ' FROM __cdb_interval_in_days, __cdb_dates', + '),', + '__cdb_interval_in_minutes AS (', + ' SELECT', + ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', + ' FROM __cdb_interval_in_hours, __cdb_dates', + '),', + '__cdb_interval_in_seconds AS (', + ' SELECT', + ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', + ' FROM __cdb_interval_in_minutes, __cdb_dates', + ')', + 'SELECT', + ' ROUND(__cdb_days / 365) AS year,', + ' ROUND(__cdb_days / 90) AS quarter,', + ' ROUND(__cdb_days / 30) AS month,', + ' ROUND(__cdb_days / 7) AS week,', + ' __cdb_days AS day,', + ' __cdb_hours AS hour,', + ' __cdb_minutes AS minute,', + ' __cdb_seconds AS second', + 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' +].join('\n')); + +var MAX_INTERVAL_VALUE = 366; var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; var filteredQueryTpl = dot.template([ - 'filtered_source AS (', + '__cdb_filtered_source AS (', ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', + ' FROM ({{=it._query}}) __cdb_filtered_source_query', ' WHERE', ' {{=it._column}} IS NOT NULL', ' {{?it._isFloatColumn}}AND', @@ -26,74 +67,74 @@ var filteredQueryTpl = dot.template([ ].join(' \n')); var basicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', - ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', + ' FROM __cdb_filtered_source', ')' ].join(' \n')); var overrideBasicsQueryTpl = dot.template([ - 'basics AS (', + '__cdb_basics AS (', ' SELECT', - ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', - ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM filtered_source', + ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', + ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', + ' FROM __cdb_filtered_source', ')' ].join('\n')); var iqrQueryTpl = dot.template([ - 'iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + '__cdb_iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', ' FROM (', ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM filtered_source) _cdb_quartiles', + ' FROM __cdb_filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', - ' ) _cdb_iqr', + ' ) __cdb_iqr', ')' ].join('\n')); var binsQueryTpl = dot.template([ - 'bins AS (', - ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + '__cdb_bins AS (', + ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', ' THEN 1', ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', ' LEAST(', - ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', ' {{=it._maxBins}}', ' )', ' )', - ' END AS bins_number', - ' FROM basics, iqrange, filtered_source', + ' END AS __cdb_bins_number', + ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', ' LIMIT 1', ')' ].join('\n')); var overrideBinsQueryTpl = dot.template([ - 'bins AS (', - ' SELECT {{=it._bins}} AS bins_number', + '__cdb_bins AS (', + ' SELECT {{=it._bins}} AS __cdb_bins_number', ')' ].join('\n')); var nullsQueryTpl = dot.template([ - 'nulls AS (', + '__cdb_nulls AS (', ' SELECT', - ' count(*) AS nulls_count', - ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' count(*) AS __cdb_nulls_count', + ' FROM ({{=it._query}}) __cdb_histogram_nulls', ' WHERE {{=it._column}} IS NULL', ')' ].join('\n')); var infinitiesQueryTpl = dot.template([ - 'infinities AS (', + '__cdb_infinities AS (', ' SELECT', - ' count(*) AS infinities_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' count(*) AS __cdb_infinities_count', + ' FROM ({{=it._query}}) __cdb_infinities_query', ' WHERE', ' {{=it._column}} = \'infinity\'::float', ' OR', @@ -102,46 +143,151 @@ var infinitiesQueryTpl = dot.template([ ].join('\n')); var nansQueryTpl = dot.template([ - 'nans AS (', + '__cdb_nans AS (', ' SELECT', - ' count(*) AS nans_count', - ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' count(*) AS __cdb_nans_count', + ' FROM ({{=it._query}}) __cdb_nans_query', ' WHERE {{=it._column}} = \'NaN\'::float', ')' ].join('\n')); var histogramQueryTpl = dot.template([ 'SELECT', - ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', - ' bins_number,', - ' nulls_count,', - ' {{?it._isFloatColumn}}infinities_count,', - ' nans_count,{{?}}', - ' avg_val,', - ' CASE WHEN min_val = max_val', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', + ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', + ' __cdb_nans_count AS nans_count,{{?}}', + ' __cdb_avg_val AS avg_val,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', ' THEN 0', - ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' ELSE GREATEST(', + ' 1,', + ' LEAST(', + ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', + ' __cdb_bins_number', + ' )', + ' ) - 1', ' END AS bin,', ' min({{=it._column}})::numeric AS min,', ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', + ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count,', ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); +var dateBasicsQueryTpl = dot.template([ + '__cdb_basics AS (', + ' SELECT', + ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', + ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', + ' min(date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' )) AS __cdb_start_date,', + ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', + ' FROM ({{=it._query}}) __cdb_basics_query', + ')' +].join(' \n')); + +var dateOverrideBasicsQueryTpl = dot.template([ + '__cdb_basics AS (', + ' SELECT', + ' max({{=it._end}}) AS __cdb_max_val,', + ' min({{=it._start}}) AS __cdb_min_val,', + ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', + ' min(', + ' date_trunc(', + ' \'{{=it._aggregation}}\',', + ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' )', + ' ) AS __cdb_start_date,', + ' max(', + ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' ) AS __cdb_end_date,', + ' count(1) AS __cdb_total_rows', + ' FROM ({{=it._query}}) __cdb_basics_query', + ')' +].join(' \n')); + +var dateBinsQueryTpl = dot.template([ + '__cdb_bins AS (', + ' SELECT', + ' __cdb_bins_array,', + ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', + ' FROM (', + ' SELECT', + ' ARRAY(', + ' SELECT GENERATE_SERIES(', + ' __cdb_start_date::timestamptz,', + ' __cdb_end_date::timestamptz,', + ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', + ' )', + ' ) AS __cdb_bins_array', + ' FROM __cdb_basics', + ' ) __cdb_bins_array_query', + ')' +].join('\n')); + +var dateHistogramQueryTpl = dot.template([ + 'SELECT', + ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', + ' __cdb_bins_number AS bins_number,', + ' __cdb_nulls_count AS nulls_count,', + ' CASE WHEN __cdb_min_val = __cdb_max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(', + ' WIDTH_BUCKET(', + ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', + ' __cdb_bins_array', + ' ),', + ' __cdb_bins_number', + ' )) - 1', + ' END AS bin,', + ' min(', + ' date_part(', + ' \'epoch\', ', + ' date_trunc(', + ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', + ' ) AT TIME ZONE \'{{=it._offset}}\'', + ' )', + ' )::numeric AS timestamp,', + ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', + ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', + ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', + ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', + 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', + 'ORDER BY bin' +].join('\n')); var TYPE = 'histogram'; /** - { - type: 'histogram', - options: { - column: 'name', - bins: 10 // OPTIONAL - } +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} + +Time series: +{ + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) + offset: -7200 // OPTIONAL (UTC offset in seconds) + } } */ function Histogram(query, options, queries) { @@ -153,6 +299,8 @@ function Histogram(query, options, queries) { this.queries = queries; this.column = options.column; this.bins = options.bins; + this.aggregation = options.aggregation; + this.offset = options.offset; this._columnType = null; } @@ -184,16 +332,22 @@ Histogram.prototype.sql = function(psql, override, callback) { return null; } - var histogramSql = this._buildQuery(override); - - return callback(null, histogramSql); + this._buildQuery(psql, override, callback); }; -Histogram.prototype._buildQuery = function (override) { +Histogram.prototype.isDateHistogram = function (override) { + return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); +}; + +Histogram.prototype._buildQuery = function (psql, override, callback) { var filteredQuery, basicsQuery, binsQuery; var _column = this.column; var _query = this.query; + if (this.isDateHistogram(override)) { + return this._buildDateHistogramQuery(psql, override, callback); + } + if (this._columnType === 'date') { _column = columnCastTpl({column: _column}); } @@ -280,7 +434,7 @@ Histogram.prototype._buildQuery = function (override) { debug(histogramSql); - return histogramSql; + return callback(null, histogramSql); }; Histogram.prototype._shouldOverride = function (override) { @@ -291,6 +445,135 @@ Histogram.prototype._shouldOverrideBins = function (override) { return override && _.has(override, 'bins'); }; +var DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + +Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { + var _column = this.column; + var _query = this.query; + var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; + var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; + + if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (_aggregation === 'auto') { + this.getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildDateHistogramQuery(psql, override, callback); + }.bind(this)); + return null; + } + + var dateBasicsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end')) { + dateBasicsQuery = dateOverrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _start: getBinStart(override), + _end: getBinEnd(override), + _offset: parseOffset(_offset, _aggregation) + }); + } else { + dateBasicsQuery = dateBasicsQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + } + + var dateBinsQuery = [ + dateBinsQueryTpl({ + _aggregation: _aggregation + }) + ].join(',\n'); + + var nullsQuery = nullsQueryTpl({ + _query: _query, + _column: _column + }); + + var dateHistogramQuery = dateHistogramQueryTpl({ + _query: _query, + _column: _column, + _aggregation: _aggregation, + _offset: parseOffset(_offset, _aggregation) + }); + + var histogramSql = [ + "WITH", + [ + dateBasicsQuery, + dateBinsQuery, + nullsQuery + ].join(',\n'), + dateHistogramQuery + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + +Histogram.prototype.getAutomaticAggregation = function (psql, callback) { + var dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + debug(dateIntervalQuery); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + var aggegations = result.rows[0]; + var aggregation = Object.keys(aggegations) + .map(function (key) { + return { + name: key, + value: aggegations[key] + }; + }) + .reduce(function (closer, current) { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + var closerDiff = MAX_INTERVAL_VALUE - closer.value; + var currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); +}; + Histogram.prototype.format = function(result, override) { override = override || {}; var buckets = []; @@ -302,6 +585,9 @@ Histogram.prototype.format = function(result, override) { var infinities = 0; var nans = 0; var avg; + var timestampStart; + var aggregation; + var offset; if (result.rows.length) { var firstRow = result.rows[0]; @@ -309,16 +595,34 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; + timestampStart = firstRow.timestamp_start; infinities = firstRow.infinities_count; nans = firstRow.nans_count; - binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; + binsStart = populateBinStart(override, firstRow); + + if (Number.isFinite(timestampStart)) { + aggregation = getAggregation(override, this.aggregation); + offset = getOffset(override, this.offset); + } buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val'); + return _.omit( + row, + 'bins_number', + 'bin_width', + 'nulls_count', + 'infinities_count', + 'nans_count', + 'avg_val', + 'timestamp_start' + ); }); } return { + aggregation: aggregation, + offset: offset, + timestamp_start: timestampStart, bin_width: width, bins_count: binsCount, bins_start: binsStart, @@ -330,6 +634,21 @@ Histogram.prototype.format = function(result, override) { }; }; +function getAggregation(override, aggregation) { + return override && override.aggregation ? override.aggregation : aggregation; +} + +function getOffset(override, offset) { + if (override && override.offset) { + return override.offset; + } + if (offset) { + return offset; + } + + return 0; +} + function getBinStart(override) { if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { return Math.min(override.start, override.end); @@ -359,6 +678,32 @@ function getWidth(override) { return width; } +function parseOffset(offset, aggregation) { + if (!offset) { + return '0'; + } + if (aggregation === 'hour' || aggregation === 'minute') { + return '0'; + } + + var offsetInHours = Math.ceil(offset / 3600); + return '' + offsetInHours; +} + +function populateBinStart(override, firstRow) { + var binStart; + + if (firstRow.hasOwnProperty('timestamp')) { + binStart = firstRow.timestamp; + } else if (override.hasOwnProperty('start')) { + binStart = getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; +} + Histogram.prototype.getType = function() { return TYPE; }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7b6fc33a..7aea23cd 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -329,6 +329,25 @@ function bootstrap(opts) { app.enable('jsonp callback'); app.disable('x-powered-by'); app.disable('etag'); + + // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705 + // See: http://expressjs.com/en/4x/api.html#app.set + app.set('json replacer', function (key, value) { + if (value !== value) { + return 'NaN'; + } + + if (value === Infinity) { + return 'Infinity'; + } + + if (value === -Infinity) { + return '-Infinity'; + } + + return value; + }); + app.use(bodyParser.json()); app.use(function bootstrap$prepareRequestResponse(req, res, next) { diff --git a/package.json b/package.json index fe80d464..d0e0be6b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.9.9", + "version": "3.11.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -47,6 +47,7 @@ "istanbul": "~0.4.3", "jshint": "~2.9.4", "mocha": "~3.4.1", + "moment": "~2.18.1", "nock": "~2.11.0", "redis": "~0.12.1", "semver": "~1.1.4", diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c75e9baf..3baf002b 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -2,6 +2,16 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +var moment = require('moment'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} function createMapConfig(layers, dataviews, analysis) { return { @@ -96,6 +106,598 @@ describe('histogram-dataview', function() { done(); }); }); + +}); + +describe('histogram-dataview for date column type', function() { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "datetime-histogram-source" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + datetime_histogram: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } + }, + datetime_histogram_tz: { + source: { + id: 'datetime-histogram-source-tz' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'month', + offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds + } + }, + datetime_histogram_automatic: { + source: { + id: 'datetime-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + date_histogram: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'year' + } + }, + date_histogram_automatic: { + source: { + id: 'date-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + minute_histogram: { + source: { + id: 'minute-histogram-source' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'minute' + } + } + }, + [ + { + "id": "datetime-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "datetime-histogram-source-tz", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "date-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date::date AS d", + "from generate_series(", + "'2007-02-15'::date, '2008-04-09'::date, '1 day'::interval", + ") date" + ].join(' ') + } + }, + { + "id": "minute-histogram-source", + "type": "source", + "params": { + "query": [ + "select null::geometry the_geom_webmercator, date AS d", + "from generate_series(", + "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", + ") date" + ].join(' ') + } + } + ] + ); + + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'datetime_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'datetime_histogram' + }]; + + dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { + var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes + + this.testClient = new TestClient(mapConfig, 1234); + + this.testClient.getDataview(test.dataviewId, {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00-04:00'; // EDT midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function(bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_EDT_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override aggregation in weeks ' + test.desc, function (done) { + var params = { + aggregation: 'week' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 61); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should override start and end ' + test.desc, function (done) { + var params = { + start: 1180659600, // 2007-06-01 01:00:00 UTC => '2007-05-31T21:00:00-04:00' + end: 1193792400 // 2007-10-31 01:00:00 UTC + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + dataview.bins.forEach(function (bin) { + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + + it('should return same histogram ' + test.desc, function (done) { + var params = { + start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) + end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) { + assert.ok(!err, err); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) { + assert.ok(!err, err); + + assert.deepEqual(dataview, filteredDataview); + done(); + }); + }); + }); + + + it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { + var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) + var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) + var params = { + offset: OFFSET_CEST_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00+02:00'; // CEST midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_CEST_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram overriding default offset to UTC/GMT ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 15); + + var initialTimestamp = '2007-02-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .add(index, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter' + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.type, 'histogram'); + assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); + assert.equal(dataview.bins.length, 6); + + var initialTimestamp = '2007-01-01T00:00:00Z'; // UTC midnight + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .add(index * 3, 'month') + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_UTC_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + + it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870399 // 2008-06-30T23:59:59Z, last bin end + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 6); + assert.equal(dataview.bins_count, dataview.bins.length); + done(); + }); + }); + + it('bins_count should be greater than bins length filtered by start and end ' + test.desc, function (done) { + var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC + var params = { + offset: OFFSET_UTC_IN_SECONDS, + aggregation: 'quarter', + start: 1167609600, // 2007-01-01T00:00:00Z, first bin start + end: 1214870400 // 2008-07-01T00:00:00Z, start the next bin to the last + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview(test.dataviewId, params, function (err, dataview) { + assert.ifError(err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 7); + assert.ok(dataview.bins_count > dataview.bins.length); + done(); + }); + }); + }); + + it('should find the best aggregation (automatic mode) to build the histogram', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should work with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'year'); + assert.equal(dataview.bins.length, 2); + assert.equal(dataview.bins_count, 2); + done(); + }); + }); + + + it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) { + var params = {}; + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) { + assert.ifError(err); + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.aggregation, 'week'); + assert.equal(dataview.bins.length, 61); + assert.equal(dataview.bins_count, 61); + done(); + }); + }); + + it('should not apply offset for a histogram aggregated by minutes', function (done) { + var self = this; + var params = { + offset: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + + self.testClient.getDataview('minute_histogram', {}, function (err, dataview) { + assert.ifError(err); + self.testClient.getDataview('minute_histogram', params, function (err, dataviewWithOffset) { + assert.ifError(err); + + assert.notEqual(dataview.offset, dataviewWithOffset.offset); + dataview.offset = dataviewWithOffset.offset; + assert.deepEqual(dataview, dataviewWithOffset); + done(); + }); + }); + }); + + it('should filter by "start" & "end" for a histogram aggregated by minutes', function (done) { + var self = this; + var paramsWithFilter = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600 // 2007-02-16 00:10:00 = max(date_colum) + }; + + var paramsWithOffset = { + start: 1171583400, // 2007-02-15 23:50:00 = min(date_colum) + end: 1171584600, // 2007-02-16 00:10:00 = max(date_colum) + offset: '-3600' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, dataview) { + assert.ifError(err); + + self.testClient.getDataview('minute_histogram', paramsWithFilter, function (err, filteredDataview) { + assert.ifError(err); + + assert.deepEqual(dataview, filteredDataview); + + self.testClient.getDataview('minute_histogram', paramsWithOffset, + function (err, filteredWithOffsetDataview) { + assert.ifError(err); + + assert.notEqual(filteredWithOffsetDataview.offset, filteredDataview.offset); + filteredWithOffsetDataview.offset = filteredDataview.offset; + assert.deepEqual(filteredWithOffsetDataview, filteredDataview); + done(); + }); + }); + }); + }); + + + it('should return an histogram aggregated by days', function (done) { + var self = this; + var paramsWithDailyAgg = { + aggregation: 'day', + }; + + // data: from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggFixture = { + aggregation: 'day', + bin_width: 600, + bins_count: 2, + bins_start: 1171497600, + timestamp_start: 1171497600, + offset: 0, + nulls: 0, + bins: + [{ + bin: 0, + timestamp: 1171497600, + min: 1171583400, + max: 1171583940, + avg: 1171583670, + freq: 10 + }, + { + bin: 1, + timestamp: 1171584000, + min: 1171584000, + max: 1171584600, + avg: 1171584300, + freq: 11 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAgg, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggFixture); + done(); + }); + }); + + it('should return a histogram aggregated by days with offset', function (done) { + var self = this; + + var paramsWithDailyAggAndOffset = { + aggregation: 'day', + offset: '-3600' + }; + + // data (UTC): from 2007-02-15 23:50:00 to 2007-02-16 00:10:00 + + var dataviewWithDailyAggAndOffsetFixture = { + aggregation: 'day', + bin_width: 1200, + bins_count: 1, + bins_start: 1171501200, + timestamp_start: 1171497600, + nulls: 0, + offset: -3600, + bins: + [{ + bin: 0, + timestamp: 1171501200, + min: 1171583400, + max: 1171584600, + avg: 1171584000, + freq: 21 + }], + type: 'histogram' + }; + + self.testClient = new TestClient(mapConfig, 1234); + self.testClient.getDataview('minute_histogram', paramsWithDailyAggAndOffset, function (err, dataview) { + assert.ifError(err); + + assert.deepEqual(dataview, dataviewWithDailyAggAndOffsetFixture); + done(); + }); + }); }); @@ -164,3 +766,412 @@ describe('histogram-dataview: special float valuer', function() { }); }); }); + +describe('histogram-dates: aggregation input value', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'day' + } + }, + bad_agg_value_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'wadus' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'2007-02-15 01:00:00\'::timestamp,', + '\'2008-04-09 01:00:00\'::timestamp,', + ' \'1 day\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should fail when aggregation values is not valid while instantiating the map', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + response: { + status: 400 + } + }; + + this.testClient.getDataview('bad_agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); + + it('should fail when aggregation values is not valid while fetching dataview result', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = { + aggregation: 'wadus', + response: { + status: 400 + } + }; + + this.testClient.getDataview('agg_value_histogram', override, function(err, dataviewError) { + assert.ifError(err); + + assert.deepEqual(dataviewError, { + errors: [ + 'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year' + ], + errors_with_context: [{ + type: 'unknown', + message: [ + 'Invalid aggregation value. ', + 'Valid ones: auto, minute, hour, day, week, month, quarter, year' + ].join('') + }] + }); + + done(); + }); + }); +}); + +describe('histogram-dates: timestamp starts at epoch', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + epoch_start_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-04 10:00:00\'::timestamp,', + '\'1984-01-04 10:00:00\'::timestamp,', + ' \'1 month\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should work when timestamp_start is epoch (1970-01-01 = 0)', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + const override = {}; + + this.testClient.getDataview('epoch_start_histogram', override, function(err, dataview) { + assert.ifError(err); + + const { aggregation, timestamp_start } = dataview; + + assert.equal(timestamp_start, 0); + assert.equal(aggregation, 'month'); + + done(); + }); + }); +}); + +describe('histogram-dates: trunc timestamp for each bin respecting user\'s timezone', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + }, + timezone_epoch_histogram_tz: { + source: { + id: 'a1' + }, + type: 'histogram', + options: { + column: 'd', + aggregation: 'auto' + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + }, + { + id: 'a1', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamptz,', + '\'1970-01-01 01:59:00\'::timestamptz,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + var dateHistogramsUseCases = [{ + desc: 'supporting timestamp with offset', + dataviewId: 'timezone_epoch_histogram_tz' + }, { + desc: 'supporting timestamp without offset', + dataviewId: 'timezone_epoch_histogram' + }]; + + dateHistogramsUseCases.forEach(function (test) { + it('should return histogram with two buckets ' + test.desc , function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview(test.dataviewId, override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); + }); +}); + + +describe('histogram: be able to override with aggregation for histograms instantiated w/o aggregation', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + type: "cartodb", + options: { + source: { + id: "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + { + timezone_epoch_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'd', + } + } + }, + [ + { + id: 'a0', + type: 'source', + params: { + query: [ + 'select null::geometry the_geom_webmercator, date AS d', + 'from generate_series(', + '\'1970-01-01 00:00:00\'::timestamp,', + '\'1970-01-01 01:59:00\'::timestamp,', + ' \'1 minute\'::interval', + ') date' + ].join(' ') + } + } + ] + ); + + it('should apply aggregation to the histogram', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + + const override = { + aggregation: 'day', + offset: '-3600' + }; + + this.testClient.getDataview('timezone_epoch_histogram', override, function(err, dataview) { + assert.ifError(err); + + var OFFSET_IN_MINUTES = -1 * 60; // GMT-01 + var initialTimestamp = '1969-12-31T00:00:00-01:00'; + var binsStartInMilliseconds = dataview.bins_start * 1000; + var binsStartFormatted = moment.utc(binsStartInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + assert.equal(binsStartFormatted, initialTimestamp); + + dataview.bins.forEach(function (bin, index) { + var binTimestampExpected = moment.utc(initialTimestamp) + .utcOffset(OFFSET_IN_MINUTES) + .add(index, override.aggregation) + .format(); + var binsTimestampInMilliseconds = bin.timestamp * 1000; + var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds) + .utcOffset(OFFSET_IN_MINUTES) + .format(); + + assert.equal(binTimestampFormatted, binTimestampExpected); + assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin)); + assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); + }); + + done(); + }); + }); +}); diff --git a/test/acceptance/special-numeric-values.js b/test/acceptance/special-numeric-values.js new file mode 100644 index 00000000..2b596b8e --- /dev/null +++ b/test/acceptance/special-numeric-values.js @@ -0,0 +1,71 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); + +describe('special numeric values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var ATTRIBUTES_LAYER = 1; + + function createMapConfig(sql, id, columns) { + return { + version: '1.6.0', + layers: [ + { + type: 'mapnik', + options: { + sql: "select 1 as id, 'SRID=4326;POINT(0 0)'::geometry as the_geom", + cartocss: '#style { }', + cartocss_version: '2.0.1' + } + }, + { + type: 'mapnik', + options: { + sql: sql || "select 1 as i, 6 as n, 'SRID=4326;POINT(0 0)'::geometry as the_geom", + attributes: { + id: id || 'i', + columns: columns || ['n'] + }, + cartocss: '#style { }', + cartocss_version: '2.0.1' + } + } + ] + }; + } + + it('should retrieve special numeric values', function (done) { + var featureId = 1; + var sql = [ + 'SELECT', + ' 1 as cartodb_id,', + ' null::geometry the_geom_webmercator,', + ' \'infinity\'::float as infinity,', + ' \'-infinity\'::float as _infinity,', + ' \'NaN\'::float as nan' + ].join('\n'); + var id = 'cartodb_id'; + var columns = ['infinity', '_infinity', 'nan']; + + var mapConfig = createMapConfig(sql, id, columns); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getFeatureAttributes(featureId, ATTRIBUTES_LAYER, {}, function (err, attributes) { + assert.ifError(err); + assert.equal(attributes.infinity, 'Infinity'); + assert.equal(attributes._infinity, '-Infinity'); + assert.equal(attributes.nan, 'NaN'); + done(); + }); + }); +}); + diff --git a/test/support/test-client.js b/test/support/test-client.js index 15448f68..196e1a8e 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -411,7 +411,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } @@ -455,6 +455,105 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { ); }; +TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var extraParams = {}; + if (this.apiKey) { + extraParams.api_key = this.apiKey; + } + if (params && params.filters) { + extraParams.filters = JSON.stringify(params.filters); + } + + var url = '/api/v1/map'; + if (Object.keys(extraParams).length > 0) { + url += '?' + qs.stringify(extraParams); + } + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + var parsedBody = JSON.parse(res.body); + + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getFeatureAttributes(err, layergroupId) { + assert.ifError(err); + + var next = this; + + url = '/api/v1/map/' + layergroupId + '/' + layerId + '/attributes/' + featureId; + + assert.response(server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + expectedResponse, + function(res, err) { + if (err) { + return next(err); + } + + next(null, JSON.parse(res.body)); + } + ); + }, + function finish(err, attributes) { + if (err) { + return callback(err); + } + + return callback(null, attributes); + } + ); +}; + TestClient.prototype.getTile = function(z, x, y, params, callback) { var self = this; diff --git a/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js new file mode 100644 index 00000000..7e774023 --- /dev/null +++ b/test/unit/cartodb/backends/turbo-carto-postgres-datasource.js @@ -0,0 +1,44 @@ +var PostgresDatasource = require('../../../../lib/cartodb/backends/turbo-carto-postgres-datasource'); +var PSQL = require('cartodb-psql'); +var _ = require('underscore'); +var assert = require('assert'); + +describe('turbo-carto-postgres-datasource', function() { + + beforeEach(function () { + const dbname = _.template(global.environment.postgres_auth_user, { user_id: 1 }) + '_db'; + const psql = new PSQL({ + user: 'postgres', + dbname: dbname, + host: global.environment.postgres.host, + port: global.environment.postgres.port + }); + const sql = [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS values', + 'FROM generate_series(1, 1000) x' + ].join('\n'); + this.datasource = new PostgresDatasource(psql, sql); + }); + + it('should ignore NaNs and Infinities when computing ramps', function(done) { + var column = 'values'; + var buckets = 4; + var method = 'equal'; + this.datasource.getRamp(column, buckets, method, function(err, result) { + var expected_result = { + ramp: [ 252, 501, 750, 999 ], + stats: { min_val: 3, max_val: 999, avg_val: 501 }, + strategy: undefined + }; + assert.deepEqual(result, expected_result); + done(); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index e6aa6bce..bf5e11e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -205,7 +205,7 @@ camshaft@0.55.6: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -231,7 +231,7 @@ carto@0.16.3: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1296,7 +1296,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6: +moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"