diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index c15f0506..0d576614 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -5,11 +5,32 @@ var debug = require('debug')('windshaft:widget:aggregation'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' count(1) AS count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -18,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -44,22 +65,25 @@ var categoriesSummaryCountQueryTpl = dot.template([ ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,', - ' count, categories_count', + 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', + ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); @@ -84,7 +108,7 @@ var TYPE = 'aggregation'; } } */ -function Aggregation(query, options) { +function Aggregation(query, options, queries) { if (!_.isString(options.column)) { throw new Error('Aggregation expects `column` in widget options'); } @@ -108,9 +132,11 @@ function Aggregation(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = new BaseWidget(); @@ -119,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var _query = this.query; var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -140,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } else { aggregationSql = [ - this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn), + this.getCategoriesCTESql( + _query, + this.column, + this.aggregation, + this.aggregationColumn, + this._isFloatColumn + ), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', @@ -155,30 +208,38 @@ Aggregation.prototype.sql = function(psql, override, callback) { return callback(null, aggregationSql); }; -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) { +Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { return [ - "WITH", - [ - summaryQueryTpl({ - _query: query, - _column: column - }), - rankedCategoriesQueryTpl({ - _query: query, - _column: column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column - }), - categoriesSummaryCountQueryTpl({ - _query: query, - _column: column - }) - ].join(',\n') - ].join('\n'); + "WITH", + [ + filteredQueryTpl({ + _isFloatColumn: isFloatColumn, + _query: this.query, + _column: this.column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + summaryQueryTpl({ + _isFloatColumn: isFloatColumn, + _query: query, + _column: column, + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + rankedCategoriesQueryTpl({ + _query: query, + _column: column, + _aggregation: this.getAggregationSql(), + _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null + }), + categoriesSummaryMinMaxQueryTpl({ + _query: query, + _column: column + }), + categoriesSummaryCountQueryTpl({ + _query: query, + _column: column + }) + ].join(',\n') + ].join('\n'); }; var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); @@ -193,6 +254,8 @@ Aggregation.prototype.format = function(result) { var categories = []; var count = 0; var nulls = 0; + var nans = 0; + var infinities = 0; var minValue = 0; var maxValue = 0; var categoriesCount = 0; @@ -202,12 +265,15 @@ Aggregation.prototype.format = function(result) { var firstRow = result.rows[0]; count = firstRow.count; nulls = firstRow.nulls_count; + nans = firstRow.nans_count; + infinities = firstRow.infinities_count; minValue = firstRow.min_val; maxValue = firstRow.max_val; categoriesCount = firstRow.categories_count; result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', + 'max_val', 'categories_count', 'nans_count', 'infinities_count')); }); } @@ -215,6 +281,8 @@ Aggregation.prototype.format = function(result) { aggregation: this.aggregation, count: count, nulls: nulls, + nans: nans, + infinities: infinities, min: minValue, max: maxValue, categoriesCount: categoriesCount, diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js index b2e2f188..f8eb1027 100644 --- a/lib/cartodb/models/dataview/base.js +++ b/lib/cartodb/models/dataview/base.js @@ -1,3 +1,6 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + function BaseDataview() {} module.exports = BaseDataview; @@ -24,3 +27,42 @@ BaseDataview.prototype.getResult = function(psql, override, callback) { BaseDataview.prototype.search = function(psql, userQuery, callback) { return callback(null, this.format({ rows: [] })); }; + +var FLOAT_OIDS = { + 700: true, + 701: true, + 1700: true +}; + +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1' +); + +BaseDataview.prototype.getColumnType = function (psql, column, query, callback) { + var readOnlyTransaction = true; + + var columnTypeQuery = columnTypeQueryTpl({ + column: column, query: query + }); + + psql.query(columnTypeQuery, function(err, result) { + if (err) { + return callback(err); + } + var pgType = result.rows[0].pg_typeof; + callback(null, getPGTypeName(pgType)); + }, readOnlyTransaction); +}; + +function getPGTypeName (pgType) { + return { + float: FLOAT_OIDS.hasOwnProperty(pgType), + date: DATE_OIDS.hasOwnProperty(pgType) + }; +} diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 156985c5..7ec356b7 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -7,9 +7,19 @@ dot.templateSettings.strip = false; var formulaQueryTpl = dot.template([ 'SELECT', - '{{=it._operation}}({{=it._column}}) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' + ' {{=it._operation}}({{=it._column}}) AS result,', + ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' ].join('\n')); var VALID_OPERATIONS = { @@ -31,7 +41,7 @@ var TYPE = 'formula'; } } */ -function Formula(query, options) { +function Formula(query, options, queries) { if (!_.isString(options.operation)) { throw new Error('Formula expects `operation` in widget options'); } @@ -47,8 +57,10 @@ function Formula(query, options) { BaseWidget.apply(this); this.query = query; + this.queries = queries; this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; } Formula.prototype = new BaseWidget(); @@ -57,14 +69,27 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var _query = this.query; + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var formulaSql = formulaQueryTpl({ - _query: _query, + _isFloatColumn: this._isFloatColumn, + _query: this.query, _operation: this.operation, _column: this.column }); @@ -78,13 +103,17 @@ Formula.prototype.format = function(result) { var formattedResult = { operation: this.operation, result: 0, - nulls: 0 + nulls: 0, + nans: 0, + infinities: 0 }; if (result.rows.length) { formattedResult.operation = this.operation; formattedResult.result = result.rows[0].result; formattedResult.nulls = result.rows[0].nulls_count; + formattedResult.nans = result.rows[0].nans_count; + formattedResult.infinities = result.rows[0].infinities_count; } return formattedResult; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 5d102bf5..73cc8724 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -5,20 +5,32 @@ var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' {{?it._isFloatColumn}}AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -27,7 +39,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -38,7 +50,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -57,7 +69,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -77,11 +89,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -91,9 +126,9 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' avg({{=it._column}})::numeric AS avg,', ' count(*) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}', + 'GROUP BY bin, bins_number, bin_width, nulls_count,', + ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); @@ -127,50 +162,49 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.queries.no_filters - }); - if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction return null; } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; + var _query = this.query; + if (this._columnType === 'date') { _column = columnCastTpl({column: _column}); } - var _query = this.query; + filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', + _query: _query, + _column: _column + }); - var basicsQuery, binsQuery; - - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, @@ -190,7 +224,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -211,18 +245,34 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - basicsQuery, - binsQuery, - nullsQueryTpl({ + if (this._columnType === 'float') { + cteSql.push( + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) @@ -230,7 +280,15 @@ Histogram.prototype.sql = function(psql, override, callback) { debug(histogramSql); - return callback(null, histogramSql); + return histogramSql; +}; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); }; Histogram.prototype.format = function(result, override) { @@ -241,6 +299,8 @@ Histogram.prototype.format = function(result, override) { var width = getWidth(override); var binsStart = getBinStart(override); var nulls = 0; + var infinities = 0; + var nans = 0; var avg; if (result.rows.length) { @@ -249,10 +309,12 @@ Histogram.prototype.format = function(result, override) { width = firstRow.bin_width || width; avg = firstRow.avg_val; nulls = firstRow.nulls_count; + infinities = firstRow.infinities_count; + nans = firstRow.nans_count; binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min; buckets = result.rows.map(function(row) { - return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val'); }); } @@ -261,6 +323,8 @@ Histogram.prototype.format = function(result, override) { bins_count: binsCount, bins_start: binsStart, nulls: nulls, + infinities: infinities, + nans: nans, avg: avg, bins: buckets }; diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index da63b27f..5df092f4 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,14 +1,36 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); +var debug = require('debug')('windshaft:widget:aggregation:overview'); var dot = require('dot'); dot.templateSettings.strip = false; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', + ' {{=it._aggregationColumn}} != \'infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'-infinity\'::float', + ' AND', + ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var summaryQueryTpl = dot.template([ 'summary AS (', ' SELECT', ' sum(_feature_count) AS count,', ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' {{?it._isFloatColumn}},sum(', + ' CASE', + ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', + ' THEN 1', + ' ELSE 0', + ' END', + ' ) AS infinities_count,', + ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', ' FROM ({{=it._query}}) _cdb_aggregation_nulls', ')' ].join('\n')); @@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([ 'categories AS(', ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' FROM filtered_source', ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', ' GROUP BY {{=it._column}}', ' ORDER BY 2 DESC', @@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([ ' SELECT count(1) AS categories_count', ' FROM (', ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', + ' FROM filtered_source', ' GROUP BY {{=it._column}}', ' ) _cdb_categories_count', ')' ].join('\n')); var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank < {{=it._limit}}', 'UNION ALL', - 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,', + ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', ' FROM categories, summary, categories_summary_min_max, categories_summary_count', ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count, categories_count' + 'GROUP BY nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' ].join('\n')); var aggregationQueryTpl = dot.template([ 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', + 'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count', + 'GROUP BY category, nulls_count, min_val, max_val, count,', + ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', 'ORDER BY value DESC' ].join('\n')); var CATEGORIES_LIMIT = 6; - function Aggregation(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; + this.queries = queries; this.column = options.column; this.aggregation = options.aggregation; this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; Aggregation.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } var _query = this.rewrittenQuery(this.query); + var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null; + + if (this.aggregationColumn && this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } var aggregationSql; if (!!override.ownFilter) { aggregationSql = [ "WITH", [ - summaryQueryTpl({ + filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn + }), + summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), aggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), @@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) { aggregationSql = [ "WITH", [ - summaryQueryTpl({ + filteredQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, - _column: this.column + _column: this.column, + _aggregationColumn: _aggregationColumn + }), + summaryQueryTpl({ + _isFloatColumn: this._isFloatColumn, + _query: _query, + _column: this.column, + _aggregationColumn: _aggregationColumn }), rankedCategoriesQueryTpl({ _query: _query, _column: this.column, _aggregation: this.getAggregationSql(), - _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null + _aggregationColumn: _aggregationColumn }), categoriesSummaryMinMaxQueryTpl({ _query: _query, @@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) { }) ].join(',\n'), rankedAggregationQueryTpl({ + _isFloatColumn: this._isFloatColumn, _query: _query, _column: this.column, _limit: CATEGORIES_LIMIT @@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) { ].join('\n'); } + debug(aggregationSql); + return callback(null, aggregationSql); }; diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 1425e2d1..38b2c119 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,14 +1,15 @@ var _ = require('underscore'); var BaseDataview = require('../base'); -function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { +function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) { this.BaseDataview = BaseDataview; this.query = query; this.queryOptions = queryOptions; this.queryRewriter = queryRewriter; this.queryRewriteData = queryRewriteData; this.options = options; - this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); + this.queries = queries; + this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries); } module.exports = BaseOverviewsDataview; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 9e331f0b..64d612c9 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,34 +1,61 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); +var debug = require('debug')('windshaft:widget:formula:overview'); var dot = require('dot'); dot.templateSettings.strip = false; var formulaQueryTpls = { - 'count': dot.template([ - 'SELECT', - 'sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'sum': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'avg': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), + 'count': dot.template([ + 'SELECT', + 'sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'sum': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), + 'avg': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', + ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', + ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', + ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', + 'FROM ({{=it._query}}) _cdb_formula', + '{{?it._isFloatColumn}}WHERE', + ' {{=it._column}} != \'infinity\'::float', + 'AND', + ' {{=it._column}} != \'-infinity\'::float', + 'AND', + ' {{=it._column}} != \'NaN\'::float{{?}}' + ].join('\n')), }; -function Formula(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function Formula(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.column = options.column || '1'; this.operation = options.operation; + this._isFloatColumn = null; + this.queries = queries; } Formula.prototype = Object.create(BaseOverviewsDataview.prototype); @@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula; module.exports = Formula; -Formula.prototype.sql = function(psql, override, callback) { +Formula.prototype.sql = function (psql, override, callback) { + var self = this; var formulaQueryTpl = formulaQueryTpls[this.operation]; - if ( formulaQueryTpl ) { + if (formulaQueryTpl) { // supported formula for use with overviews + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } + var formulaSql = formulaQueryTpl({ - _query: this.rewrittenQuery(this.query), + _isFloatColumn: this._isFloatColumn, + _query: this.rewrittenQuery(this.query), _operation: this.operation, - _column: this.column + _column: this.column }); + callback = callback || override; + debug(formulaSql); + return callback(null, formulaSql); } + // default behaviour return this.defaultSql(psql, override, callback); }; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 67da4514..6674f6a0 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,23 +1,35 @@ var _ = require('underscore'); var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../histogram'); +var debug = require('debug')('windshaft:dataview:histogram:overview'); var dot = require('dot'); dot.templateSettings.strip = false; -var columnTypeQueryTpl = dot.template( - 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' -); - var BIN_MIN_NUMBER = 6; var BIN_MAX_NUMBER = 48; +var filteredQueryTpl = dot.template([ + 'filtered_source AS (', + ' SELECT *', + ' FROM ({{=it._query}}) _cdb_filtered_source', + ' WHERE', + ' {{=it._column}} IS NOT NULL', + ' {{?it._isFloatColumn}}AND', + ' {{=it._column}} != \'infinity\'::float', + ' AND', + ' {{=it._column}} != \'-infinity\'::float', + ' AND', + ' {{=it._column}} != \'NaN\'::float{{?}}', + ')' +].join(' \n')); + var basicsQueryTpl = dot.template([ 'basics AS (', ' SELECT', ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join(' \n')); @@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([ ' SELECT', ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', - ' FROM ({{=it._query}}) _cdb_basics', + ' FROM filtered_source', ')' ].join('\n')); @@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([ ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', ' ) AS quartile', - ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' FROM filtered_source) _cdb_quartiles', ' WHERE quartile = 1 or quartile = 3', ' GROUP BY quartile', ' ) _cdb_iqr', @@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([ ' )', ' )', ' END AS bins_number', - ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' FROM basics, iqrange, filtered_source', ' LIMIT 1', ')' ].join('\n')); @@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([ ')' ].join('\n')); +var infinitiesQueryTpl = dot.template([ + 'infinities AS (', + ' SELECT', + ' count(*) AS infinities_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE', + ' {{=it._column}} = \'infinity\'::float', + ' OR', + ' {{=it._column}} = \'-infinity\'::float', + ')' +].join('\n')); + +var nansQueryTpl = dot.template([ + 'nans AS (', + ' SELECT', + ' count(*) AS nans_count', + ' FROM ({{=it._query}}) _cdb_histogram_infinities', + ' WHERE {{=it._column}} = \'NaN\'::float', + ')' +].join('\n')); + var histogramQueryTpl = dot.template([ 'SELECT', ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', ' bins_number,', ' nulls_count,', + ' {{?it._isFloatColumn}}infinities_count,', + ' nans_count,{{?}}', ' avg_val,', ' CASE WHEN min_val = max_val', ' THEN 0', @@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([ ' max({{=it._column}})::numeric AS max,', ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', ' sum(_feature_count) AS freq', - 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', - 'WHERE {{=it._column}} IS NOT NULL', + 'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}', 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + ' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', 'ORDER BY bin' ].join('\n')); function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.query = query; this.queries = queries; @@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram; module.exports = Histogram; - -var DATE_OIDS = { - 1082: true, - 1114: true, - 1184: true -}; - Histogram.prototype.sql = function(psql, override, callback) { + var self = this; + if (!callback) { callback = override; override = {}; } - var self = this; - - var _column = this.column; - - var columnTypeQuery = columnTypeQueryTpl({ - column: _column, query: this.rewrittenQuery(this.queries.no_filters) - }); if (this._columnType === null) { - psql.query(columnTypeQuery, function(err, result) { + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { // assume numeric, will fail later self._columnType = 'numeric'; - if (!err && !!result.rows[0]) { - var pgType = result.rows[0].pg_typeof; - if (DATE_OIDS.hasOwnProperty(pgType)) { - self._columnType = 'date'; - } + if (!err && !!type) { + self._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); } self.sql(psql, override, callback); }, true); // use read-only transaction @@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) { return this.defaultSql(psql, override, callback); } + var histogramSql = this._buildQuery(override); + + return callback(null, histogramSql); +}; + +Histogram.prototype._buildQuery = function (override) { + var filteredQuery, basicsQuery, binsQuery; + var _column = this.column; var _query = this.rewrittenQuery(this.query); - var basicsQuery, binsQuery; + filteredQuery = filteredQueryTpl({ + _isFloatColumn: this._columnType === 'float', + _query: _query, + _column: _column + }); - if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + if (this._shouldOverride(override)) { + debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, _column: _column, @@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) { _column: _column }); - if (override && _.has(override, 'bins')) { + if (this._shouldOverrideBins(override)) { binsQuery = [ overrideBinsQueryTpl({ _bins: override.bins @@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) { } } + var cteSql = [ + filteredQuery, + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ]; - var histogramSql = [ - "WITH", - [ - basicsQuery, - binsQuery, - nullsQueryTpl({ + if (this._columnType === 'float') { + cteSql.push( + infinitiesQueryTpl({ + _query: _query, + _column: _column + }), + nansQueryTpl({ _query: _query, _column: _column }) - ].join(',\n'), + ); + } + + var histogramSql = [ + "WITH", + cteSql.join(',\n'), histogramQueryTpl({ + _isFloatColumn: this._columnType === 'float', _query: _query, _column: _column }) ].join('\n'); - return callback(null, histogramSql); + debug(histogramSql); + + return histogramSql; }; + +Histogram.prototype._shouldOverride = function (override) { + return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); +}; + +Histogram.prototype._shouldOverrideBins = function (override) { + return override && _.has(override, 'bins'); +}; + + diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js index 7e3b3161..6ec731f4 100644 --- a/lib/cartodb/models/dataview/overviews/list.js +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -1,8 +1,8 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../list'); -function List(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +function List(query, options, queryRewriter, queryRewriteData, params, queries) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); } List.prototype = Object.create(BaseOverviewsDataview.prototype); diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 259cd2af..697004e8 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -146,3 +146,152 @@ describe('aggregations happy cases', function() { }); }); }); + +describe('aggregation-dataview: special float values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_aggregation: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'avg', + aggregationColumn: 'val' + } + }, + sum_aggregation_numeric: { + source: { + id: 'a1' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + }, { + "id": "a1", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 3 = 0 THEN \'NaN\'::numeric', + ' WHEN x % 3 = 1 THEN x', + ' ELSE x', + ' END AS val,', + ' CASE', + ' WHEN x % 2 = 0 THEN \'category_1\'', + ' ELSE \'category_2\'', + ' END AS cat', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + // Source a0 + // ----------------------------------------------- + // the_geom_webmercator | val | cat + // ----------------------+-----------+------------ + // | -Infinity | category_2 + // | NaN | category_1 + // | 3 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 7 | category_2 + // | Infinity | category_1 + // | -Infinity | category_2 + // | NaN | category_1 + // | 11 | category_2 + // | " | " + + var filters = [{ own_filter: 0 }, {}]; + filters.forEach(function (filter) { + it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + assert.ok(dataview.categories.length === 1); + dataview.categories.forEach(function (category) { + assert.ok(category.category === 'category_2'); + assert.ok(category.value === 501); + }); + done(); + }); + }); + + it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) { + assert.ifError(err); + assert.ok(dataview.nans === 333); + assert.ok(dataview.categories.length === 2); + dataview.categories.forEach(function (category) { + assert.ok(category.value !== null); + }); + done(); + }); + }); + }); +}); diff --git a/test/acceptance/dataviews/formula.js b/test/acceptance/dataviews/formula.js new file mode 100644 index 00000000..941dc07e --- /dev/null +++ b/test/acceptance/dataviews/formula.js @@ -0,0 +1,80 @@ +require('../../support/test_helper'); +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + +describe('formula-dataview: special float values', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_formula: { + source: { + id: 'a0' + }, + type: 'formula', + options: { + column: 'val', + operation: 'avg' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_formula', {}, function(err, dataview) { + assert.ok(!err, err); + assert.equal(dataview.result, 501); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index abbaef61..c75e9baf 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -3,6 +3,15 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); +function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; +} + describe('histogram-dataview', function() { afterEach(function(done) { @@ -13,15 +22,6 @@ describe('histogram-dataview', function() { } }); - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - var mapConfig = createMapConfig( [ { @@ -89,7 +89,6 @@ describe('histogram-dataview', function() { this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('pop_max_histogram', params, function(err, res) { assert.ok(!err, err); - assert.ok(res.errors); assert.equal(res.errors.length, 1); assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/)); @@ -98,3 +97,70 @@ describe('histogram-dataview', function() { }); }); }); + + +describe('histogram-dataview: special float valuer', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "a0" + }, + "cartocss": "#points { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0" + } + } + ], + { + val_histogram: { + source: { + id: 'a0' + }, + type: 'histogram', + options: { + column: 'val' + } + } + }, + [ + { + "id": "a0", + "type": "source", + "params": { + "query": [ + 'SELECT', + ' null::geometry the_geom_webmercator,', + ' CASE', + ' WHEN x % 4 = 0 THEN \'infinity\'::float', + ' WHEN x % 4 = 1 THEN \'-infinity\'::float', + ' WHEN x % 4 = 2 THEN \'NaN\'::float', + ' ELSE x', + ' END AS val', + 'FROM generate_series(1, 1000) x' + ].join('\n') + } + } + ] + ); + + it('should filter infinities out and count them in the summary', function(done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('val_histogram', {}, function(err, dataview) { + assert.ok(!err, err); + assert.ok(dataview.infinities === (250 + 250)); + assert.ok(dataview.nans === 250); + done(); + }); + }); +}); diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js index d23519ae..dcff687f 100644 --- a/test/acceptance/dataviews/overviews.js +++ b/test/acceptance/dataviews/overviews.js @@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() { params: { query: 'select * from test_table_overviews' } + }, + { + id: 'data-source-special-float-values', + type: 'source', + params: { + query: 'select * from test_special_float_values_table_overviews' + } } ], dataviews: { @@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() { aggregationColumn: 'name', } }, + test_categories_special_values: { + type: 'aggregation', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'name', + aggregation: 'sum', + aggregationColumn: 'value', + } + }, test_histogram: { type: 'histogram', source: {id: 'data-source'}, @@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() { bins: 2 } }, + test_histogram_special_values: { + type: 'histogram', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + bins: 2 + } + }, test_avg: { type: 'formula', source: {id: 'data-source'}, @@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() { operation: 'avg' } }, + test_formula_sum_special_values: { + type: 'formula', + source: { + id: 'data-source-special-float-values' + }, + options: { + column: 'value', + operation: 'sum' + } + }, test_count: { type: 'formula', source: {id: 'data-source'}, @@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() { cartocss_version: '2.3.0', source: { id: 'data-source' } } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_special_float_values_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { + id: 'data-source-special-float-values' + } + } } ] }; @@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":3, + "nulls":0, + "type":"formula", + "infinities": 0, + "nans": 0 + }); testClient.drain(done); }); @@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":5, + "nulls":0, + "type":"formula", + "infinities": 0, + "nans": 0 + }); testClient.drain(done); }); @@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 5, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":15, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"avg", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); @@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"count", + "result":1, + "infinities": 0, + "nans": 0, + "nulls":0, + "type":"formula" + }); testClient.drain(done); }); @@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "max", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation": "min", + "result": 1, + "nulls": 0, + "infinities": 0, + "nans": 0, + "type": "formula" + }); testClient.drain(done); }); @@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() { if (err) { return done(err); } - assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + assert.deepEqual(formula_result, { + "operation":"sum", + "result":1, + "nulls":0, + "infinities": 0, + "nans": 0, + "type":"formula" + }); testClient.drain(done); }); }); @@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() { }); + describe('aggregation special float values', function () { + var params = {}; + + it("should expose an aggregation dataview filtering special float values out", function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_categories_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + aggregation: 'sum', + count: 5, + nulls: 0, + nans: 1, + infinities: 1, + min: 6, + max: 6, + categoriesCount: 1, + categories: [ { category: 'Hawai', value: 6, agg: false } ], + type: 'aggregation' + }); + testClient.drain(done); + }); + }); + + it('should expose a histogram dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_histogram_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + bin_width: 0, + bins_count: 1, + bins_start: 3, + nulls: 0, + infinities: 1, + nans: 1, + avg: 3, + bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ], + type: 'histogram' + }); + testClient.drain(done); + }); + }); + + it('should expose a formula (sum) dataview filtering special float values out', function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) { + if (err) { + return done(err); + } + assert.deepEqual(dataview, { + operation: 'sum', + result: 6, + nulls: 0, + nans: 1, + infinities: 1, + type: 'formula' + }); + testClient.drain(done); + }); + }); + }); }); }); diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 91484950..996c6b3e 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5); +-- table with overviews whit special float values + +CREATE TABLE test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id; + +SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true); + +ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass); + +INSERT INTO test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1), +(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1), +(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1); + +ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id); + +CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom); +CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator); + +GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER; + +CREATE TABLE _vovw_1_test_special_float_values_table_overviews ( + cartodb_id integer NOT NULL, + name character varying, + address character varying, + value float8, + the_geom geometry, + the_geom_webmercator geometry, + _feature_count integer, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER; + +INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES +(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2), +(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2); -- analysis tables ----------------------------------------------- diff --git a/test/support/test-client.js b/test/support/test-client.js index 04980a5a..659d6686 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } params.placeholders = params.placeholders || {}; - + assert.response(server, { url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),