Compare commits
45 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f1cacdfc7 | ||
|
|
3a442bea44 | ||
|
|
49f5b0b480 | ||
|
|
b2f3735e95 | ||
|
|
166e29e8ce | ||
|
|
32274e66fd | ||
|
|
15b88c6a67 | ||
|
|
2dae09c35b | ||
|
|
14e71b929a | ||
|
|
5555b8ad8e | ||
|
|
e44d418db3 | ||
|
|
17aebf53e2 | ||
|
|
02f117af1b | ||
|
|
b1ac5b8ca9 | ||
|
|
e2b976d9d0 | ||
|
|
8e95cf20c0 | ||
|
|
20d7f1a7c5 | ||
|
|
115d8fe685 | ||
|
|
2a366ec16f | ||
|
|
ad570ab6f2 | ||
|
|
443c1100d7 | ||
|
|
7d0af4e259 | ||
|
|
81f60959e5 | ||
|
|
ef849aec34 | ||
|
|
dee00e6abd | ||
|
|
06d40e8b1e | ||
|
|
3f17c8b15a | ||
|
|
668b22628c | ||
|
|
c08db78a0b | ||
|
|
551b6d409a | ||
|
|
3ae66e4143 | ||
|
|
227937bf4c | ||
|
|
cb7ec5d556 | ||
|
|
8b2fa27ba7 | ||
|
|
962fa05574 | ||
|
|
75d07745e6 | ||
|
|
7b5111614c | ||
|
|
e60bb770db | ||
|
|
ba6dc62a38 | ||
|
|
e6aededf08 | ||
|
|
0aae29fb4b | ||
|
|
9ba65bd5a4 | ||
|
|
7a3498e8ec | ||
|
|
cd53eda0a5 | ||
|
|
6c301403e3 |
@@ -1,7 +1,7 @@
|
||||
1. Test (make clean all check), fix if broken before proceeding
|
||||
2. Ensure proper version in package.json
|
||||
3. Ensure NEWS section exists for the new version, review it, add release date
|
||||
4. Recreate yarn.lock with: `yarn upgrade`
|
||||
4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
|
||||
5. Commit package.json, yarn.lock, NEWS
|
||||
6. git tag -a Major.Minor.Patch # use NEWS section as content
|
||||
7. Stub NEWS/package for next version
|
||||
|
||||
18
NEWS.md
18
NEWS.md
@@ -1,5 +1,23 @@
|
||||
# Changelog
|
||||
|
||||
## 3.9.5
|
||||
Released 2017-06-27
|
||||
|
||||
- Dataviews: support special numeric values (±Infinity, NaN) #700
|
||||
|
||||
|
||||
## 3.9.4
|
||||
Released 2017-06-22
|
||||
|
||||
Announcements:
|
||||
- Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
|
||||
|
||||
## 3.9.3
|
||||
Released 2017-06-16
|
||||
|
||||
Announcements:
|
||||
- Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5).
|
||||
|
||||
## 3.9.2
|
||||
Released 2017-06-16
|
||||
|
||||
|
||||
@@ -5,11 +5,32 @@ var debug = require('debug')('windshaft:widget:aggregation');
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var filteredQueryTpl = dot.template([
|
||||
'filtered_source AS (',
|
||||
' SELECT *',
|
||||
' FROM ({{=it._query}}) _cdb_filtered_source',
|
||||
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
|
||||
' {{=it._aggregationColumn}} != \'infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._aggregationColumn}} != \'-infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
|
||||
')'
|
||||
].join(' \n'));
|
||||
|
||||
var summaryQueryTpl = dot.template([
|
||||
'summary AS (',
|
||||
' SELECT',
|
||||
' count(1) AS count,',
|
||||
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
|
||||
' {{?it._isFloatColumn}},sum(',
|
||||
' CASE',
|
||||
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
|
||||
' THEN 1',
|
||||
' ELSE 0',
|
||||
' END',
|
||||
' ) AS infinities_count,',
|
||||
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
|
||||
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
|
||||
')'
|
||||
].join('\n'));
|
||||
@@ -18,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
|
||||
'categories AS(',
|
||||
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
|
||||
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
|
||||
' FROM ({{=it._query}}) _cdb_aggregation_all',
|
||||
' FROM filtered_source',
|
||||
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
|
||||
' GROUP BY {{=it._column}}',
|
||||
' ORDER BY 2 DESC',
|
||||
@@ -44,22 +65,25 @@ var categoriesSummaryCountQueryTpl = dot.template([
|
||||
].join('\n'));
|
||||
|
||||
var rankedAggregationQueryTpl = dot.template([
|
||||
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
|
||||
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
|
||||
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
|
||||
' WHERE rank < {{=it._limit}}',
|
||||
'UNION ALL',
|
||||
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
|
||||
' count, categories_count',
|
||||
'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
|
||||
' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
|
||||
' WHERE rank >= {{=it._limit}}',
|
||||
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
|
||||
'GROUP BY nulls_count, min_val, max_val, count,',
|
||||
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
|
||||
].join('\n'));
|
||||
|
||||
var aggregationQueryTpl = dot.template([
|
||||
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
|
||||
' nulls_count, min_val, max_val, count, categories_count',
|
||||
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
|
||||
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
|
||||
'GROUP BY category, nulls_count, min_val, max_val, count,',
|
||||
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
'ORDER BY value DESC'
|
||||
].join('\n'));
|
||||
|
||||
@@ -84,7 +108,7 @@ var TYPE = 'aggregation';
|
||||
}
|
||||
}
|
||||
*/
|
||||
function Aggregation(query, options) {
|
||||
function Aggregation(query, options, queries) {
|
||||
if (!_.isString(options.column)) {
|
||||
throw new Error('Aggregation expects `column` in widget options');
|
||||
}
|
||||
@@ -108,9 +132,11 @@ function Aggregation(query, options) {
|
||||
BaseWidget.apply(this);
|
||||
|
||||
this.query = query;
|
||||
this.queries = queries;
|
||||
this.column = options.column;
|
||||
this.aggregation = options.aggregation;
|
||||
this.aggregationColumn = options.aggregationColumn;
|
||||
this._isFloatColumn = null;
|
||||
}
|
||||
|
||||
Aggregation.prototype = new BaseWidget();
|
||||
@@ -119,19 +145,39 @@ Aggregation.prototype.constructor = Aggregation;
|
||||
module.exports = Aggregation;
|
||||
|
||||
Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!callback) {
|
||||
callback = override;
|
||||
override = {};
|
||||
}
|
||||
|
||||
if (this.aggregationColumn && this._isFloatColumn === null) {
|
||||
this._isFloatColumn = false;
|
||||
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
|
||||
if (!err && !!type) {
|
||||
self._isFloatColumn = type.float;
|
||||
}
|
||||
self.sql(psql, override, callback);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
var _query = this.query;
|
||||
|
||||
var aggregationSql;
|
||||
|
||||
if (!!override.ownFilter) {
|
||||
aggregationSql = [
|
||||
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
|
||||
this.getCategoriesCTESql(
|
||||
_query,
|
||||
this.column,
|
||||
this.aggregation,
|
||||
this.aggregationColumn,
|
||||
this._isFloatColumn
|
||||
),
|
||||
aggregationQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregation: this.getAggregationSql(),
|
||||
@@ -140,8 +186,15 @@ Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
].join('\n');
|
||||
} else {
|
||||
aggregationSql = [
|
||||
this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
|
||||
this.getCategoriesCTESql(
|
||||
_query,
|
||||
this.column,
|
||||
this.aggregation,
|
||||
this.aggregationColumn,
|
||||
this._isFloatColumn
|
||||
),
|
||||
rankedAggregationQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
|
||||
@@ -155,30 +208,38 @@ Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
return callback(null, aggregationSql);
|
||||
};
|
||||
|
||||
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
|
||||
Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
|
||||
return [
|
||||
"WITH",
|
||||
[
|
||||
summaryQueryTpl({
|
||||
_query: query,
|
||||
_column: column
|
||||
}),
|
||||
rankedCategoriesQueryTpl({
|
||||
_query: query,
|
||||
_column: column,
|
||||
_aggregation: this.getAggregationSql(),
|
||||
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
|
||||
}),
|
||||
categoriesSummaryMinMaxQueryTpl({
|
||||
_query: query,
|
||||
_column: column
|
||||
}),
|
||||
categoriesSummaryCountQueryTpl({
|
||||
_query: query,
|
||||
_column: column
|
||||
})
|
||||
].join(',\n')
|
||||
].join('\n');
|
||||
"WITH",
|
||||
[
|
||||
filteredQueryTpl({
|
||||
_isFloatColumn: isFloatColumn,
|
||||
_query: this.query,
|
||||
_column: this.column,
|
||||
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
|
||||
}),
|
||||
summaryQueryTpl({
|
||||
_isFloatColumn: isFloatColumn,
|
||||
_query: query,
|
||||
_column: column,
|
||||
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
|
||||
}),
|
||||
rankedCategoriesQueryTpl({
|
||||
_query: query,
|
||||
_column: column,
|
||||
_aggregation: this.getAggregationSql(),
|
||||
_aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
|
||||
}),
|
||||
categoriesSummaryMinMaxQueryTpl({
|
||||
_query: query,
|
||||
_column: column
|
||||
}),
|
||||
categoriesSummaryCountQueryTpl({
|
||||
_query: query,
|
||||
_column: column
|
||||
})
|
||||
].join(',\n')
|
||||
].join('\n');
|
||||
};
|
||||
|
||||
var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
|
||||
@@ -193,6 +254,8 @@ Aggregation.prototype.format = function(result) {
|
||||
var categories = [];
|
||||
var count = 0;
|
||||
var nulls = 0;
|
||||
var nans = 0;
|
||||
var infinities = 0;
|
||||
var minValue = 0;
|
||||
var maxValue = 0;
|
||||
var categoriesCount = 0;
|
||||
@@ -202,12 +265,15 @@ Aggregation.prototype.format = function(result) {
|
||||
var firstRow = result.rows[0];
|
||||
count = firstRow.count;
|
||||
nulls = firstRow.nulls_count;
|
||||
nans = firstRow.nans_count;
|
||||
infinities = firstRow.infinities_count;
|
||||
minValue = firstRow.min_val;
|
||||
maxValue = firstRow.max_val;
|
||||
categoriesCount = firstRow.categories_count;
|
||||
|
||||
result.rows.forEach(function(row) {
|
||||
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
|
||||
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
|
||||
'max_val', 'categories_count', 'nans_count', 'infinities_count'));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -215,6 +281,8 @@ Aggregation.prototype.format = function(result) {
|
||||
aggregation: this.aggregation,
|
||||
count: count,
|
||||
nulls: nulls,
|
||||
nans: nans,
|
||||
infinities: infinities,
|
||||
min: minValue,
|
||||
max: maxValue,
|
||||
categoriesCount: categoriesCount,
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
function BaseDataview() {}
|
||||
|
||||
module.exports = BaseDataview;
|
||||
@@ -24,3 +27,42 @@ BaseDataview.prototype.getResult = function(psql, override, callback) {
|
||||
BaseDataview.prototype.search = function(psql, userQuery, callback) {
|
||||
return callback(null, this.format({ rows: [] }));
|
||||
};
|
||||
|
||||
var FLOAT_OIDS = {
|
||||
700: true,
|
||||
701: true,
|
||||
1700: true
|
||||
};
|
||||
|
||||
var DATE_OIDS = {
|
||||
1082: true,
|
||||
1114: true,
|
||||
1184: true
|
||||
};
|
||||
|
||||
var columnTypeQueryTpl = dot.template(
|
||||
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1'
|
||||
);
|
||||
|
||||
BaseDataview.prototype.getColumnType = function (psql, column, query, callback) {
|
||||
var readOnlyTransaction = true;
|
||||
|
||||
var columnTypeQuery = columnTypeQueryTpl({
|
||||
column: column, query: query
|
||||
});
|
||||
|
||||
psql.query(columnTypeQuery, function(err, result) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
var pgType = result.rows[0].pg_typeof;
|
||||
callback(null, getPGTypeName(pgType));
|
||||
}, readOnlyTransaction);
|
||||
};
|
||||
|
||||
function getPGTypeName (pgType) {
|
||||
return {
|
||||
float: FLOAT_OIDS.hasOwnProperty(pgType),
|
||||
date: DATE_OIDS.hasOwnProperty(pgType)
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,9 +7,19 @@ dot.templateSettings.strip = false;
|
||||
|
||||
var formulaQueryTpl = dot.template([
|
||||
'SELECT',
|
||||
'{{=it._operation}}({{=it._column}}) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
' {{=it._operation}}({{=it._column}}) AS result,',
|
||||
' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
|
||||
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
|
||||
' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
|
||||
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
|
||||
'FROM ({{=it._query}}) _cdb_formula',
|
||||
'{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
|
||||
' {{=it._column}} != \'infinity\'::float',
|
||||
'AND',
|
||||
' {{=it._column}} != \'-infinity\'::float',
|
||||
'AND',
|
||||
' {{=it._column}} != \'NaN\'::float{{?}}'
|
||||
].join('\n'));
|
||||
|
||||
var VALID_OPERATIONS = {
|
||||
@@ -31,7 +41,7 @@ var TYPE = 'formula';
|
||||
}
|
||||
}
|
||||
*/
|
||||
function Formula(query, options) {
|
||||
function Formula(query, options, queries) {
|
||||
if (!_.isString(options.operation)) {
|
||||
throw new Error('Formula expects `operation` in widget options');
|
||||
}
|
||||
@@ -47,8 +57,10 @@ function Formula(query, options) {
|
||||
BaseWidget.apply(this);
|
||||
|
||||
this.query = query;
|
||||
this.queries = queries;
|
||||
this.column = options.column || '1';
|
||||
this.operation = options.operation;
|
||||
this._isFloatColumn = null;
|
||||
}
|
||||
|
||||
Formula.prototype = new BaseWidget();
|
||||
@@ -57,14 +69,27 @@ Formula.prototype.constructor = Formula;
|
||||
module.exports = Formula;
|
||||
|
||||
Formula.prototype.sql = function(psql, override, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!callback) {
|
||||
callback = override;
|
||||
override = {};
|
||||
}
|
||||
|
||||
var _query = this.query;
|
||||
if (this._isFloatColumn === null) {
|
||||
this._isFloatColumn = false;
|
||||
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
|
||||
if (!err && !!type) {
|
||||
self._isFloatColumn = type.float;
|
||||
}
|
||||
self.sql(psql, override, callback);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
var formulaSql = formulaQueryTpl({
|
||||
_query: _query,
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: this.query,
|
||||
_operation: this.operation,
|
||||
_column: this.column
|
||||
});
|
||||
@@ -78,13 +103,17 @@ Formula.prototype.format = function(result) {
|
||||
var formattedResult = {
|
||||
operation: this.operation,
|
||||
result: 0,
|
||||
nulls: 0
|
||||
nulls: 0,
|
||||
nans: 0,
|
||||
infinities: 0
|
||||
};
|
||||
|
||||
if (result.rows.length) {
|
||||
formattedResult.operation = this.operation;
|
||||
formattedResult.result = result.rows[0].result;
|
||||
formattedResult.nulls = result.rows[0].nulls_count;
|
||||
formattedResult.nans = result.rows[0].nans_count;
|
||||
formattedResult.infinities = result.rows[0].infinities_count;
|
||||
}
|
||||
|
||||
return formattedResult;
|
||||
|
||||
@@ -5,20 +5,32 @@ var debug = require('debug')('windshaft:dataview:histogram');
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var columnTypeQueryTpl = dot.template(
|
||||
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
|
||||
);
|
||||
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
|
||||
|
||||
var BIN_MIN_NUMBER = 6;
|
||||
var BIN_MAX_NUMBER = 48;
|
||||
|
||||
var filteredQueryTpl = dot.template([
|
||||
'filtered_source AS (',
|
||||
' SELECT *',
|
||||
' FROM ({{=it._query}}) _cdb_filtered_source',
|
||||
' WHERE',
|
||||
' {{=it._column}} IS NOT NULL',
|
||||
' {{?it._isFloatColumn}}AND',
|
||||
' {{=it._column}} != \'infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._column}} != \'-infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._column}} != \'NaN\'::float{{?}}',
|
||||
')'
|
||||
].join(' \n'));
|
||||
|
||||
var basicsQueryTpl = dot.template([
|
||||
'basics AS (',
|
||||
' SELECT',
|
||||
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
|
||||
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
|
||||
' FROM ({{=it._query}}) _cdb_basics',
|
||||
' FROM filtered_source',
|
||||
')'
|
||||
].join(' \n'));
|
||||
|
||||
@@ -27,7 +39,7 @@ var overrideBasicsQueryTpl = dot.template([
|
||||
' SELECT',
|
||||
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
|
||||
' avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
|
||||
' FROM ({{=it._query}}) _cdb_basics',
|
||||
' FROM filtered_source',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
@@ -38,7 +50,7 @@ var iqrQueryTpl = dot.template([
|
||||
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
|
||||
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
|
||||
' ) AS quartile',
|
||||
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
|
||||
' FROM filtered_source) _cdb_quartiles',
|
||||
' WHERE quartile = 1 or quartile = 3',
|
||||
' GROUP BY quartile',
|
||||
' ) _cdb_iqr',
|
||||
@@ -57,7 +69,7 @@ var binsQueryTpl = dot.template([
|
||||
' )',
|
||||
' )',
|
||||
' END AS bins_number',
|
||||
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
|
||||
' FROM basics, iqrange, filtered_source',
|
||||
' LIMIT 1',
|
||||
')'
|
||||
].join('\n'));
|
||||
@@ -77,11 +89,34 @@ var nullsQueryTpl = dot.template([
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var infinitiesQueryTpl = dot.template([
|
||||
'infinities AS (',
|
||||
' SELECT',
|
||||
' count(*) AS infinities_count',
|
||||
' FROM ({{=it._query}}) _cdb_histogram_infinities',
|
||||
' WHERE',
|
||||
' {{=it._column}} = \'infinity\'::float',
|
||||
' OR',
|
||||
' {{=it._column}} = \'-infinity\'::float',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var nansQueryTpl = dot.template([
|
||||
'nans AS (',
|
||||
' SELECT',
|
||||
' count(*) AS nans_count',
|
||||
' FROM ({{=it._query}}) _cdb_histogram_infinities',
|
||||
' WHERE {{=it._column}} = \'NaN\'::float',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var histogramQueryTpl = dot.template([
|
||||
'SELECT',
|
||||
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
|
||||
' bins_number,',
|
||||
' nulls_count,',
|
||||
' {{?it._isFloatColumn}}infinities_count,',
|
||||
' nans_count,{{?}}',
|
||||
' avg_val,',
|
||||
' CASE WHEN min_val = max_val',
|
||||
' THEN 0',
|
||||
@@ -91,9 +126,9 @@ var histogramQueryTpl = dot.template([
|
||||
' max({{=it._column}})::numeric AS max,',
|
||||
' avg({{=it._column}})::numeric AS avg,',
|
||||
' count(*) AS freq',
|
||||
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
|
||||
'WHERE {{=it._column}} IS NOT NULL',
|
||||
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
|
||||
'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}}, infinities, nans{{?}}',
|
||||
'GROUP BY bin, bins_number, bin_width, nulls_count,',
|
||||
' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
|
||||
'ORDER BY bin'
|
||||
].join('\n'));
|
||||
|
||||
@@ -127,50 +162,49 @@ Histogram.prototype.constructor = Histogram;
|
||||
|
||||
module.exports = Histogram;
|
||||
|
||||
var DATE_OIDS = {
|
||||
1082: true,
|
||||
1114: true,
|
||||
1184: true
|
||||
};
|
||||
|
||||
Histogram.prototype.sql = function(psql, override, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!callback) {
|
||||
callback = override;
|
||||
override = {};
|
||||
}
|
||||
|
||||
var self = this;
|
||||
|
||||
var _column = this.column;
|
||||
|
||||
var columnTypeQuery = columnTypeQueryTpl({
|
||||
column: _column, query: this.queries.no_filters
|
||||
});
|
||||
|
||||
if (this._columnType === null) {
|
||||
psql.query(columnTypeQuery, function(err, result) {
|
||||
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
|
||||
// assume numeric, will fail later
|
||||
self._columnType = 'numeric';
|
||||
if (!err && !!result.rows[0]) {
|
||||
var pgType = result.rows[0].pg_typeof;
|
||||
if (DATE_OIDS.hasOwnProperty(pgType)) {
|
||||
self._columnType = 'date';
|
||||
}
|
||||
if (!err && !!type) {
|
||||
self._columnType = Object.keys(type).find(function (key) {
|
||||
return type[key];
|
||||
});
|
||||
}
|
||||
self.sql(psql, override, callback);
|
||||
}, true); // use read-only transaction
|
||||
return null;
|
||||
}
|
||||
|
||||
var histogramSql = this._buildQuery(override);
|
||||
|
||||
return callback(null, histogramSql);
|
||||
};
|
||||
|
||||
Histogram.prototype._buildQuery = function (override) {
|
||||
var filteredQuery, basicsQuery, binsQuery;
|
||||
var _column = this.column;
|
||||
var _query = this.query;
|
||||
|
||||
if (this._columnType === 'date') {
|
||||
_column = columnCastTpl({column: _column});
|
||||
}
|
||||
|
||||
var _query = this.query;
|
||||
filteredQuery = filteredQueryTpl({
|
||||
_isFloatColumn: this._columnType === 'float',
|
||||
_query: _query,
|
||||
_column: _column
|
||||
});
|
||||
|
||||
var basicsQuery, binsQuery;
|
||||
|
||||
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
|
||||
if (this._shouldOverride(override)) {
|
||||
debug('overriding with %j', override);
|
||||
basicsQuery = overrideBasicsQueryTpl({
|
||||
_query: _query,
|
||||
@@ -190,7 +224,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
|
||||
_column: _column
|
||||
});
|
||||
|
||||
if (override && _.has(override, 'bins')) {
|
||||
if (this._shouldOverrideBins(override)) {
|
||||
binsQuery = [
|
||||
overrideBinsQueryTpl({
|
||||
_bins: override.bins
|
||||
@@ -211,18 +245,34 @@ Histogram.prototype.sql = function(psql, override, callback) {
|
||||
}
|
||||
}
|
||||
|
||||
var cteSql = [
|
||||
filteredQuery,
|
||||
basicsQuery,
|
||||
binsQuery,
|
||||
nullsQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column
|
||||
})
|
||||
];
|
||||
|
||||
var histogramSql = [
|
||||
"WITH",
|
||||
[
|
||||
basicsQuery,
|
||||
binsQuery,
|
||||
nullsQueryTpl({
|
||||
if (this._columnType === 'float') {
|
||||
cteSql.push(
|
||||
infinitiesQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column
|
||||
}),
|
||||
nansQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column
|
||||
})
|
||||
].join(',\n'),
|
||||
);
|
||||
}
|
||||
|
||||
var histogramSql = [
|
||||
"WITH",
|
||||
cteSql.join(',\n'),
|
||||
histogramQueryTpl({
|
||||
_isFloatColumn: this._columnType === 'float',
|
||||
_query: _query,
|
||||
_column: _column
|
||||
})
|
||||
@@ -230,7 +280,15 @@ Histogram.prototype.sql = function(psql, override, callback) {
|
||||
|
||||
debug(histogramSql);
|
||||
|
||||
return callback(null, histogramSql);
|
||||
return histogramSql;
|
||||
};
|
||||
|
||||
Histogram.prototype._shouldOverride = function (override) {
|
||||
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
|
||||
};
|
||||
|
||||
Histogram.prototype._shouldOverrideBins = function (override) {
|
||||
return override && _.has(override, 'bins');
|
||||
};
|
||||
|
||||
Histogram.prototype.format = function(result, override) {
|
||||
@@ -241,6 +299,8 @@ Histogram.prototype.format = function(result, override) {
|
||||
var width = getWidth(override);
|
||||
var binsStart = getBinStart(override);
|
||||
var nulls = 0;
|
||||
var infinities = 0;
|
||||
var nans = 0;
|
||||
var avg;
|
||||
|
||||
if (result.rows.length) {
|
||||
@@ -249,10 +309,12 @@ Histogram.prototype.format = function(result, override) {
|
||||
width = firstRow.bin_width || width;
|
||||
avg = firstRow.avg_val;
|
||||
nulls = firstRow.nulls_count;
|
||||
infinities = firstRow.infinities_count;
|
||||
nans = firstRow.nans_count;
|
||||
binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
|
||||
|
||||
buckets = result.rows.map(function(row) {
|
||||
return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
|
||||
return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'infinities_count', 'nans_count', 'avg_val');
|
||||
});
|
||||
}
|
||||
|
||||
@@ -261,6 +323,8 @@ Histogram.prototype.format = function(result, override) {
|
||||
bins_count: binsCount,
|
||||
bins_start: binsStart,
|
||||
nulls: nulls,
|
||||
infinities: infinities,
|
||||
nans: nans,
|
||||
avg: avg,
|
||||
bins: buckets
|
||||
};
|
||||
|
||||
@@ -1,14 +1,36 @@
|
||||
var BaseOverviewsDataview = require('./base');
|
||||
var BaseDataview = require('../aggregation');
|
||||
var debug = require('debug')('windshaft:widget:aggregation:overview');
|
||||
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var filteredQueryTpl = dot.template([
|
||||
'filtered_source AS (',
|
||||
' SELECT *',
|
||||
' FROM ({{=it._query}}) _cdb_filtered_source',
|
||||
' {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
|
||||
' {{=it._aggregationColumn}} != \'infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._aggregationColumn}} != \'-infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
|
||||
')'
|
||||
].join(' \n'));
|
||||
|
||||
var summaryQueryTpl = dot.template([
|
||||
'summary AS (',
|
||||
' SELECT',
|
||||
' sum(_feature_count) AS count,',
|
||||
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
|
||||
' {{?it._isFloatColumn}},sum(',
|
||||
' CASE',
|
||||
' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
|
||||
' THEN 1',
|
||||
' ELSE 0',
|
||||
' END',
|
||||
' ) AS infinities_count,',
|
||||
' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
|
||||
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
|
||||
')'
|
||||
].join('\n'));
|
||||
@@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
|
||||
'categories AS(',
|
||||
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
|
||||
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
|
||||
' FROM ({{=it._query}}) _cdb_aggregation_all',
|
||||
' FROM filtered_source',
|
||||
' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
|
||||
' GROUP BY {{=it._column}}',
|
||||
' ORDER BY 2 DESC',
|
||||
@@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([
|
||||
' SELECT count(1) AS categories_count',
|
||||
' FROM (',
|
||||
' SELECT {{=it._column}} AS category',
|
||||
' FROM ({{=it._query}}) _cdb_categories',
|
||||
' FROM filtered_source',
|
||||
' GROUP BY {{=it._column}}',
|
||||
' ) _cdb_categories_count',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var rankedAggregationQueryTpl = dot.template([
|
||||
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
|
||||
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
|
||||
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
|
||||
' WHERE rank < {{=it._limit}}',
|
||||
'UNION ALL',
|
||||
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
|
||||
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
|
||||
' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
' FROM categories, summary, categories_summary_min_max, categories_summary_count',
|
||||
' WHERE rank >= {{=it._limit}}',
|
||||
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
|
||||
'GROUP BY nulls_count, min_val, max_val, count,',
|
||||
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
|
||||
].join('\n'));
|
||||
|
||||
var aggregationQueryTpl = dot.template([
|
||||
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
|
||||
' nulls_count, min_val, max_val, count, categories_count',
|
||||
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
|
||||
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
|
||||
' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
|
||||
'GROUP BY category, nulls_count, min_val, max_val, count,',
|
||||
' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
|
||||
'ORDER BY value DESC'
|
||||
].join('\n'));
|
||||
|
||||
var CATEGORIES_LIMIT = 6;
|
||||
|
||||
function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
|
||||
function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
|
||||
|
||||
this.query = query;
|
||||
this.queries = queries;
|
||||
this.column = options.column;
|
||||
this.aggregation = options.aggregation;
|
||||
this.aggregationColumn = options.aggregationColumn;
|
||||
this._isFloatColumn = null;
|
||||
}
|
||||
|
||||
Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
|
||||
@@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation;
|
||||
module.exports = Aggregation;
|
||||
|
||||
Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!callback) {
|
||||
callback = override;
|
||||
override = {};
|
||||
}
|
||||
|
||||
var _query = this.rewrittenQuery(this.query);
|
||||
var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
|
||||
|
||||
if (this.aggregationColumn && this._isFloatColumn === null) {
|
||||
this._isFloatColumn = false;
|
||||
this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
|
||||
if (!err && !!type) {
|
||||
self._isFloatColumn = type.float;
|
||||
}
|
||||
self.sql(psql, override, callback);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
var aggregationSql;
|
||||
if (!!override.ownFilter) {
|
||||
aggregationSql = [
|
||||
"WITH",
|
||||
[
|
||||
summaryQueryTpl({
|
||||
filteredQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column
|
||||
_column: this.column,
|
||||
_aggregationColumn: _aggregationColumn
|
||||
}),
|
||||
summaryQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregationColumn: _aggregationColumn
|
||||
}),
|
||||
rankedCategoriesQueryTpl({
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregation: this.getAggregationSql(),
|
||||
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
|
||||
_aggregationColumn: _aggregationColumn
|
||||
}),
|
||||
categoriesSummaryMinMaxQueryTpl({
|
||||
_query: _query,
|
||||
@@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
})
|
||||
].join(',\n'),
|
||||
aggregationQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregation: this.getAggregationSql(),
|
||||
@@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
aggregationSql = [
|
||||
"WITH",
|
||||
[
|
||||
summaryQueryTpl({
|
||||
filteredQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column
|
||||
_column: this.column,
|
||||
_aggregationColumn: _aggregationColumn
|
||||
}),
|
||||
summaryQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregationColumn: _aggregationColumn
|
||||
}),
|
||||
rankedCategoriesQueryTpl({
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_aggregation: this.getAggregationSql(),
|
||||
_aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
|
||||
_aggregationColumn: _aggregationColumn
|
||||
}),
|
||||
categoriesSummaryMinMaxQueryTpl({
|
||||
_query: _query,
|
||||
@@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
})
|
||||
].join(',\n'),
|
||||
rankedAggregationQueryTpl({
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: _query,
|
||||
_column: this.column,
|
||||
_limit: CATEGORIES_LIMIT
|
||||
@@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) {
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
debug(aggregationSql);
|
||||
|
||||
return callback(null, aggregationSql);
|
||||
};
|
||||
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
var _ = require('underscore');
|
||||
var BaseDataview = require('../base');
|
||||
|
||||
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
|
||||
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
|
||||
this.BaseDataview = BaseDataview;
|
||||
this.query = query;
|
||||
this.queryOptions = queryOptions;
|
||||
this.queryRewriter = queryRewriter;
|
||||
this.queryRewriteData = queryRewriteData;
|
||||
this.options = options;
|
||||
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
|
||||
this.queries = queries;
|
||||
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
|
||||
}
|
||||
|
||||
module.exports = BaseOverviewsDataview;
|
||||
|
||||
@@ -1,34 +1,61 @@
|
||||
var BaseOverviewsDataview = require('./base');
|
||||
var BaseDataview = require('../formula');
|
||||
var debug = require('debug')('windshaft:widget:formula:overview');
|
||||
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var formulaQueryTpls = {
|
||||
'count': dot.template([
|
||||
'SELECT',
|
||||
'sum(_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
'sum': dot.template([
|
||||
'SELECT',
|
||||
'sum({{=it._column}}*_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
'avg': dot.template([
|
||||
'SELECT',
|
||||
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
'count': dot.template([
|
||||
'SELECT',
|
||||
'sum(_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
|
||||
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
|
||||
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
|
||||
'FROM ({{=it._query}}) _cdb_formula'
|
||||
].join('\n')),
|
||||
'sum': dot.template([
|
||||
'SELECT',
|
||||
'sum({{=it._column}}*_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
|
||||
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
|
||||
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
|
||||
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
|
||||
'FROM ({{=it._query}}) _cdb_formula',
|
||||
'{{?it._isFloatColumn}}WHERE',
|
||||
' {{=it._column}} != \'infinity\'::float',
|
||||
'AND',
|
||||
' {{=it._column}} != \'-infinity\'::float',
|
||||
'AND',
|
||||
' {{=it._column}} != \'NaN\'::float{{?}}'
|
||||
].join('\n')),
|
||||
'avg': dot.template([
|
||||
'SELECT',
|
||||
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
|
||||
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
|
||||
'{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
|
||||
' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
|
||||
',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
|
||||
' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
|
||||
'FROM ({{=it._query}}) _cdb_formula',
|
||||
'{{?it._isFloatColumn}}WHERE',
|
||||
' {{=it._column}} != \'infinity\'::float',
|
||||
'AND',
|
||||
' {{=it._column}} != \'-infinity\'::float',
|
||||
'AND',
|
||||
' {{=it._column}} != \'NaN\'::float{{?}}'
|
||||
].join('\n')),
|
||||
};
|
||||
|
||||
function Formula(query, options, queryRewriter, queryRewriteData, params) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
|
||||
function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
|
||||
this.column = options.column || '1';
|
||||
this.operation = options.operation;
|
||||
this._isFloatColumn = null;
|
||||
this.queries = queries;
|
||||
}
|
||||
|
||||
Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
|
||||
@@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula;
|
||||
|
||||
module.exports = Formula;
|
||||
|
||||
Formula.prototype.sql = function(psql, override, callback) {
|
||||
Formula.prototype.sql = function (psql, override, callback) {
|
||||
var self = this;
|
||||
var formulaQueryTpl = formulaQueryTpls[this.operation];
|
||||
|
||||
if ( formulaQueryTpl ) {
|
||||
if (formulaQueryTpl) {
|
||||
// supported formula for use with overviews
|
||||
if (this._isFloatColumn === null) {
|
||||
this._isFloatColumn = false;
|
||||
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
|
||||
if (!err && !!type) {
|
||||
self._isFloatColumn = type.float;
|
||||
}
|
||||
self.sql(psql, override, callback);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
var formulaSql = formulaQueryTpl({
|
||||
_query: this.rewrittenQuery(this.query),
|
||||
_isFloatColumn: this._isFloatColumn,
|
||||
_query: this.rewrittenQuery(this.query),
|
||||
_operation: this.operation,
|
||||
_column: this.column
|
||||
_column: this.column
|
||||
});
|
||||
|
||||
callback = callback || override;
|
||||
|
||||
debug(formulaSql);
|
||||
|
||||
return callback(null, formulaSql);
|
||||
}
|
||||
|
||||
|
||||
// default behaviour
|
||||
return this.defaultSql(psql, override, callback);
|
||||
};
|
||||
|
||||
@@ -1,23 +1,35 @@
|
||||
var _ = require('underscore');
|
||||
var BaseOverviewsDataview = require('./base');
|
||||
var BaseDataview = require('../histogram');
|
||||
var debug = require('debug')('windshaft:dataview:histogram:overview');
|
||||
|
||||
var dot = require('dot');
|
||||
dot.templateSettings.strip = false;
|
||||
|
||||
var columnTypeQueryTpl = dot.template(
|
||||
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
|
||||
);
|
||||
|
||||
var BIN_MIN_NUMBER = 6;
|
||||
var BIN_MAX_NUMBER = 48;
|
||||
|
||||
var filteredQueryTpl = dot.template([
|
||||
'filtered_source AS (',
|
||||
' SELECT *',
|
||||
' FROM ({{=it._query}}) _cdb_filtered_source',
|
||||
' WHERE',
|
||||
' {{=it._column}} IS NOT NULL',
|
||||
' {{?it._isFloatColumn}}AND',
|
||||
' {{=it._column}} != \'infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._column}} != \'-infinity\'::float',
|
||||
' AND',
|
||||
' {{=it._column}} != \'NaN\'::float{{?}}',
|
||||
')'
|
||||
].join(' \n'));
|
||||
|
||||
var basicsQueryTpl = dot.template([
|
||||
'basics AS (',
|
||||
' SELECT',
|
||||
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
|
||||
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
|
||||
' FROM ({{=it._query}}) _cdb_basics',
|
||||
' FROM filtered_source',
|
||||
')'
|
||||
].join(' \n'));
|
||||
|
||||
@@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([
|
||||
' SELECT',
|
||||
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
|
||||
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
|
||||
' FROM ({{=it._query}}) _cdb_basics',
|
||||
' FROM filtered_source',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
@@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([
|
||||
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
|
||||
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
|
||||
' ) AS quartile',
|
||||
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
|
||||
' FROM filtered_source) _cdb_quartiles',
|
||||
' WHERE quartile = 1 or quartile = 3',
|
||||
' GROUP BY quartile',
|
||||
' ) _cdb_iqr',
|
||||
@@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([
|
||||
' )',
|
||||
' )',
|
||||
' END AS bins_number',
|
||||
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
|
||||
' FROM basics, iqrange, filtered_source',
|
||||
' LIMIT 1',
|
||||
')'
|
||||
].join('\n'));
|
||||
@@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var infinitiesQueryTpl = dot.template([
|
||||
'infinities AS (',
|
||||
' SELECT',
|
||||
' count(*) AS infinities_count',
|
||||
' FROM ({{=it._query}}) _cdb_histogram_infinities',
|
||||
' WHERE',
|
||||
' {{=it._column}} = \'infinity\'::float',
|
||||
' OR',
|
||||
' {{=it._column}} = \'-infinity\'::float',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var nansQueryTpl = dot.template([
|
||||
'nans AS (',
|
||||
' SELECT',
|
||||
' count(*) AS nans_count',
|
||||
' FROM ({{=it._query}}) _cdb_histogram_infinities',
|
||||
' WHERE {{=it._column}} = \'NaN\'::float',
|
||||
')'
|
||||
].join('\n'));
|
||||
|
||||
var histogramQueryTpl = dot.template([
|
||||
'SELECT',
|
||||
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
|
||||
' bins_number,',
|
||||
' nulls_count,',
|
||||
' {{?it._isFloatColumn}}infinities_count,',
|
||||
' nans_count,{{?}}',
|
||||
' avg_val,',
|
||||
' CASE WHEN min_val = max_val',
|
||||
' THEN 0',
|
||||
@@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([
|
||||
' max({{=it._column}})::numeric AS max,',
|
||||
' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
|
||||
' sum(_feature_count) AS freq',
|
||||
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
|
||||
'WHERE {{=it._column}} IS NOT NULL',
|
||||
'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
|
||||
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
|
||||
' {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
|
||||
'ORDER BY bin'
|
||||
].join('\n'));
|
||||
|
||||
function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
|
||||
|
||||
this.query = query;
|
||||
this.queries = queries;
|
||||
@@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram;
|
||||
|
||||
module.exports = Histogram;
|
||||
|
||||
|
||||
var DATE_OIDS = {
|
||||
1082: true,
|
||||
1114: true,
|
||||
1184: true
|
||||
};
|
||||
|
||||
Histogram.prototype.sql = function(psql, override, callback) {
|
||||
var self = this;
|
||||
|
||||
if (!callback) {
|
||||
callback = override;
|
||||
override = {};
|
||||
}
|
||||
|
||||
var self = this;
|
||||
|
||||
var _column = this.column;
|
||||
|
||||
var columnTypeQuery = columnTypeQueryTpl({
|
||||
column: _column, query: this.rewrittenQuery(this.queries.no_filters)
|
||||
});
|
||||
|
||||
if (this._columnType === null) {
|
||||
psql.query(columnTypeQuery, function(err, result) {
|
||||
this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
|
||||
// assume numeric, will fail later
|
||||
self._columnType = 'numeric';
|
||||
if (!err && !!result.rows[0]) {
|
||||
var pgType = result.rows[0].pg_typeof;
|
||||
if (DATE_OIDS.hasOwnProperty(pgType)) {
|
||||
self._columnType = 'date';
|
||||
}
|
||||
if (!err && !!type) {
|
||||
self._columnType = Object.keys(type).find(function (key) {
|
||||
return type[key];
|
||||
});
|
||||
}
|
||||
self.sql(psql, override, callback);
|
||||
}, true); // use read-only transaction
|
||||
@@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) {
|
||||
return this.defaultSql(psql, override, callback);
|
||||
}
|
||||
|
||||
var histogramSql = this._buildQuery(override);
|
||||
|
||||
return callback(null, histogramSql);
|
||||
};
|
||||
|
||||
Histogram.prototype._buildQuery = function (override) {
|
||||
var filteredQuery, basicsQuery, binsQuery;
|
||||
var _column = this.column;
|
||||
var _query = this.rewrittenQuery(this.query);
|
||||
|
||||
var basicsQuery, binsQuery;
|
||||
filteredQuery = filteredQueryTpl({
|
||||
_isFloatColumn: this._columnType === 'float',
|
||||
_query: _query,
|
||||
_column: _column
|
||||
});
|
||||
|
||||
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
|
||||
if (this._shouldOverride(override)) {
|
||||
debug('overriding with %j', override);
|
||||
basicsQuery = overrideBasicsQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column,
|
||||
@@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
|
||||
_column: _column
|
||||
});
|
||||
|
||||
if (override && _.has(override, 'bins')) {
|
||||
if (this._shouldOverrideBins(override)) {
|
||||
binsQuery = [
|
||||
overrideBinsQueryTpl({
|
||||
_bins: override.bins
|
||||
@@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) {
|
||||
}
|
||||
}
|
||||
|
||||
var cteSql = [
|
||||
filteredQuery,
|
||||
basicsQuery,
|
||||
binsQuery,
|
||||
nullsQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column
|
||||
})
|
||||
];
|
||||
|
||||
var histogramSql = [
|
||||
"WITH",
|
||||
[
|
||||
basicsQuery,
|
||||
binsQuery,
|
||||
nullsQueryTpl({
|
||||
if (this._columnType === 'float') {
|
||||
cteSql.push(
|
||||
infinitiesQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column
|
||||
}),
|
||||
nansQueryTpl({
|
||||
_query: _query,
|
||||
_column: _column
|
||||
})
|
||||
].join(',\n'),
|
||||
);
|
||||
}
|
||||
|
||||
var histogramSql = [
|
||||
"WITH",
|
||||
cteSql.join(',\n'),
|
||||
histogramQueryTpl({
|
||||
_isFloatColumn: this._columnType === 'float',
|
||||
_query: _query,
|
||||
_column: _column
|
||||
})
|
||||
].join('\n');
|
||||
|
||||
return callback(null, histogramSql);
|
||||
debug(histogramSql);
|
||||
|
||||
return histogramSql;
|
||||
};
|
||||
|
||||
Histogram.prototype._shouldOverride = function (override) {
|
||||
return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
|
||||
};
|
||||
|
||||
Histogram.prototype._shouldOverrideBins = function (override) {
|
||||
return override && _.has(override, 'bins');
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
var BaseOverviewsDataview = require('./base');
|
||||
var BaseDataview = require('../list');
|
||||
|
||||
function List(query, options, queryRewriter, queryRewriteData, params) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
|
||||
function List(query, options, queryRewriter, queryRewriteData, params, queries) {
|
||||
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
|
||||
}
|
||||
|
||||
List.prototype = Object.create(BaseOverviewsDataview.prototype);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "3.9.2",
|
||||
"version": "3.9.5",
|
||||
"description": "A map tile server for CartoDB",
|
||||
"keywords": [
|
||||
"cartodb"
|
||||
@@ -20,7 +20,7 @@
|
||||
],
|
||||
"dependencies": {
|
||||
"body-parser": "~1.14.0",
|
||||
"camshaft": "0.55.4",
|
||||
"camshaft": "0.55.6",
|
||||
"cartodb-psql": "0.8.0",
|
||||
"cartodb-query-tables": "0.2.0",
|
||||
"cartodb-redis": "0.13.2",
|
||||
|
||||
@@ -146,3 +146,152 @@ describe('aggregations happy cases', function() {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('aggregation-dataview: special float values', function() {
|
||||
|
||||
afterEach(function(done) {
|
||||
if (this.testClient) {
|
||||
this.testClient.drain(done);
|
||||
} else {
|
||||
done();
|
||||
}
|
||||
});
|
||||
|
||||
function createMapConfig(layers, dataviews, analysis) {
|
||||
return {
|
||||
version: '1.5.0',
|
||||
layers: layers,
|
||||
dataviews: dataviews || {},
|
||||
analyses: analysis || []
|
||||
};
|
||||
}
|
||||
|
||||
var mapConfig = createMapConfig(
|
||||
[
|
||||
{
|
||||
"type": "cartodb",
|
||||
"options": {
|
||||
"source": {
|
||||
"id": "a0"
|
||||
},
|
||||
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
|
||||
"cartocss_version": "2.3.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
{
|
||||
val_aggregation: {
|
||||
source: {
|
||||
id: 'a0'
|
||||
},
|
||||
type: 'aggregation',
|
||||
options: {
|
||||
column: 'cat',
|
||||
aggregation: 'avg',
|
||||
aggregationColumn: 'val'
|
||||
}
|
||||
},
|
||||
sum_aggregation_numeric: {
|
||||
source: {
|
||||
id: 'a1'
|
||||
},
|
||||
type: 'aggregation',
|
||||
options: {
|
||||
column: 'cat',
|
||||
aggregation: 'sum',
|
||||
aggregationColumn: 'val'
|
||||
}
|
||||
}
|
||||
},
|
||||
[
|
||||
{
|
||||
"id": "a0",
|
||||
"type": "source",
|
||||
"params": {
|
||||
"query": [
|
||||
'SELECT',
|
||||
' null::geometry the_geom_webmercator,',
|
||||
' CASE',
|
||||
' WHEN x % 4 = 0 THEN \'infinity\'::float',
|
||||
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
|
||||
' WHEN x % 4 = 2 THEN \'NaN\'::float',
|
||||
' ELSE x',
|
||||
' END AS val,',
|
||||
' CASE',
|
||||
' WHEN x % 2 = 0 THEN \'category_1\'',
|
||||
' ELSE \'category_2\'',
|
||||
' END AS cat',
|
||||
'FROM generate_series(1, 1000) x'
|
||||
].join('\n')
|
||||
}
|
||||
}, {
|
||||
"id": "a1",
|
||||
"type": "source",
|
||||
"params": {
|
||||
"query": [
|
||||
'SELECT',
|
||||
' null::geometry the_geom_webmercator,',
|
||||
' CASE',
|
||||
' WHEN x % 3 = 0 THEN \'NaN\'::numeric',
|
||||
' WHEN x % 3 = 1 THEN x',
|
||||
' ELSE x',
|
||||
' END AS val,',
|
||||
' CASE',
|
||||
' WHEN x % 2 = 0 THEN \'category_1\'',
|
||||
' ELSE \'category_2\'',
|
||||
' END AS cat',
|
||||
'FROM generate_series(1, 1000) x'
|
||||
].join('\n')
|
||||
}
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
// Source a0
|
||||
// -----------------------------------------------
|
||||
// the_geom_webmercator | val | cat
|
||||
// ----------------------+-----------+------------
|
||||
// | -Infinity | category_2
|
||||
// | NaN | category_1
|
||||
// | 3 | category_2
|
||||
// | Infinity | category_1
|
||||
// | -Infinity | category_2
|
||||
// | NaN | category_1
|
||||
// | 7 | category_2
|
||||
// | Infinity | category_1
|
||||
// | -Infinity | category_2
|
||||
// | NaN | category_1
|
||||
// | 11 | category_2
|
||||
// | " | "
|
||||
|
||||
var filters = [{ own_filter: 0 }, {}];
|
||||
filters.forEach(function (filter) {
|
||||
it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) {
|
||||
this.testClient = new TestClient(mapConfig, 1234);
|
||||
this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) {
|
||||
assert.ifError(err);
|
||||
assert.ok(dataview.infinities === (250 + 250));
|
||||
assert.ok(dataview.nans === 250);
|
||||
assert.ok(dataview.categories.length === 1);
|
||||
dataview.categories.forEach(function (category) {
|
||||
assert.ok(category.category === 'category_2');
|
||||
assert.ok(category.value === 501);
|
||||
});
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) {
|
||||
this.testClient = new TestClient(mapConfig, 1234);
|
||||
this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) {
|
||||
assert.ifError(err);
|
||||
assert.ok(dataview.nans === 333);
|
||||
assert.ok(dataview.categories.length === 2);
|
||||
dataview.categories.forEach(function (category) {
|
||||
assert.ok(category.value !== null);
|
||||
});
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
80
test/acceptance/dataviews/formula.js
Normal file
80
test/acceptance/dataviews/formula.js
Normal file
@@ -0,0 +1,80 @@
|
||||
require('../../support/test_helper');
|
||||
var assert = require('../../support/assert');
|
||||
var TestClient = require('../../support/test-client');
|
||||
|
||||
function createMapConfig(layers, dataviews, analysis) {
|
||||
return {
|
||||
version: '1.5.0',
|
||||
layers: layers,
|
||||
dataviews: dataviews || {},
|
||||
analyses: analysis || []
|
||||
};
|
||||
}
|
||||
|
||||
describe('formula-dataview: special float values', function() {
|
||||
|
||||
afterEach(function(done) {
|
||||
if (this.testClient) {
|
||||
this.testClient.drain(done);
|
||||
} else {
|
||||
done();
|
||||
}
|
||||
});
|
||||
|
||||
var mapConfig = createMapConfig(
|
||||
[
|
||||
{
|
||||
"type": "cartodb",
|
||||
"options": {
|
||||
"source": {
|
||||
"id": "a0"
|
||||
},
|
||||
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
|
||||
"cartocss_version": "2.3.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
{
|
||||
val_formula: {
|
||||
source: {
|
||||
id: 'a0'
|
||||
},
|
||||
type: 'formula',
|
||||
options: {
|
||||
column: 'val',
|
||||
operation: 'avg'
|
||||
}
|
||||
}
|
||||
},
|
||||
[
|
||||
{
|
||||
"id": "a0",
|
||||
"type": "source",
|
||||
"params": {
|
||||
"query": [
|
||||
'SELECT',
|
||||
' null::geometry the_geom_webmercator,',
|
||||
' CASE',
|
||||
' WHEN x % 4 = 0 THEN \'infinity\'::float',
|
||||
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
|
||||
' WHEN x % 4 = 2 THEN \'NaN\'::float',
|
||||
' ELSE x',
|
||||
' END AS val',
|
||||
'FROM generate_series(1, 1000) x'
|
||||
].join('\n')
|
||||
}
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
it('should filter infinities out and count them in the summary', function(done) {
|
||||
this.testClient = new TestClient(mapConfig, 1234);
|
||||
this.testClient.getDataview('val_formula', {}, function(err, dataview) {
|
||||
assert.ok(!err, err);
|
||||
assert.equal(dataview.result, 501);
|
||||
assert.ok(dataview.infinities === (250 + 250));
|
||||
assert.ok(dataview.nans === 250);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,15 @@ require('../../support/test_helper');
|
||||
var assert = require('../../support/assert');
|
||||
var TestClient = require('../../support/test-client');
|
||||
|
||||
function createMapConfig(layers, dataviews, analysis) {
|
||||
return {
|
||||
version: '1.5.0',
|
||||
layers: layers,
|
||||
dataviews: dataviews || {},
|
||||
analyses: analysis || []
|
||||
};
|
||||
}
|
||||
|
||||
describe('histogram-dataview', function() {
|
||||
|
||||
afterEach(function(done) {
|
||||
@@ -13,15 +22,6 @@ describe('histogram-dataview', function() {
|
||||
}
|
||||
});
|
||||
|
||||
function createMapConfig(layers, dataviews, analysis) {
|
||||
return {
|
||||
version: '1.5.0',
|
||||
layers: layers,
|
||||
dataviews: dataviews || {},
|
||||
analyses: analysis || []
|
||||
};
|
||||
}
|
||||
|
||||
var mapConfig = createMapConfig(
|
||||
[
|
||||
{
|
||||
@@ -89,7 +89,6 @@ describe('histogram-dataview', function() {
|
||||
this.testClient = new TestClient(mapConfig, 1234);
|
||||
this.testClient.getDataview('pop_max_histogram', params, function(err, res) {
|
||||
assert.ok(!err, err);
|
||||
|
||||
assert.ok(res.errors);
|
||||
assert.equal(res.errors.length, 1);
|
||||
assert.ok(res.errors[0].match(/Invalid number format for parameter 'bins'/));
|
||||
@@ -98,3 +97,70 @@ describe('histogram-dataview', function() {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('histogram-dataview: special float valuer', function() {
|
||||
|
||||
afterEach(function(done) {
|
||||
if (this.testClient) {
|
||||
this.testClient.drain(done);
|
||||
} else {
|
||||
done();
|
||||
}
|
||||
});
|
||||
|
||||
var mapConfig = createMapConfig(
|
||||
[
|
||||
{
|
||||
"type": "cartodb",
|
||||
"options": {
|
||||
"source": {
|
||||
"id": "a0"
|
||||
},
|
||||
"cartocss": "#points { marker-width: 10; marker-fill: red; }",
|
||||
"cartocss_version": "2.3.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
{
|
||||
val_histogram: {
|
||||
source: {
|
||||
id: 'a0'
|
||||
},
|
||||
type: 'histogram',
|
||||
options: {
|
||||
column: 'val'
|
||||
}
|
||||
}
|
||||
},
|
||||
[
|
||||
{
|
||||
"id": "a0",
|
||||
"type": "source",
|
||||
"params": {
|
||||
"query": [
|
||||
'SELECT',
|
||||
' null::geometry the_geom_webmercator,',
|
||||
' CASE',
|
||||
' WHEN x % 4 = 0 THEN \'infinity\'::float',
|
||||
' WHEN x % 4 = 1 THEN \'-infinity\'::float',
|
||||
' WHEN x % 4 = 2 THEN \'NaN\'::float',
|
||||
' ELSE x',
|
||||
' END AS val',
|
||||
'FROM generate_series(1, 1000) x'
|
||||
].join('\n')
|
||||
}
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
it('should filter infinities out and count them in the summary', function(done) {
|
||||
this.testClient = new TestClient(mapConfig, 1234);
|
||||
this.testClient.getDataview('val_histogram', {}, function(err, dataview) {
|
||||
assert.ok(!err, err);
|
||||
assert.ok(dataview.infinities === (250 + 250));
|
||||
assert.ok(dataview.nans === 250);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() {
|
||||
params: {
|
||||
query: 'select * from test_table_overviews'
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'data-source-special-float-values',
|
||||
type: 'source',
|
||||
params: {
|
||||
query: 'select * from test_special_float_values_table_overviews'
|
||||
}
|
||||
}
|
||||
],
|
||||
dataviews: {
|
||||
@@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() {
|
||||
aggregationColumn: 'name',
|
||||
}
|
||||
},
|
||||
test_categories_special_values: {
|
||||
type: 'aggregation',
|
||||
source: {
|
||||
id: 'data-source-special-float-values'
|
||||
},
|
||||
options: {
|
||||
column: 'name',
|
||||
aggregation: 'sum',
|
||||
aggregationColumn: 'value',
|
||||
}
|
||||
},
|
||||
test_histogram: {
|
||||
type: 'histogram',
|
||||
source: {id: 'data-source'},
|
||||
@@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() {
|
||||
bins: 2
|
||||
}
|
||||
},
|
||||
test_histogram_special_values: {
|
||||
type: 'histogram',
|
||||
source: {
|
||||
id: 'data-source-special-float-values'
|
||||
},
|
||||
options: {
|
||||
column: 'value',
|
||||
bins: 2
|
||||
}
|
||||
},
|
||||
test_avg: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
@@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() {
|
||||
operation: 'avg'
|
||||
}
|
||||
},
|
||||
test_formula_sum_special_values: {
|
||||
type: 'formula',
|
||||
source: {
|
||||
id: 'data-source-special-float-values'
|
||||
},
|
||||
options: {
|
||||
column: 'value',
|
||||
operation: 'sum'
|
||||
}
|
||||
},
|
||||
test_count: {
|
||||
type: 'formula',
|
||||
source: {id: 'data-source'},
|
||||
@@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() {
|
||||
cartocss_version: '2.3.0',
|
||||
source: { id: 'data-source' }
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'mapnik',
|
||||
options: {
|
||||
sql: 'select * from test_special_float_values_table_overviews',
|
||||
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
|
||||
cartocss_version: '2.3.0',
|
||||
source: {
|
||||
id: 'data-source-special-float-values'
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
@@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"sum",
|
||||
"result":15,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"nulls":0,
|
||||
"type":"formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"avg",
|
||||
"result":3,
|
||||
"nulls":0,
|
||||
"type":"formula",
|
||||
"infinities": 0,
|
||||
"nans": 0
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"count",
|
||||
"result":5,
|
||||
"nulls":0,
|
||||
"type":"formula",
|
||||
"infinities": 0,
|
||||
"nans": 0
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation": "max",
|
||||
"result": 5,
|
||||
"nulls": 0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type": "formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation": "min",
|
||||
"result": 1,
|
||||
"nulls": 0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type": "formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"sum",
|
||||
"result":15,
|
||||
"nulls":0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type":"formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"sum",
|
||||
"result":1,
|
||||
"nulls":0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type":"formula"
|
||||
});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
@@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"avg",
|
||||
"result":1,
|
||||
"nulls":0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type":"formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"count",
|
||||
"result":1,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"nulls":0,
|
||||
"type":"formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation": "max",
|
||||
"result": 1,
|
||||
"nulls": 0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type": "formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation": "min",
|
||||
"result": 1,
|
||||
"nulls": 0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type": "formula"
|
||||
});
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
@@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
|
||||
assert.deepEqual(formula_result, {
|
||||
"operation":"sum",
|
||||
"result":1,
|
||||
"nulls":0,
|
||||
"infinities": 0,
|
||||
"nans": 0,
|
||||
"type":"formula"
|
||||
});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
@@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() {
|
||||
|
||||
});
|
||||
|
||||
describe('aggregation special float values', function () {
|
||||
var params = {};
|
||||
|
||||
it("should expose an aggregation dataview filtering special float values out", function (done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_categories_special_values', params, function (err, dataview) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(dataview, {
|
||||
aggregation: 'sum',
|
||||
count: 5,
|
||||
nulls: 0,
|
||||
nans: 1,
|
||||
infinities: 1,
|
||||
min: 6,
|
||||
max: 6,
|
||||
categoriesCount: 1,
|
||||
categories: [ { category: 'Hawai', value: 6, agg: false } ],
|
||||
type: 'aggregation'
|
||||
});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it('should expose a histogram dataview filtering special float values out', function (done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(dataview, {
|
||||
bin_width: 0,
|
||||
bins_count: 1,
|
||||
bins_start: 3,
|
||||
nulls: 0,
|
||||
infinities: 1,
|
||||
nans: 1,
|
||||
avg: 3,
|
||||
bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ],
|
||||
type: 'histogram'
|
||||
});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
|
||||
it('should expose a formula (sum) dataview filtering special float values out', function (done) {
|
||||
var testClient = new TestClient(overviewsMapConfig);
|
||||
testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
assert.deepEqual(dataview, {
|
||||
operation: 'sum',
|
||||
result: 6,
|
||||
nulls: 0,
|
||||
nans: 1,
|
||||
infinities: 1,
|
||||
type: 'formula'
|
||||
});
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -339,6 +339,78 @@ INSERT INTO _vovw_2_test_table_overviews VALUES
|
||||
INSERT INTO _vovw_1_test_table_overviews VALUES
|
||||
('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
|
||||
|
||||
-- table with overviews whit special float values
|
||||
|
||||
CREATE TABLE test_special_float_values_table_overviews (
|
||||
cartodb_id integer NOT NULL,
|
||||
name character varying,
|
||||
address character varying,
|
||||
value float8,
|
||||
the_geom geometry,
|
||||
the_geom_webmercator geometry,
|
||||
_feature_count integer,
|
||||
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
|
||||
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
|
||||
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
|
||||
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
|
||||
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
|
||||
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
|
||||
);
|
||||
|
||||
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
|
||||
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
|
||||
|
||||
CREATE SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
NO MINVALUE
|
||||
NO MAXVALUE
|
||||
CACHE 1;
|
||||
|
||||
ALTER SEQUENCE test_special_float_values_table_overviews_cartodb_id_seq OWNED BY test_special_float_values_table_overviews.cartodb_id;
|
||||
|
||||
SELECT pg_catalog.setval('test_special_float_values_table_overviews_cartodb_id_seq', 60, true);
|
||||
|
||||
ALTER TABLE test_special_float_values_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_special_float_values_table_overviews_cartodb_id_seq'::regclass);
|
||||
|
||||
INSERT INTO test_special_float_values_table_overviews VALUES
|
||||
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
|
||||
(2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
|
||||
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
|
||||
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
|
||||
(5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 'infinity'::float, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
|
||||
|
||||
ALTER TABLE ONLY test_special_float_values_table_overviews ADD CONSTRAINT test_special_float_values_table_overviews_pkey PRIMARY KEY (cartodb_id);
|
||||
|
||||
CREATE INDEX test_special_float_values_table_overviews_the_geom_idx ON test_special_float_values_table_overviews USING gist (the_geom);
|
||||
CREATE INDEX test_special_float_values_table_overviews_the_geom_webmercator_idx ON test_special_float_values_table_overviews USING gist (the_geom_webmercator);
|
||||
|
||||
GRANT ALL ON TABLE test_special_float_values_table_overviews TO :TESTUSER;
|
||||
GRANT SELECT ON TABLE test_special_float_values_table_overviews TO :PUBLICUSER;
|
||||
|
||||
CREATE TABLE _vovw_1_test_special_float_values_table_overviews (
|
||||
cartodb_id integer NOT NULL,
|
||||
name character varying,
|
||||
address character varying,
|
||||
value float8,
|
||||
the_geom geometry,
|
||||
the_geom_webmercator geometry,
|
||||
_feature_count integer,
|
||||
CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
|
||||
CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
|
||||
CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
|
||||
CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
|
||||
CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
|
||||
CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
|
||||
);
|
||||
|
||||
GRANT ALL ON TABLE _vovw_1_test_special_float_values_table_overviews TO :TESTUSER;
|
||||
GRANT SELECT ON TABLE _vovw_1_test_special_float_values_table_overviews TO :PUBLICUSER;
|
||||
|
||||
INSERT INTO _vovw_1_test_special_float_values_table_overviews VALUES
|
||||
(1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 2),
|
||||
(3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 'NaN'::float, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
|
||||
(4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 'infinity'::float, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 2);
|
||||
|
||||
-- analysis tables -----------------------------------------------
|
||||
|
||||
|
||||
@@ -435,7 +435,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
|
||||
}
|
||||
|
||||
params.placeholders = params.placeholders || {};
|
||||
|
||||
|
||||
assert.response(server,
|
||||
{
|
||||
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
|
||||
|
||||
@@ -198,9 +198,9 @@ camelcase@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
|
||||
|
||||
camshaft@0.55.4:
|
||||
version "0.55.4"
|
||||
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.4.tgz#9b83e2fd4adc0f471976d7b0ef319e28b8864adc"
|
||||
camshaft@0.55.6:
|
||||
version "0.55.6"
|
||||
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.55.6.tgz#11af28051c3b911fb023ae1cafb165bbd040f174"
|
||||
dependencies:
|
||||
async "^1.5.2"
|
||||
bunyan "1.8.1"
|
||||
|
||||
Reference in New Issue
Block a user