diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index ea191bea..0396e2f8 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,44 +1,42 @@ const BaseHistogram = require('./base-histogram'); const debug = require('debug')('windshaft:dataview:numeric-histogram'); +/** Cast the column column to epoch */ const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; -const filterOutSpecialNumericValues = ctx => ` - ${ctx.column} != 'infinity'::float - AND - ${ctx.column} != '-infinity'::float - AND - ${ctx.column} != 'NaN'::float -`; +/** If the column type is float, ignore any non numeric result (infinity / NaN) */ +const handleFloatColum = ctx => `${!ctx.isFloatColumn ? `${ctx.column}` : + `nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)` +}`; -const filteredQueryTpl = ctx => ` +/** Count only infinity (positive and negative) appearances */ +const countInfinites = ctx => `${!ctx.isFloatColumn ? `0` : + `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)` +}`; + +/** Count only NaNs appearances*/ +const countNaNs = ctx => `${!ctx.isFloatColumn ? `0` : + `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` +}`; + +/** Query to get min and max values from the query */ +const irqQueryTpl = ctx => ` __cdb_filtered_source AS ( SELECT * FROM (${ctx.query}) __cdb_filtered_source_query - WHERE ${ctx.column} IS NOT NULL - ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} - ) -`; - -const basicsQueryTpl = ctx => ` + WHERE ${handleFloatColum(ctx)} IS NOT NULL + ), __cdb_basics AS ( SELECT - max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val, - avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + max(${ctx.column}) AS __cdb_max_val, + min(${ctx.column}) AS __cdb_min_val, + count(1) AS __cdb_total_rows FROM __cdb_filtered_source ) `; -const overrideBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val, - avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows - FROM __cdb_filtered_source - ) -`; - -const iqrQueryTpl = ctx => ` +/* Query to calculate the number of bins (needs irqQueryTpl before it*/ +const binsQueryTpl = ctx => ` __cdb_iqrange AS ( SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr FROM ( @@ -49,10 +47,7 @@ const iqrQueryTpl = ctx => ` WHERE quartile = 1 or quartile = 3 GROUP BY quartile ) __cdb_iqr - ) -`; - -const binsQueryTpl = ctx => ` + ), __cdb_bins AS ( SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 @@ -70,83 +65,6 @@ const binsQueryTpl = ctx => ` ) `; -const overrideBinsQueryTpl = ctx => ` - __cdb_bins AS ( - SELECT ${ctx.override.bins} AS __cdb_bins_number - ) -`; - -const nullsQueryTpl = ctx => ` - __cdb_nulls AS ( - SELECT - count(*) AS __cdb_nulls_count - FROM (${ctx.query}) __cdb_histogram_nulls - WHERE ${ctx.column} IS NULL - ) -`; - -const infinitiesQueryTpl = ctx => ` - __cdb_infinities AS ( - SELECT - count(*) AS __cdb_infinities_count - FROM (${ctx.query}) __cdb_infinities_query - WHERE - ${ctx.column} = 'infinity'::float - OR - ${ctx.column} = '-infinity'::float - ) -`; - -const nansQueryTpl = ctx => ` - __cdb_nans AS ( - SELECT - count(*) AS __cdb_nans_count - FROM (${ctx.query}) __cdb_nans_query - WHERE ${ctx.column} = 'NaN'::float - ) -`; - -const specialNumericValuesColumnDefinitionTpl = () => ` - __cdb_infinities_count AS infinities_count, - __cdb_nans_count AS nans_count -`; - -const specialNumericValuesCTETpl = () => ` - __cdb_infinities, __cdb_nans -`; - -const specialNumericValuesColumnTpl = () => ` - infinities_count, nans_count -`; - -const histogramQueryTpl = ctx => ` - SELECT - (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, - __cdb_bins_number AS bins_number, - __cdb_nulls_count AS nulls_count, - ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} - __cdb_avg_val AS avg_val, - CASE WHEN __cdb_min_val = __cdb_max_val - THEN 0 - ELSE GREATEST( - 1, - LEAST( - WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), - __cdb_bins_number - ) - ) - 1 - END AS bin, - min(${ctx.column})::numeric AS min, - max(${ctx.column})::numeric AS max, - avg(${ctx.column})::numeric AS avg, - count(*) AS freq - FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins - ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} - GROUP BY bin, bins_number, bin_width, nulls_count, avg_val - ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} - ORDER BY bin -`; - const BIN_MIN_NUMBER = 6; const BIN_MAX_NUMBER = 48; @@ -167,14 +85,14 @@ module.exports = class NumericHistogram extends BaseHistogram { _buildQuery (psql, override, callback) { const histogramSql = this._buildQueryTpl({ - override: override, column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, isFloatColumn: this._columnType === 'float', query: this.query, start: this._getBinStart(override), end: this._getBinEnd(override), + bins: this._getBinsCount(override), minBins: BIN_MIN_NUMBER, - maxBins: BIN_MAX_NUMBER, + maxBins: BIN_MAX_NUMBER }); debug(histogramSql); @@ -182,19 +100,61 @@ module.exports = class NumericHistogram extends BaseHistogram { return callback(null, histogramSql); } + +/** + * ctx: Object with the following values + * ctx.column -- Column for the histogram + * ctx.isFloatColumn - Whether the column is float or not + * ctx.query -- Subquery to extract data + * ctx.start -- Start value for the bins. [>= end to force calculation] + * ctx.end -- End value for the bins. + * ctx.bins -- Numbers of bins to generate [<0 to force calculation] + * ctx.minBins - If !full min bins to calculate [Optional] + * ctx.maxBins - If !full max bins to calculate [Optional] + */ _buildQueryTpl (ctx) { + var extra_tables = ``; + var extra_queries = ``; + var extra_groupby = ``; + + if (ctx.start >= ctx.end) { + ctx.end = `__cdb_basics.__cdb_max_val`; + ctx.start = `__cdb_basics.__cdb_min_val`; + extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`; + extra_tables = `, __cdb_basics`; + extra_queries = `WITH ${irqQueryTpl(ctx)}`; + } + + if (ctx.bins <= 0) { + ctx.bins = `__cdb_bins.__cdb_bins_number`; + extra_groupby += `, __cdb_bins.__cdb_bins_number`; + extra_tables += `, __cdb_bins`; + extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`; + } + return ` - WITH - ${filteredQueryTpl(ctx)}, - ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._hasOverridenBins(ctx.override) ? - overrideBinsQueryTpl(ctx) : - `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` - }, - ${nullsQueryTpl(ctx)} - ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} - ${histogramQueryTpl(ctx)} - `; +${extra_queries} +SELECT + (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width, + ${ctx.bins} as bins_number, + sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END) AS nulls_count, + ${countInfinites(ctx)} AS infinities_count, + ${countNaNs(ctx)} AS nans_count, + min(${handleFloatColum(ctx)}) AS min, + max(${handleFloatColum(ctx)}) AS max, + avg(${handleFloatColum(ctx)}) AS avg, + sum(CASE WHEN (${handleFloatColum(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq, + CASE WHEN ${ctx.start} = ${ctx.end} + THEN 0 + ELSE GREATEST(1, LEAST(${ctx.bins}, + WIDTH_BUCKET(${handleFloatColum(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1 + END AS bin +FROM +( + ${ctx.query} +) __cdb_filtered_source_query${extra_tables} +GROUP BY bin${extra_groupby} +ORDER BY bin;`; } _hasOverridenBins (override) { @@ -204,14 +164,31 @@ module.exports = class NumericHistogram extends BaseHistogram { _getSummary (result, override) { const firstRow = result.rows[0] || {}; + var total_nulls = 0; + var total_infinities = 0; + var total_nans = 0; + var total_avg = 0; + var total_count = 0; + + result.rows.forEach(function(row) { + total_nulls += row.nulls_count; + total_infinities += row.infinities_count; + total_nans += row.nans_count; + total_avg += row.avg * row.freq; + total_count += row.freq; + }); + if (total_count !== 0) { + total_avg /= total_count; + } + return { bin_width: firstRow.bin_width, bins_count: firstRow.bins_number, bins_start: this._populateBinStart(firstRow, override), - nulls: firstRow.nulls_count, - infinities: firstRow.infinities_count, - nans: firstRow.nans_count, - avg: firstRow.avg_val, + nulls: total_nulls, + infinities: total_infinities, + nans: total_nans, + avg: total_avg }; }