cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize histogram queries

Browse Source
This commit is contained in:
Raul Marin
2017-12-11 11:27:13 +01:00
committed by Raúl Marín
parent 16a36a9d7a
commit 90c9ad18e0
1 changed files with 101 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/models/dataview/histograms/numeric-histogram.js
+101 -124
View File
@@ -1,44 +1,42 @@
const BaseHistogram = require('./base-histogram');
const debug = require('debug')('windshaft:dataview:numeric-histogram');
/** Cast the column column to epoch */
const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`;
const filterOutSpecialNumericValues = ctx => `
${ctx.column} != 'infinity'::float
AND
${ctx.column} != '-infinity'::float
AND
${ctx.column} != 'NaN'::float
`;
/** If the column type is float, ignore any non numeric result (infinity / NaN) */
const handleFloatColum = ctx => `${!ctx.isFloatColumn ? `${ctx.column}` :
`nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)`
}`;
const filteredQueryTpl = ctx => `
/** Count only infinity (positive and negative) appearances */
const countInfinites = ctx => `${!ctx.isFloatColumn ? `0` :
`sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)`
}`;
/** Count only NaNs appearances*/
const countNaNs = ctx => `${!ctx.isFloatColumn ? `0` :
`sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
}`;
/** Query to get min and max values from the query */
const irqQueryTpl = ctx => `
__cdb_filtered_source AS (
SELECT *
FROM (${ctx.query}) __cdb_filtered_source_query
WHERE ${ctx.column} IS NOT NULL
${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''}
)
`;
const basicsQueryTpl = ctx => `
WHERE ${handleFloatColum(ctx)} IS NOT NULL
),
__cdb_basics AS (
SELECT
max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val,
avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
max(${ctx.column}) AS __cdb_max_val,
min(${ctx.column}) AS __cdb_min_val,
count(1) AS __cdb_total_rows
FROM __cdb_filtered_source
)
`;
const overrideBasicsQueryTpl = ctx => `
__cdb_basics AS (
SELECT
max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
FROM __cdb_filtered_source
)
`;
const iqrQueryTpl = ctx => `
/* Query to calculate the number of bins (needs irqQueryTpl before it*/
const binsQueryTpl = ctx => `
__cdb_iqrange AS (
SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
FROM (
@@ -49,10 +47,7 @@ const iqrQueryTpl = ctx => `
WHERE quartile = 1 or quartile = 3
GROUP BY quartile
) __cdb_iqr
)
`;
const binsQueryTpl = ctx => `
),
__cdb_bins AS (
SELECT
CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
@@ -70,83 +65,6 @@ const binsQueryTpl = ctx => `
)
`;
const overrideBinsQueryTpl = ctx => `
__cdb_bins AS (
SELECT ${ctx.override.bins} AS __cdb_bins_number
)
`;
const nullsQueryTpl = ctx => `
__cdb_nulls AS (
SELECT
count(*) AS __cdb_nulls_count
FROM (${ctx.query}) __cdb_histogram_nulls
WHERE ${ctx.column} IS NULL
)
`;
const infinitiesQueryTpl = ctx => `
__cdb_infinities AS (
SELECT
count(*) AS __cdb_infinities_count
FROM (${ctx.query}) __cdb_infinities_query
WHERE
${ctx.column} = 'infinity'::float
OR
${ctx.column} = '-infinity'::float
)
`;
const nansQueryTpl = ctx => `
__cdb_nans AS (
SELECT
count(*) AS __cdb_nans_count
FROM (${ctx.query}) __cdb_nans_query
WHERE ${ctx.column} = 'NaN'::float
)
`;
const specialNumericValuesColumnDefinitionTpl = () => `
__cdb_infinities_count AS infinities_count,
__cdb_nans_count AS nans_count
`;
const specialNumericValuesCTETpl = () => `
__cdb_infinities, __cdb_nans
`;
const specialNumericValuesColumnTpl = () => `
infinities_count, nans_count
`;
const histogramQueryTpl = ctx => `
SELECT
(__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
__cdb_bins_number AS bins_number,
__cdb_nulls_count AS nulls_count,
${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
__cdb_avg_val AS avg_val,
CASE WHEN __cdb_min_val = __cdb_max_val
THEN 0
ELSE GREATEST(
1,
LEAST(
WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
__cdb_bins_number
)
) - 1
END AS bin,
min(${ctx.column})::numeric AS min,
max(${ctx.column})::numeric AS max,
avg(${ctx.column})::numeric AS avg,
count(*) AS freq
FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
ORDER BY bin
`;
const BIN_MIN_NUMBER = 6;
const BIN_MAX_NUMBER = 48;
@@ -167,14 +85,14 @@ module.exports = class NumericHistogram extends BaseHistogram {
_buildQuery (psql, override, callback) {
const histogramSql = this._buildQueryTpl({
override: override,
column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
isFloatColumn: this._columnType === 'float',
query: this.query,
start: this._getBinStart(override),
end: this._getBinEnd(override),
bins: this._getBinsCount(override),
minBins: BIN_MIN_NUMBER,
maxBins: BIN_MAX_NUMBER,
maxBins: BIN_MAX_NUMBER
});
debug(histogramSql);
@@ -182,19 +100,61 @@ module.exports = class NumericHistogram extends BaseHistogram {
return callback(null, histogramSql);
}
/**
* ctx: Object with the following values
* ctx.column -- Column for the histogram
* ctx.isFloatColumn - Whether the column is float or not
* ctx.query -- Subquery to extract data
* ctx.start -- Start value for the bins. [>= end to force calculation]
* ctx.end -- End value for the bins.
* ctx.bins -- Numbers of bins to generate [<0 to force calculation]
* ctx.minBins - If !full min bins to calculate [Optional]
* ctx.maxBins - If !full max bins to calculate [Optional]
*/
_buildQueryTpl (ctx) {
var extra_tables = ``;
var extra_queries = ``;
var extra_groupby = ``;
if (ctx.start >= ctx.end) {
ctx.end = `__cdb_basics.__cdb_max_val`;
ctx.start = `__cdb_basics.__cdb_min_val`;
extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`;
extra_tables = `, __cdb_basics`;
extra_queries = `WITH ${irqQueryTpl(ctx)}`;
}
if (ctx.bins <= 0) {
ctx.bins = `__cdb_bins.__cdb_bins_number`;
extra_groupby += `, __cdb_bins.__cdb_bins_number`;
extra_tables += `, __cdb_bins`;
extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`;
}
return `
WITH
${filteredQueryTpl(ctx)},
${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)},
${this._hasOverridenBins(ctx.override) ?
overrideBinsQueryTpl(ctx) :
`${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`
},
${nullsQueryTpl(ctx)}
${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''}
${histogramQueryTpl(ctx)}
`;
${extra_queries}
SELECT
(${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width,
${ctx.bins} as bins_number,
sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END) AS nulls_count,
${countInfinites(ctx)} AS infinities_count,
${countNaNs(ctx)} AS nans_count,
min(${handleFloatColum(ctx)}) AS min,
max(${handleFloatColum(ctx)}) AS max,
avg(${handleFloatColum(ctx)}) AS avg,
sum(CASE WHEN (${handleFloatColum(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq,
CASE WHEN ${ctx.start} = ${ctx.end}
THEN 0
ELSE GREATEST(1, LEAST(${ctx.bins},
WIDTH_BUCKET(${handleFloatColum(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1
END AS bin
FROM
(
${ctx.query}
) __cdb_filtered_source_query${extra_tables}
GROUP BY bin${extra_groupby}
ORDER BY bin;`;
}
_hasOverridenBins (override) {
@@ -204,14 +164,31 @@ module.exports = class NumericHistogram extends BaseHistogram {
_getSummary (result, override) {
const firstRow = result.rows[0] || {};
var total_nulls = 0;
var total_infinities = 0;
var total_nans = 0;
var total_avg = 0;
var total_count = 0;
result.rows.forEach(function(row) {
total_nulls += row.nulls_count;
total_infinities += row.infinities_count;
total_nans += row.nans_count;
total_avg += row.avg * row.freq;
total_count += row.freq;
});
if (total_count !== 0) {
total_avg /= total_count;
}
return {
bin_width: firstRow.bin_width,
bins_count: firstRow.bins_number,
bins_start: this._populateBinStart(firstRow, override),
nulls: firstRow.nulls_count,
infinities: firstRow.infinities_count,
nans: firstRow.nans_count,
avg: firstRow.avg_val,
nulls: total_nulls,
infinities: total_infinities,
nans: total_nans,
avg: total_avg
};
}