Add safety limit to sample metadata
The sampling probability is now being computed using an estimate of the table row count This could led to too high probabilities (to large samples) if the estimate is not accurate. To avoid potential problems with large samples we've added a LIMIT to the sampling queries.
This commit is contained in:
@@ -172,13 +172,14 @@ function mergeColumns(results) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function _sample(ctx, numRows) {
|
||||
if (ctx.metaOptions.sample) {
|
||||
const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1);
|
||||
// We'll use a safety limit just in case numRows is a bad estimate
|
||||
const limit = Math.ceil(ctx.metaOptions.sample * 1.5);
|
||||
return queryPromise(
|
||||
ctx.dbConnection,
|
||||
_getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb)),
|
||||
_getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)),
|
||||
res => ({ sample: res.rows })
|
||||
);
|
||||
}
|
||||
|
||||
@@ -88,11 +88,12 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull
|
||||
`;
|
||||
};
|
||||
|
||||
module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) {
|
||||
module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) {
|
||||
const singleTable = simpleQueryTable(query);
|
||||
if (singleTable) {
|
||||
return getTableSample(singleTable.table, singleTable.columns, sampleProb, randomSeed);
|
||||
}
|
||||
const limitClause = limit ? `LIMIT ${limit}` : '';
|
||||
return `
|
||||
WITH __cdb_rndseed AS (
|
||||
SELECT setseed(${randomSeed})
|
||||
@@ -100,14 +101,16 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) {
|
||||
SELECT *
|
||||
FROM (${query}) AS __cdb_query
|
||||
WHERE random() < ${sampleProb}
|
||||
${limitClause}
|
||||
`;
|
||||
};
|
||||
|
||||
function getTableSample(table, columns, sampleProb, randomSeed) {
|
||||
function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) {
|
||||
const limitClause = limit ? `LIMIT ${limit}` : '';
|
||||
sampleProb *= 100;
|
||||
randomSeed *= Math.pow(2, 31) -1;
|
||||
return `
|
||||
SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed})
|
||||
SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause}
|
||||
`;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user