Add safety limit to sample metadata

The sampling probability is now being computed using an estimate of the table row count
This could led to too high probabilities (to large samples) if the estimate is not accurate.
To avoid potential problems with large samples we've added a LIMIT to the sampling queries.
This commit is contained in:
Javier Goizueta
2018-05-21 12:45:16 +02:00
parent fecd63e582
commit 11cdcc65ad
2 changed files with 9 additions and 5 deletions
@@ -172,13 +172,14 @@ function mergeColumns(results) {
}
}
function _sample(ctx, numRows) {
if (ctx.metaOptions.sample) {
const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1);
// We'll use a safety limit just in case numRows is a bad estimate
const limit = Math.ceil(ctx.metaOptions.sample * 1.5);
return queryPromise(
ctx.dbConnection,
_getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb)),
_getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)),
res => ({ sample: res.rows })
);
}
+6 -3
View File
@@ -88,11 +88,12 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull
`;
};
module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) {
module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) {
const singleTable = simpleQueryTable(query);
if (singleTable) {
return getTableSample(singleTable.table, singleTable.columns, sampleProb, randomSeed);
}
const limitClause = limit ? `LIMIT ${limit}` : '';
return `
WITH __cdb_rndseed AS (
SELECT setseed(${randomSeed})
@@ -100,14 +101,16 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) {
SELECT *
FROM (${query}) AS __cdb_query
WHERE random() < ${sampleProb}
${limitClause}
`;
};
function getTableSample(table, columns, sampleProb, randomSeed) {
function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) {
const limitClause = limit ? `LIMIT ${limit}` : '';
sampleProb *= 100;
randomSeed *= Math.pow(2, 31) -1;
return `
SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed})
SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause}
`;
}