Add parameters to select metadata sample columns

This commit is contained in:
Javier Goizueta
2018-05-28 16:08:31 +02:00
parent f7c80a0101
commit 4bb8914d9a
2 changed files with 46 additions and 9 deletions

View File

@@ -146,13 +146,36 @@ function mergeColumns(results) {
}
}
function _sample(ctx, numRows) {
const SAMPLE_SEED = 0.5;
const DEFAULT_SAMPLE_ROWS = 100;
function exclude(items, excludedItems) {
if (excludedItems) {
return items.filter(item => !excludedItems.includes(item));
}
return items;
}
function _sample(ctx, numRows, availableColumns = null) {
if (ctx.metaOptions.sample) {
const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1);
const sampleProb = Math.min(ctx.metaOptions.sample.num_rows / numRows, 1);
// We'll use a safety limit just in case numRows is a bad estimate
const limit = Math.ceil(ctx.metaOptions.sample * 1.5);
return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)))
.then(res => ({ sample: res.rows }));
const requestedRows = ctx.metaOptions.sample.num_rows || DEFAULT_SAMPLE_ROWS;
const limit = Math.ceil(requestedRows * 1.5);
let columns = ctx.metaOptions.sample.include_columns;
if (columns) {
columns = exclude(columns, ctx.metaOptions.sample.exclude_columns);
}
else if (ctx.metaOptions.sample.exclude_columns) {
if (availableColumns === null) {
return Promise.reject(new Error('Column stats are needed to use the sample exclude columns options'));
}
columns = exclude(availableColumns, ctx.metaOptions.sample.exclude_columns);
}
return queryPromise(ctx.dbConnection, _getSQL(
ctx,
sql => queryUtils.getQuerySample(sql, sampleProb, limit, SAMPLE_SEED, columns)
)).then(res => ({ sample: res.rows }));
}
return Promise.resolve();
}
@@ -265,6 +288,8 @@ function (layer, dbConnection, callback) {
// (if metaOptions.geometryType) from it.
// TODO: compute _sample with _featureCount when available
// TODO: add support for sample.exclude option by, in that case, forcing the columns query and
// passing the results to the sample query function.
Promise.all([
_estimatedFeatureCount(ctx).then(

View File

@@ -88,17 +88,27 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull
`;
};
module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) {
function columnSelector(columns) {
if (!columns) {
return '*';
}
if (typeof(columns) === 'string') {
return columns;
}
return columns.map(name => `"${name}"`).join(', ');
}
module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5, columns = null) {
const singleTable = simpleQueryTable(query);
if (singleTable) {
return getTableSample(singleTable.table, singleTable.columns, sampleProb, limit, randomSeed);
return getTableSample(singleTable.table, columns || singleTable.columns, sampleProb, limit, randomSeed);
}
const limitClause = limit ? `LIMIT ${limit}` : '';
return `
WITH __cdb_rndseed AS (
SELECT setseed(${randomSeed})
)
SELECT *
SELECT ${columnSelector(columns)}
FROM (${query}) AS __cdb_query
WHERE random() < ${sampleProb}
${limitClause}
@@ -110,7 +120,9 @@ function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0
sampleProb *= 100;
randomSeed *= Math.pow(2, 31) -1;
return `
SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause}
SELECT ${columnSelector(columns)}
FROM ${table}
TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause}
`;
}