cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Send dates as unix epoch instead strings in .mvt files

Browse Source 
This commit creates a new ConfigAdapter used in vector maps instantiations.
This adapter generates a new sql query for ONE SINGLE LAYER (carto-vl currently only supports one layer per mvt)
where the date columns are wrapped into a epoch using the `date_part` function.

Due this mvt files are smaller since we use numbers instead strings to represent dates, this is also faster in carto-gl
where we interpolate linearly between 0 and 1 to create animations.

Notice we should add a parameter to make this transformation optional.
We also should take into account the epoch precission.
This commit is contained in:
IagoLast
2018-05-31 17:49:40 +02:00
parent f732ed970b
commit 3850bbb68e
5 changed files with 128 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/utils/query-utils.js
+26 -24
View File
@@ -1,16 +1,16 @@
function prepareQuery(sql) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
@@ -59,25 +59,25 @@ module.exports.handleFloatColumn = function handleFloatColumn(ctx) {
};
/** Count NULL appearances */
module.exports.countNULLs= function countNULLs(ctx) {
module.exports.countNULLs = function countNULLs(ctx) {
return `sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END)`;
};
/** Count only infinity (positive and negative) appearances */
module.exports.countInfinites = function countInfinites(ctx) {
return `${!ctx.isFloatColumn ? `0` :
return `${!ctx.isFloatColumn ? '0' :
`sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)`
}`;
};
/** Count only NaNs appearances*/
module.exports.countNaNs = function countNaNs(ctx) {
return `${!ctx.isFloatColumn ? `0` :
return `${!ctx.isFloatColumn ? '0' :
`sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
}`;
};
module.exports.getQueryTopCategories = function(query, column, topN, includeNulls=false) {
module.exports.getQueryTopCategories = function (query, column, topN, includeNulls = false) {
const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`;
return `
SELECT ${column} AS category, COUNT(*) AS frequency
@@ -101,7 +101,7 @@ function columnSelector(columns) {
throw new TypeError(`Bad argument type for columns: ${typeof columns}`);
}
module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5, columns = null) {
module.exports.getQuerySample = function (query, sampleProb, limit = null, randomSeed = 0.5, columns = null) {
const singleTable = simpleQueryTable(query);
if (singleTable) {
return getTableSample(singleTable.table, columns || singleTable.columns, sampleProb, limit, randomSeed);
@@ -121,7 +121,7 @@ module.exports.getQuerySample = function(query, sampleProb, limit = null, random
function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) {
const limitClause = limit ? `LIMIT ${limit}` : '';
sampleProb *= 100;
randomSeed *= Math.pow(2, 31) -1;
randomSeed *= Math.pow(2, 31) - 1;
return `
SELECT ${columnSelector(columns)}
FROM ${table}
@@ -132,12 +132,12 @@ function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0
function simpleQueryTable(sql) {
const basicQuery =
/\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i');
const unwrappedQuery = new RegExp('^' + basicQuery.source + '$', 'i');
// queries for named maps are wrapped like this:
var wrappedQuery = new RegExp(
"^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" +
'^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(' +
basicQuery.source +
"\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$",
'\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$',
'i'
);
let match = sql.match(unwrappedQuery);
@@ -147,13 +147,13 @@ function simpleQueryTable(sql) {
if (match) {
const columns = match[1];
const schema = match[3];
const table = match[4];
const table = match[4];
return { table: schema ? `${schema}.${table}` : table, columns };
}
return false;
}
module.exports.getQueryGeometryType = function(query, geometryColumn) {
module.exports.getQueryGeometryType = function (query, geometryColumn) {
return `
SELECT ST_GeometryType(${geometryColumn}) AS geom_type
FROM (${query}) AS __cdb_query
@@ -162,10 +162,12 @@ module.exports.getQueryGeometryType = function(query, geometryColumn) {
`;
};
module.exports.getQueryLimited = function(query, limit=0) {
function getQueryLimited(query, limit = 0) {
return `
SELECT *
FROM (${query}) AS __cdb_query
LIMIT ${limit}
`;
};
}
module.exports.getQueryLimited = getQueryLimited;