cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Build dataviews with factory to generalise them

Browse Source
This commit is contained in:
Raul Ochoa
2016-03-22 13:10:42 +01:00
parent e44b5eaccd
commit 2bd3e46a4d
8 changed files with 505 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/backends/dataview.js
+3 -3
View File
@@ -8,7 +8,7 @@ var step = require('step');
var Timer = require('../stats/timer');
var BBoxFilter = require('../models/filter/bbox');
var Histogram = require('../models/dataview/histogram');
var DataviewFactory = require('../models/dataview/factory');
function DataviewBackend() {
}
@@ -122,8 +122,8 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
{ownFilter: ownFilter}
);
var histogramDataview = new Histogram(query, dataviewDefinition.options);
histogramDataview.getResult(pg, overrideParams, this);
var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, overrideParams, this);
},
function returnCallback(err, result) {
return callback(err, result, timer.getTimes());
lib/cartodb/models/dataview/aggregation.js
+278
View File
@@ -0,0 +1,278 @@
var _ = require('underscore');
var BaseWidget = require('./base');
var debug = require('debug')('windshaft:widget:aggregation');
var dot = require('dot');
dot.templateSettings.strip = false;
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' count(1) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
')'
].join('\n'));
var categoriesSummaryQueryTpl = dot.template([
'categories_summary AS(',
' SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
' FROM categories',
')'
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
].join('\n'));
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
'ORDER BY value DESC'
].join('\n'));
var CATEGORIES_LIMIT = 6;
var VALID_OPERATIONS = {
count: [],
sum: ['aggregationColumn']
};
var TYPE = 'aggregation';
/**
{
type: 'aggregation',
options: {
column: 'name',
aggregation: 'count' // it could be, e.g., sum if column is numeric
}
}
*/
function Aggregation(query, options) {
if (!_.isString(options.column)) {
throw new Error('Aggregation expects `column` in widget options');
}
if (!_.isString(options.aggregation)) {
throw new Error('Aggregation expects `aggregation` operation in widget options');
}
if (!VALID_OPERATIONS[options.aggregation]) {
throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
}
var requiredOptions = VALID_OPERATIONS[options.aggregation];
var missingOptions = _.difference(requiredOptions, Object.keys(options));
if (missingOptions.length > 0) {
throw new Error(
"Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
);
}
BaseWidget.apply(this);
this.query = query;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
}
Aggregation.prototype = new BaseWidget();
Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, filters, override, callback) {
if (!callback) {
callback = override;
override = {};
}
var _query = this.query;
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
_query: _query,
_column: this.column
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
}),
categoriesSummaryQueryTpl({
_query: _query,
_column: this.column
})
].join(',\n'),
aggregationQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_limit: CATEGORIES_LIMIT
})
].join('\n');
} else {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
_query: _query,
_column: this.column
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
}),
categoriesSummaryQueryTpl({
_query: _query,
_column: this.column
})
].join(',\n'),
rankedAggregationQueryTpl({
_query: _query,
_column: this.column,
_limit: CATEGORIES_LIMIT
})
].join('\n');
}
debug(aggregationSql);
return callback(null, aggregationSql);
};
var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
Aggregation.prototype.getAggregationSql = function() {
return aggregationFnQueryTpl({
_aggregationFn: this.aggregation,
_aggregationColumn: this.aggregationColumn || 1
});
};
Aggregation.prototype.format = function(result) {
var categories = [];
var count = 0;
var nulls = 0;
var minValue = 0;
var maxValue = 0;
var categoriesCount = 0;
if (result.rows.length) {
var firstRow = result.rows[0];
count = firstRow.count;
nulls = firstRow.nulls_count;
minValue = firstRow.min_val;
maxValue = firstRow.max_val;
categoriesCount = firstRow.categories_count;
result.rows.forEach(function(row) {
categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
});
}
return {
count: count,
nulls: nulls,
min: minValue,
max: maxValue,
categoriesCount: categoriesCount,
categories: categories
};
};
var filterCategoriesQueryTpl = dot.template([
'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
'FROM ({{=it._query}}) _cdb_aggregation_search',
'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
'GROUP BY {{=it._column}}'
].join('\n'));
var searchQueryTpl = dot.template([
'WITH',
'search_unfiltered AS (',
' {{=it._searchUnfiltered}}',
'),',
'search_filtered AS (',
' {{=it._searchFiltered}}',
'),',
'search_union AS (',
' SELECT * FROM search_unfiltered',
' UNION ALL',
' SELECT * FROM search_filtered',
')',
'SELECT category, sum(value) AS value',
'FROM search_union',
'GROUP BY category',
'ORDER BY value desc'
].join('\n'));
Aggregation.prototype.search = function(psql, userQuery, callback) {
var self = this;
var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
// TODO unfiltered will be wrong as filters are already applied at this point
var query = searchQueryTpl({
_searchUnfiltered: filterCategoriesQueryTpl({
_query: this.query,
_column: this.column,
_value: '0',
_userQuery: _userQuery
}),
_searchFiltered: filterCategoriesQueryTpl({
_query: this.query,
_column: this.column,
_value: 'count(1)',
_userQuery: _userQuery
})
});
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}
return callback(null, {type: self.getType(), categories: result.rows });
}, true); // use read-only transaction
};
Aggregation.prototype.getType = function() {
return TYPE;
};
Aggregation.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_column: this.column,
_aggregation: this.aggregation
});
};
lib/cartodb/models/dataview/base.js
+26
View File
@@ -0,0 +1,26 @@
function BaseDataview() {}
module.exports = BaseDataview;
BaseDataview.prototype.getResult = function(psql, override, callback) {
var self = this;
this.sql(psql, override, function(err, query) {
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}
result = self.format(result, override);
result.type = self.getType();
return callback(null, result);
}, true); // use read-only transaction
});
};
BaseDataview.prototype.search = function(psql, userQuery, callback) {
return callback(null, this.format({ rows: [] }));
};
lib/cartodb/models/dataview/factory.js
+18
View File
@@ -0,0 +1,18 @@
var dataviews = require('./');
var DataviewFactory = {
dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
return allDataviews;
}, {}),
getDataview: function(query, dataviewDefinition) {
var type = dataviewDefinition.type;
if (!this.dataviews[type]) {
throw new Error('Invalid dataview type: "' + type + '"');
}
return new this.dataviews[type](query, dataviewDefinition.options);
}
};
module.exports = DataviewFactory;
lib/cartodb/models/dataview/formula.js
+104
View File
@@ -0,0 +1,104 @@
var _ = require('underscore');
var BaseWidget = require('./base');
var debug = require('debug')('windshaft:widget:formula');
var dot = require('dot');
dot.templateSettings.strip = false;
var formulaQueryTpl = dot.template([
'SELECT',
'{{=it._operation}}({{=it._column}}) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n'));
var VALID_OPERATIONS = {
count: true,
avg: true,
sum: true,
min: true,
max: true
};
var TYPE = 'formula';
/**
{
type: 'formula',
options: {
column: 'name',
operation: 'count' // count, sum, avg
}
}
*/
function Formula(query, options) {
if (!_.isString(options.operation)) {
throw new Error('Formula expects `operation` in widget options');
}
if (!VALID_OPERATIONS[options.operation]) {
throw new Error("Formula does not support '" + options.operation + "' operation");
}
if (options.operation !== 'count' && !_.isString(options.column)) {
throw new Error('Formula expects `column` in widget options');
}
BaseWidget.apply(this);
this.query = query;
this.column = options.column || '1';
this.operation = options.operation;
}
Formula.prototype = new BaseWidget();
Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, filters, override, callback) {
if (!callback) {
callback = override;
override = {};
}
var _query = this.query;
var formulaSql = formulaQueryTpl({
_query: _query,
_operation: this.operation,
_column: this.column
});
debug(formulaSql);
return callback(null, formulaSql);
};
Formula.prototype.format = function(result) {
var formattedResult = {
operation: this.operation,
result: 0,
nulls: 0
};
if (result.rows.length) {
formattedResult.operation = this.operation;
formattedResult.result = result.rows[0].result;
formattedResult.nulls = result.rows[0].nulls_count;
}
return formattedResult;
};
Formula.prototype.getType = function() {
return TYPE;
};
Formula.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_column: this.column,
_operation: this.operation
});
};
lib/cartodb/models/dataview/histogram.js
+4 -25
View File
@@ -1,4 +1,5 @@
var _ = require('underscore');
var BaseWidget = require('./base');
var debug = require('debug')('windshaft:dataview:histogram');
var dot = require('dot');
@@ -120,6 +121,9 @@ function Histogram(query, options) {
this._columnType = null;
}
Histogram.prototype = new BaseWidget();
Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
var DATE_OIDS = {
@@ -128,31 +132,6 @@ var DATE_OIDS = {
1184: true
};
Histogram.prototype.getResult = function(psql, override, callback) {
var self = this;
this.sql(psql, override, function(err, query) {
psql.query(query, function(err, result) {
if (err) {
return callback(err, result);
}
result = self.format(result, override);
result.type = self.getType();
return callback(null, result);
}, true); // use read-only transaction
});
};
Histogram.prototype.search = function(psql, filters, userQuery, callback) {
return callback(null, this.format({ rows: [] }));
};
Histogram.prototype.sql = function(psql, override, callback) {
if (!callback) {
callback = override;
lib/cartodb/models/dataview/index.js
+6
View File
@@ -0,0 +1,6 @@
module.exports = {
Aggregation: require('./aggregation'),
Formula: require('./formula'),
Histogram: require('./histogram'),
List: require('./list')
};
lib/cartodb/models/dataview/list.js
+66
View File
@@ -0,0 +1,66 @@
var dot = require('dot');
dot.templateSettings.strip = false;
var BaseWidget = require('./base');
var TYPE = 'list';
var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
/**
{
type: 'list',
options: {
columns: ['name', 'description']
}
}
*/
function List(query, options) {
options = options || {};
if (!Array.isArray(options.columns)) {
throw new Error('List expects `columns` array in widget options');
}
BaseWidget.apply(this);
this.query = query;
this.columns = options.columns;
}
List.prototype = new BaseWidget();
List.prototype.constructor = List;
module.exports = List;
List.prototype.sql = function(psql, filters, override, callback) {
if (!callback) {
callback = override;
}
var listSql = listSqlTpl({
_query: this.query,
_columns: this.columns.join(', ')
});
return callback(null, listSql);
};
List.prototype.format = function(result) {
return {
rows: result.rows
};
};
List.prototype.getType = function() {
return TYPE;
};
List.prototype.toString = function() {
return JSON.stringify({
_type: TYPE,
_query: this.query,
_columns: this.columns.join(', ')
});
};