diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index fd88ac68..ef11fc34 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -8,7 +8,7 @@ var step = require('step'); var Timer = require('../stats/timer'); var BBoxFilter = require('../models/filter/bbox'); -var Histogram = require('../models/dataview/histogram'); +var DataviewFactory = require('../models/dataview/factory'); function DataviewBackend() { } @@ -122,8 +122,8 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); - var histogramDataview = new Histogram(query, dataviewDefinition.options); - histogramDataview.getResult(pg, overrideParams, this); + var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + dataview.getResult(pg, overrideParams, this); }, function returnCallback(err, result) { return callback(err, result, timer.getTimes()); diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js new file mode 100644 index 00000000..f7a49210 --- /dev/null +++ b/lib/cartodb/models/dataview/aggregation.js @@ -0,0 +1,278 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:widget:aggregation'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var summaryQueryTpl = dot.template([ + 'summary AS (', + ' SELECT', + ' count(1) AS count,', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_aggregation_nulls', + ')' +].join('\n')); + +var rankedCategoriesQueryTpl = dot.template([ + 'categories AS(', + ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', + ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', + ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' GROUP BY {{=it._column}}', + ' ORDER BY 2 DESC', + ')' +].join('\n')); + +var categoriesSummaryQueryTpl = dot.template([ + 'categories_summary AS(', + ' SELECT count(1) categories_count, max(value) max_val, min(value) min_val', + ' FROM categories', + ')' +].join('\n')); + +var rankedAggregationQueryTpl = dot.template([ + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank < {{=it._limit}}', + 'UNION ALL', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank >= {{=it._limit}}', + 'GROUP BY nulls_count, min_val, max_val, count, categories_count' +].join('\n')); + +var aggregationQueryTpl = dot.template([ + 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', + ' nulls_count, min_val, max_val, count, categories_count', + 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'ORDER BY value DESC' +].join('\n')); + +var CATEGORIES_LIMIT = 6; + +var VALID_OPERATIONS = { + count: [], + sum: ['aggregationColumn'] +}; + +var TYPE = 'aggregation'; + +/** + { + type: 'aggregation', + options: { + column: 'name', + aggregation: 'count' // it could be, e.g., sum if column is numeric + } + } + */ +function Aggregation(query, options) { + if (!_.isString(options.column)) { + throw new Error('Aggregation expects `column` in widget options'); + } + + if (!_.isString(options.aggregation)) { + throw new Error('Aggregation expects `aggregation` operation in widget options'); + } + + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error("Aggregation does not support '" + options.aggregation + "' operation"); + } + + var requiredOptions = VALID_OPERATIONS[options.aggregation]; + var missingOptions = _.difference(requiredOptions, Object.keys(options)); + if (missingOptions.length > 0) { + throw new Error( + "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') + ); + } + + BaseWidget.apply(this); + + this.query = query; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; +} + +Aggregation.prototype = new BaseWidget(); +Aggregation.prototype.constructor = Aggregation; + +module.exports = Aggregation; + +Aggregation.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.query; + + var aggregationSql; + if (!!override.ownFilter) { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + aggregationQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql(), + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } else { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + rankedAggregationQueryTpl({ + _query: _query, + _column: this.column, + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } + + debug(aggregationSql); + + return callback(null, aggregationSql); +}; + +var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); +Aggregation.prototype.getAggregationSql = function() { + return aggregationFnQueryTpl({ + _aggregationFn: this.aggregation, + _aggregationColumn: this.aggregationColumn || 1 + }); +}; + +Aggregation.prototype.format = function(result) { + var categories = []; + var count = 0; + var nulls = 0; + var minValue = 0; + var maxValue = 0; + var categoriesCount = 0; + + + if (result.rows.length) { + var firstRow = result.rows[0]; + count = firstRow.count; + nulls = firstRow.nulls_count; + minValue = firstRow.min_val; + maxValue = firstRow.max_val; + categoriesCount = firstRow.categories_count; + + result.rows.forEach(function(row) { + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + }); + } + + return { + count: count, + nulls: nulls, + min: minValue, + max: maxValue, + categoriesCount: categoriesCount, + categories: categories + }; +}; + +var filterCategoriesQueryTpl = dot.template([ + 'SELECT {{=it._column}} AS category, {{=it._value}} AS value', + 'FROM ({{=it._query}}) _cdb_aggregation_search', + 'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}', + 'GROUP BY {{=it._column}}' +].join('\n')); + +var searchQueryTpl = dot.template([ + 'WITH', + 'search_unfiltered AS (', + ' {{=it._searchUnfiltered}}', + '),', + 'search_filtered AS (', + ' {{=it._searchFiltered}}', + '),', + 'search_union AS (', + ' SELECT * FROM search_unfiltered', + ' UNION ALL', + ' SELECT * FROM search_filtered', + ')', + 'SELECT category, sum(value) AS value', + 'FROM search_union', + 'GROUP BY category', + 'ORDER BY value desc' +].join('\n')); + + +Aggregation.prototype.search = function(psql, userQuery, callback) { + var self = this; + + var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); + + // TODO unfiltered will be wrong as filters are already applied at this point + var query = searchQueryTpl({ + _searchUnfiltered: filterCategoriesQueryTpl({ + _query: this.query, + _column: this.column, + _value: '0', + _userQuery: _userQuery + }), + _searchFiltered: filterCategoriesQueryTpl({ + _query: this.query, + _column: this.column, + _value: 'count(1)', + _userQuery: _userQuery + }) + }); + + psql.query(query, function(err, result) { + if (err) { + return callback(err, result); + } + + return callback(null, {type: self.getType(), categories: result.rows }); + }, true); // use read-only transaction +}; + +Aggregation.prototype.getType = function() { + return TYPE; +}; + +Aggregation.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _aggregation: this.aggregation + }); +}; diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js new file mode 100644 index 00000000..b2e2f188 --- /dev/null +++ b/lib/cartodb/models/dataview/base.js @@ -0,0 +1,26 @@ +function BaseDataview() {} + +module.exports = BaseDataview; + +BaseDataview.prototype.getResult = function(psql, override, callback) { + var self = this; + this.sql(psql, override, function(err, query) { + psql.query(query, function(err, result) { + + if (err) { + return callback(err, result); + } + + result = self.format(result, override); + result.type = self.getType(); + + return callback(null, result); + + }, true); // use read-only transaction + }); + +}; + +BaseDataview.prototype.search = function(psql, userQuery, callback) { + return callback(null, this.format({ rows: [] })); +}; diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js new file mode 100644 index 00000000..464a0d44 --- /dev/null +++ b/lib/cartodb/models/dataview/factory.js @@ -0,0 +1,18 @@ +var dataviews = require('./'); + +var DataviewFactory = { + dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) { + allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; + return allDataviews; + }, {}), + + getDataview: function(query, dataviewDefinition) { + var type = dataviewDefinition.type; + if (!this.dataviews[type]) { + throw new Error('Invalid dataview type: "' + type + '"'); + } + return new this.dataviews[type](query, dataviewDefinition.options); + } +}; + +module.exports = DataviewFactory; diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js new file mode 100644 index 00000000..c3c176dd --- /dev/null +++ b/lib/cartodb/models/dataview/formula.js @@ -0,0 +1,104 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:widget:formula'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var formulaQueryTpl = dot.template([ + 'SELECT', + '{{=it._operation}}({{=it._column}}) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' +].join('\n')); + +var VALID_OPERATIONS = { + count: true, + avg: true, + sum: true, + min: true, + max: true +}; + +var TYPE = 'formula'; + +/** + { + type: 'formula', + options: { + column: 'name', + operation: 'count' // count, sum, avg + } + } + */ +function Formula(query, options) { + if (!_.isString(options.operation)) { + throw new Error('Formula expects `operation` in widget options'); + } + + if (!VALID_OPERATIONS[options.operation]) { + throw new Error("Formula does not support '" + options.operation + "' operation"); + } + + if (options.operation !== 'count' && !_.isString(options.column)) { + throw new Error('Formula expects `column` in widget options'); + } + + BaseWidget.apply(this); + + this.query = query; + this.column = options.column || '1'; + this.operation = options.operation; +} + +Formula.prototype = new BaseWidget(); +Formula.prototype.constructor = Formula; + +module.exports = Formula; + +Formula.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.query; + var formulaSql = formulaQueryTpl({ + _query: _query, + _operation: this.operation, + _column: this.column + }); + + debug(formulaSql); + + return callback(null, formulaSql); +}; + +Formula.prototype.format = function(result) { + var formattedResult = { + operation: this.operation, + result: 0, + nulls: 0 + }; + + if (result.rows.length) { + formattedResult.operation = this.operation; + formattedResult.result = result.rows[0].result; + formattedResult.nulls = result.rows[0].nulls_count; + } + + return formattedResult; +}; + +Formula.prototype.getType = function() { + return TYPE; +}; + +Formula.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _operation: this.operation + }); +}; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index e5a74d36..d5d611b4 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,4 +1,5 @@ var _ = require('underscore'); +var BaseWidget = require('./base'); var debug = require('debug')('windshaft:dataview:histogram'); var dot = require('dot'); @@ -120,6 +121,9 @@ function Histogram(query, options) { this._columnType = null; } +Histogram.prototype = new BaseWidget(); +Histogram.prototype.constructor = Histogram; + module.exports = Histogram; var DATE_OIDS = { @@ -128,31 +132,6 @@ var DATE_OIDS = { 1184: true }; - -Histogram.prototype.getResult = function(psql, override, callback) { - var self = this; - this.sql(psql, override, function(err, query) { - psql.query(query, function(err, result) { - - if (err) { - return callback(err, result); - } - - result = self.format(result, override); - result.type = self.getType(); - - return callback(null, result); - - }, true); // use read-only transaction - }); - -}; - -Histogram.prototype.search = function(psql, filters, userQuery, callback) { - return callback(null, this.format({ rows: [] })); -}; - - Histogram.prototype.sql = function(psql, override, callback) { if (!callback) { callback = override; diff --git a/lib/cartodb/models/dataview/index.js b/lib/cartodb/models/dataview/index.js new file mode 100644 index 00000000..d508f1bb --- /dev/null +++ b/lib/cartodb/models/dataview/index.js @@ -0,0 +1,6 @@ +module.exports = { + Aggregation: require('./aggregation'), + Formula: require('./formula'), + Histogram: require('./histogram'), + List: require('./list') +}; diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js new file mode 100644 index 00000000..4127103a --- /dev/null +++ b/lib/cartodb/models/dataview/list.js @@ -0,0 +1,66 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + +var BaseWidget = require('./base'); + +var TYPE = 'list'; + +var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); + +/** +{ + type: 'list', + options: { + columns: ['name', 'description'] + } +} +*/ + +function List(query, options) { + options = options || {}; + + if (!Array.isArray(options.columns)) { + throw new Error('List expects `columns` array in widget options'); + } + + BaseWidget.apply(this); + + this.query = query; + this.columns = options.columns; +} + +List.prototype = new BaseWidget(); +List.prototype.constructor = List; + +module.exports = List; + +List.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + } + + var listSql = listSqlTpl({ + _query: this.query, + _columns: this.columns.join(', ') + }); + + return callback(null, listSql); +}; + +List.prototype.format = function(result) { + return { + rows: result.rows + }; +}; + +List.prototype.getType = function() { + return TYPE; +}; + +List.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _columns: this.columns.join(', ') + }); +};