diff --git a/INSTALL.md b/INSTALL.md index 2de2cb09..45654b0b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are - Core - Node.js >=6.9.x - - yarn >=0.21.3 + - yarn >=0.27.5 <1.0.0 - PostgreSQL >8.3.x, PostGIS >1.5.x - Redis >2.4.0 (http://www.redis.io) - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik). diff --git a/NEWS.md b/NEWS.md index a5d3e7d3..7c914557 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,14 @@ # Changelog -## 3.13.0 +## 3.13.1 Released 2017-mm-dd + - Upgrades yarn minimum version requirement to v0.27.5 + +Bugfixes: + - + +## 3.13.0 +Released 2017-10-02 - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. - Upgrades express, debug, request and body-parser for sequrity reasons diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 77f37723..b4c59af2 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,95 +1,178 @@ -var _ = require('underscore'); -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:widget:aggregation'); +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:aggregation'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const filteredQueryTpl = ctx => ` + filtered_source AS ( + SELECT * + FROM (${ctx.query}) _cdb_filtered_source + ${ctx.aggregationColumn && ctx.isFloatColumn ? ` + WHERE + ${ctx.aggregationColumn} != 'infinity'::float + AND + ${ctx.aggregationColumn} != '-infinity'::float + AND + ${ctx.aggregationColumn} != 'NaN'::float` : + '' + } + ) +`; -var filteredQueryTpl = dot.template([ - 'filtered_source AS (', - ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', - ' {{=it._aggregationColumn}} != \'infinity\'::float', - ' AND', - ' {{=it._aggregationColumn}} != \'-infinity\'::float', - ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', - ')' -].join(' \n')); +const summaryQueryTpl = ctx => ` + summary AS ( + SELECT + count(1) AS count, + sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count + ${ctx.isFloatColumn ? `, + sum( + CASE + WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float + THEN 1 + ELSE 0 + END + ) AS infinities_count, + sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` : + '' + } + FROM (${ctx.query}) _cdb_aggregation_nulls + ) +`; -var summaryQueryTpl = dot.template([ - 'summary AS (', - ' SELECT', - ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', - ' {{?it._isFloatColumn}},sum(', - ' CASE', - ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', - ' THEN 1', - ' ELSE 0', - ' END', - ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', - ' FROM ({{=it._query}}) _cdb_aggregation_nulls', - ')' -].join('\n')); +const rankedCategoriesQueryTpl = ctx => ` + categories AS( + SELECT + ${ctx.column} AS category, + ${ctx.aggregationFn} AS value, + row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank + FROM filtered_source + ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} + GROUP BY ${ctx.column} + ORDER BY 2 DESC + ) +`; -var rankedCategoriesQueryTpl = dot.template([ - 'categories AS(', - ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', - ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM filtered_source', - ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', - ' GROUP BY {{=it._column}}', - ' ORDER BY 2 DESC', - ')' -].join('\n')); +const categoriesSummaryMinMaxQueryTpl = () => ` + categories_summary_min_max AS( + SELECT + max(value) max_val, + min(value) min_val + FROM categories + ) +`; -var categoriesSummaryMinMaxQueryTpl = dot.template([ - 'categories_summary_min_max AS(', - ' SELECT max(value) max_val, min(value) min_val', - ' FROM categories', - ')' -].join('\n')); +const categoriesSummaryCountQueryTpl = ctx => ` + categories_summary_count AS( + SELECT count(1) AS categories_count + FROM ( + SELECT ${ctx.column} AS category + FROM (${ctx.query}) _cdb_categories + GROUP BY ${ctx.column} + ) _cdb_categories_count + ) +`; -var categoriesSummaryCountQueryTpl = dot.template([ - 'categories_summary_count AS(', - ' SELECT count(1) AS categories_count', - ' FROM (', - ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', - ' GROUP BY {{=it._column}}', - ' ) _cdb_categories_count', - ')' -].join('\n')); +const specialNumericValuesColumns = () => `, nans_count, infinities_count`; -var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - ' FROM categories, summary, categories_summary_min_max, categories_summary_count', - ' WHERE rank < {{=it._limit}}', - 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', - ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - ' FROM categories, summary, categories_summary_min_max, categories_summary_count', - ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count,', - ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' -].join('\n')); +const rankedAggregationQueryTpl = ctx => ` + SELECT + CAST(category AS text), + value, + false as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM categories, summary, categories_summary_min_max, categories_summary_count + WHERE rank < ${ctx.limit} + UNION ALL + SELECT + 'Other' category, + ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value, + true as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM categories, summary, categories_summary_min_max, categories_summary_count + WHERE rank >= ${ctx.limit} + GROUP BY + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } +`; -var aggregationQueryTpl = dot.template([ - 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count,', - ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - 'ORDER BY value DESC' -].join('\n')); +const aggregationQueryTpl = ctx => ` + SELECT + CAST(${ctx.column} AS text) AS category, + ${ctx.aggregationFn} AS value, + false as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count + GROUP BY + category, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ORDER BY value DESC +`; -var CATEGORIES_LIMIT = 6; +const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`; -var VALID_OPERATIONS = { +const aggregationDataviewQueryTpl = ctx => ` + WITH + ${filteredQueryTpl(ctx)}, + ${summaryQueryTpl(ctx)}, + ${rankedCategoriesQueryTpl(ctx)}, + ${categoriesSummaryMinMaxQueryTpl(ctx)}, + ${categoriesSummaryCountQueryTpl(ctx)} + ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`} +`; + +const filterCategoriesQueryTpl = ctx => ` + SELECT + ${ctx.column} AS category, + ${ctx.value} AS value + FROM (${ctx.query}) _cdb_aggregation_search + WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery} + GROUP BY ${ctx.column} +`; + +const searchQueryTpl = ctx => ` + WITH + search_unfiltered AS ( + ${ctx.searchUnfiltered} + ), + search_filtered AS ( + ${ctx.searchFiltered} + ), + search_union AS ( + SELECT * FROM search_unfiltered + UNION ALL + SELECT * FROM search_filtered + ) + SELECT category, sum(value) AS value + FROM search_union + GROUP BY category + ORDER BY value desc +`; + +const CATEGORIES_LIMIT = 6; + +const VALID_OPERATIONS = { count: [], sum: ['aggregationColumn'], avg: ['aggregationColumn'], @@ -97,7 +180,7 @@ var VALID_OPERATIONS = { max: ['aggregationColumn'] }; -var TYPE = 'aggregation'; +const TYPE = 'aggregation'; /** { @@ -108,256 +191,150 @@ var TYPE = 'aggregation'; } } */ -function Aggregation(query, options, queries) { - if (!_.isString(options.column)) { - throw new Error('Aggregation expects `column` in widget options'); +module.exports = class Aggregation extends BaseDataview { + constructor (query, options = {}, queries = {}) { + super(); + + this._checkOptions(options); + + this.query = query; + this.queries = queries; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } - if (!_.isString(options.aggregation)) { - throw new Error('Aggregation expects `aggregation` operation in widget options'); - } - - if (!VALID_OPERATIONS[options.aggregation]) { - throw new Error("Aggregation does not support '" + options.aggregation + "' operation"); - } - - var requiredOptions = VALID_OPERATIONS[options.aggregation]; - var missingOptions = _.difference(requiredOptions, Object.keys(options)); - if (missingOptions.length > 0) { - throw new Error( - "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') - ); - } - - BaseWidget.apply(this); - - this.query = query; - this.queries = queries; - this.column = options.column; - this.aggregation = options.aggregation; - this.aggregationColumn = options.aggregationColumn; - this._isFloatColumn = null; -} - -Aggregation.prototype = new BaseWidget(); -Aggregation.prototype.constructor = Aggregation; - -module.exports = Aggregation; - -Aggregation.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this.aggregationColumn && this._isFloatColumn === null) { - this._isFloatColumn = false; - this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { - if (!err && !!type) { - self._isFloatColumn = type.float; - } - self.sql(psql, override, callback); - }); - return null; - } - - var _query = this.query; - - var aggregationSql; - - if (!!override.ownFilter) { - aggregationSql = [ - this.getCategoriesCTESql( - _query, - this.column, - this.aggregation, - this.aggregationColumn, - this._isFloatColumn - ), - aggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: _query, - _column: this.column, - _aggregation: this.getAggregationSql(), - _limit: CATEGORIES_LIMIT - }) - ].join('\n'); - } else { - aggregationSql = [ - this.getCategoriesCTESql( - _query, - this.column, - this.aggregation, - this.aggregationColumn, - this._isFloatColumn - ), - rankedAggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: _query, - _column: this.column, - _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', - _limit: CATEGORIES_LIMIT - }) - ].join('\n'); - } - - debug(aggregationSql); - - return callback(null, aggregationSql); -}; - -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { - return [ - "WITH", - [ - filteredQueryTpl({ - _isFloatColumn: isFloatColumn, - _query: this.query, - _column: this.column, - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - summaryQueryTpl({ - _isFloatColumn: isFloatColumn, - _query: query, - _column: column, - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - rankedCategoriesQueryTpl({ - _query: query, - _column: column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column - }), - categoriesSummaryCountQueryTpl({ - _query: query, - _column: column - }) - ].join(',\n') - ].join('\n'); -}; - -var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); -Aggregation.prototype.getAggregationSql = function() { - return aggregationFnQueryTpl({ - _aggregationFn: this.aggregation, - _aggregationColumn: this.aggregationColumn || 1 - }); -}; - -Aggregation.prototype.format = function(result) { - var categories = []; - var count = 0; - var nulls = 0; - var nans = 0; - var infinities = 0; - var minValue = 0; - var maxValue = 0; - var categoriesCount = 0; - - - if (result.rows.length) { - var firstRow = result.rows[0]; - count = firstRow.count; - nulls = firstRow.nulls_count; - nans = firstRow.nans_count; - infinities = firstRow.infinities_count; - minValue = firstRow.min_val; - maxValue = firstRow.max_val; - categoriesCount = firstRow.categories_count; - - result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', - 'max_val', 'categories_count', 'nans_count', 'infinities_count')); - }); - } - - return { - aggregation: this.aggregation, - count: count, - nulls: nulls, - nans: nans, - infinities: infinities, - min: minValue, - max: maxValue, - categoriesCount: categoriesCount, - categories: categories - }; -}; - -var filterCategoriesQueryTpl = dot.template([ - 'SELECT {{=it._column}} AS category, {{=it._value}} AS value', - 'FROM ({{=it._query}}) _cdb_aggregation_search', - 'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}', - 'GROUP BY {{=it._column}}' -].join('\n')); - -var searchQueryTpl = dot.template([ - 'WITH', - 'search_unfiltered AS (', - ' {{=it._searchUnfiltered}}', - '),', - 'search_filtered AS (', - ' {{=it._searchFiltered}}', - '),', - 'search_union AS (', - ' SELECT * FROM search_unfiltered', - ' UNION ALL', - ' SELECT * FROM search_filtered', - ')', - 'SELECT category, sum(value) AS value', - 'FROM search_union', - 'GROUP BY category', - 'ORDER BY value desc' -].join('\n')); - - -Aggregation.prototype.search = function(psql, userQuery, callback) { - var self = this; - - var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); - var _value = this.aggregation !== 'count' && this.aggregationColumn ? - this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; - - // TODO unfiltered will be wrong as filters are already applied at this point - var query = searchQueryTpl({ - _searchUnfiltered: filterCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _value: '0', - _userQuery: _userQuery - }), - _searchFiltered: filterCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _value: _value, - _userQuery: _userQuery - }) - }); - - psql.query(query, function(err, result) { - if (err) { - return callback(err, result); + _checkOptions (options) { + if (typeof options.column !== 'string') { + throw new Error(`Aggregation expects 'column' in dataview options`); } - return callback(null, {type: self.getType(), categories: result.rows }); - }, true); // use read-only transaction -}; + if (typeof options.aggregation !== 'string') { + throw new Error(`Aggregation expects 'aggregation' operation in dataview options`); + } -Aggregation.prototype.getType = function() { - return TYPE; -}; + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error(`Aggregation does not support '${options.aggregation}' operation`); + } -Aggregation.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _column: this.column, - _aggregation: this.aggregation - }); + const requiredOptions = VALID_OPERATIONS[options.aggregation]; + const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + + if (missingOptions.length > 0) { + throw new Error( + `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}` + ); + } + } + + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + if (this._shouldCheckColumnType()) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => { + if (!err && !!type) { + this._isFloatColumn = type.float; + } + this.sql(psql, override, callback); + }); + return null; + } + + const aggregationSql = aggregationDataviewQueryTpl({ + override: override, + query: this.query, + column: this.column, + aggregation: this.aggregation, + aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null, + aggregationFn: aggregationFnQueryTpl({ + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn || 1 + }), + isFloatColumn: this._isFloatColumn, + limit: CATEGORIES_LIMIT + }); + + debug(aggregationSql); + + return callback(null, aggregationSql); + } + + _shouldCheckColumnType () { + return this.aggregationColumn && this._isFloatColumn === null; + } + + format (result) { + const { + count = 0, + nulls_count = 0, + nans_count = 0, + infinities_count = 0, + min_val = 0, + max_val = 0, + categories_count = 0 + } = result.rows[0] || {}; + + return { + aggregation: this.aggregation, + count: count, + nulls: nulls_count, + nans: nans_count, + infinities: infinities_count, + min: min_val, + max: max_val, + categoriesCount: categories_count, + categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg })) + }; + } + + search (psql, userQuery, callback) { + const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`); + const value = this.aggregation !== 'count' && this.aggregationColumn ? + `${this.aggregation}(${this.aggregationColumn})` : + 'count(1)'; + + // TODO unfiltered will be wrong as filters are already applied at this point + const query = searchQueryTpl({ + searchUnfiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: '0', + userQuery: escapedUserQuery + }), + searchFiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: value, + userQuery: escapedUserQuery + }) + }); + + debug(query); + + psql.query(query, (err, result) => { + if (err) { + return callback(err, result); + } + + return callback(null, {type: this.getType(), categories: result.rows }); + }, true); // use read-only transaction + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _aggregation: this.aggregation + }); + } }; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6e62ad9d..affbbd25 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,717 +1,72 @@ -var _ = require('underscore'); -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:dataview:histogram'); +const debug = require('debug')('windshaft:dataview:histogram'); +const NumericHistogram = require('./histograms/numeric-histogram'); +const DateHistogram = require('./histograms/date-histogram'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const DATE_HISTOGRAM = 'DateHistogram'; +const NUMERIC_HISTOGRAM = 'NumericHistogram'; -var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +module.exports = class Histogram { + constructor (query, options, queries) { + this.query = query; + this.options = options || {}; + this.queries = queries; -var dateIntervalQueryTpl = dot.template([ - 'WITH', - '__cdb_dates AS (', - ' SELECT', - ' MAX({{=it.column}}::timestamp) AS __cdb_end,', - ' MIN({{=it.column}}::timestamp) AS __cdb_start', - ' FROM ({{=it.query}}) __cdb_source', - '),', - '__cdb_interval_in_days AS (', - ' SELECT' , - ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', - ' FROM __cdb_dates', - '),', - '__cdb_interval_in_hours AS (', - ' SELECT', - ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', - ' FROM __cdb_interval_in_days, __cdb_dates', - '),', - '__cdb_interval_in_minutes AS (', - ' SELECT', - ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', - ' FROM __cdb_interval_in_hours, __cdb_dates', - '),', - '__cdb_interval_in_seconds AS (', - ' SELECT', - ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', - ' FROM __cdb_interval_in_minutes, __cdb_dates', - ')', - 'SELECT', - ' ROUND(__cdb_days / 365) AS year,', - ' ROUND(__cdb_days / 90) AS quarter,', - ' ROUND(__cdb_days / 30) AS month,', - ' ROUND(__cdb_days / 7) AS week,', - ' __cdb_days AS day,', - ' __cdb_hours AS hour,', - ' __cdb_minutes AS minute,', - ' __cdb_seconds AS second', - 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' -].join('\n')); - -var MAX_INTERVAL_VALUE = 366; -var BIN_MIN_NUMBER = 6; -var BIN_MAX_NUMBER = 48; - -var filteredQueryTpl = dot.template([ - '__cdb_filtered_source AS (', - ' SELECT *', - ' FROM ({{=it._query}}) __cdb_filtered_source_query', - ' WHERE', - ' {{=it._column}} IS NOT NULL', - ' {{?it._isFloatColumn}}AND', - ' {{=it._column}} != \'infinity\'::float', - ' AND', - ' {{=it._column}} != \'-infinity\'::float', - ' AND', - ' {{=it._column}} != \'NaN\'::float{{?}}', - ')' -].join(' \n')); - -var basicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', - ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', - ' FROM __cdb_filtered_source', - ')' -].join(' \n')); - -var overrideBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', - ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', - ' FROM __cdb_filtered_source', - ')' -].join('\n')); - -var iqrQueryTpl = dot.template([ - '__cdb_iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', - ' FROM (', - ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', - ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', - ' ) AS quartile', - ' FROM __cdb_filtered_source) _cdb_quartiles', - ' WHERE quartile = 1 or quartile = 3', - ' GROUP BY quartile', - ' ) __cdb_iqr', - ')' -].join('\n')); - -var binsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', - ' THEN 1', - ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', - ' LEAST(', - ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', - ' {{=it._maxBins}}', - ' )', - ' )', - ' END AS __cdb_bins_number', - ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', - ' LIMIT 1', - ')' -].join('\n')); - -var overrideBinsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT {{=it._bins}} AS __cdb_bins_number', - ')' -].join('\n')); - -var nullsQueryTpl = dot.template([ - '__cdb_nulls AS (', - ' SELECT', - ' count(*) AS __cdb_nulls_count', - ' FROM ({{=it._query}}) __cdb_histogram_nulls', - ' WHERE {{=it._column}} IS NULL', - ')' -].join('\n')); - -var infinitiesQueryTpl = dot.template([ - '__cdb_infinities AS (', - ' SELECT', - ' count(*) AS __cdb_infinities_count', - ' FROM ({{=it._query}}) __cdb_infinities_query', - ' WHERE', - ' {{=it._column}} = \'infinity\'::float', - ' OR', - ' {{=it._column}} = \'-infinity\'::float', - ')' -].join('\n')); - -var nansQueryTpl = dot.template([ - '__cdb_nans AS (', - ' SELECT', - ' count(*) AS __cdb_nans_count', - ' FROM ({{=it._query}}) __cdb_nans_query', - ' WHERE {{=it._column}} = \'NaN\'::float', - ')' -].join('\n')); - -var histogramQueryTpl = dot.template([ - 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', - ' __cdb_bins_number AS bins_number,', - ' __cdb_nulls_count AS nulls_count,', - ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', - ' __cdb_nans_count AS nans_count,{{?}}', - ' __cdb_avg_val AS avg_val,', - ' CASE WHEN __cdb_min_val = __cdb_max_val', - ' THEN 0', - ' ELSE GREATEST(', - ' 1,', - ' LEAST(', - ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', - ' __cdb_bins_number', - ' )', - ' ) - 1', - ' END AS bin,', - ' min({{=it._column}})::numeric AS min,', - ' max({{=it._column}})::numeric AS max,', - ' avg({{=it._column}})::numeric AS avg,', - ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', - ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', - 'GROUP BY bin, bins_number, bin_width, nulls_count,', - ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', - 'ORDER BY bin' -].join('\n')); - -var dateBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', - ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', - ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )) AS __cdb_start_date,', - ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', - ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) __cdb_basics_query', - ')' -].join(' \n')); - -var dateOverrideBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._end}})::float AS __cdb_max_val,', - ' min({{=it._start}})::float AS __cdb_min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', - ' min(', - ' date_trunc(', - ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )', - ' ) AS __cdb_start_date,', - ' max(', - ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AS __cdb_end_date,', - ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) __cdb_basics_query', - ')' -].join(' \n')); - -var dateBinsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT', - ' __cdb_bins_array,', - ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', - ' FROM (', - ' SELECT', - ' ARRAY(', - ' SELECT GENERATE_SERIES(', - ' __cdb_start_date::timestamptz,', - ' __cdb_end_date::timestamptz,', - ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', - ' )', - ' ) AS __cdb_bins_array', - ' FROM __cdb_basics', - ' ) __cdb_bins_array_query', - ')' -].join('\n')); - -var dateHistogramQueryTpl = dot.template([ - 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', - ' __cdb_bins_number AS bins_number,', - ' __cdb_nulls_count AS nulls_count,', - ' CASE WHEN __cdb_min_val = __cdb_max_val', - ' THEN 0', - ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', - ' __cdb_bins_array', - ' ),', - ' __cdb_bins_number', - ' )) - 1', - ' END AS bin,', - ' min(', - ' date_part(', - ' \'epoch\', ', - ' date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AT TIME ZONE \'{{=it._offset}}\'', - ' )', - ' )::numeric AS timestamp,', - ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', - ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', - ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', - ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', - ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', - 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', - 'ORDER BY bin' -].join('\n')); - -var TYPE = 'histogram'; - -/** -Numeric histogram: -{ - type: 'histogram', - options: { - column: 'name', // column data type: numeric - bins: 10 // OPTIONAL - } -} - -Time series: -{ - type: 'histogram', - options: { - column: 'date', // column data type: date - aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) - offset: -7200 // OPTIONAL (UTC offset in seconds) - } - } - */ -function Histogram(query, options, queries) { - if (!_.isString(options.column)) { - throw new Error('Histogram expects `column` in widget options'); + this.histogramImplementation = this._getHistogramImplementation(); } - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; - this.aggregation = options.aggregation; - this.offset = options.offset; + _getHistogramImplementation (override) { + let implementation = null; - this._columnType = null; -} - -Histogram.prototype = new BaseWidget(); -Histogram.prototype.constructor = Histogram; - -module.exports = Histogram; - -Histogram.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - // assume numeric, will fail later - self._columnType = 'numeric'; - if (!err && !!type) { - self._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - self.sql(psql, override, callback); - }, true); // use read-only transaction - return null; - } - - this._buildQuery(psql, override, callback); -}; - -Histogram.prototype.isDateHistogram = function (override) { - return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); -}; - -Histogram.prototype._buildQuery = function (psql, override, callback) { - var filteredQuery, basicsQuery, binsQuery; - var _column = this.column; - var _query = this.query; - - if (this.isDateHistogram(override)) { - return this._buildDateHistogramQuery(psql, override, callback); - } - - if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); - } - - filteredQuery = filteredQueryTpl({ - _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column - }); - - if (this._shouldOverride(override)) { - debug('overriding with %j', override); - basicsQuery = overrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _start: getBinStart(override), - _end: getBinEnd(override) - }); - - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - basicsQuery = basicsQueryTpl({ - _query: _query, - _column: _column - }); - - if (this._shouldOverrideBins(override)) { - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - binsQuery = [ - iqrQueryTpl({ - _query: _query, - _column: _column - }), - binsQueryTpl({ - _query: _query, - _minBins: BIN_MIN_NUMBER, - _maxBins: BIN_MAX_NUMBER - }) - ].join(',\n'); - } - } - - var cteSql = [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }) - ]; - - if (this._columnType === 'float') { - cteSql.push( - infinitiesQueryTpl({ - _query: _query, - _column: _column - }), - nansQueryTpl({ - _query: _query, - _column: _column - }) - ); - } - - var histogramSql = [ - "WITH", - cteSql.join(',\n'), - histogramQueryTpl({ - _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column - }) - ].join('\n'); - - debug(histogramSql); - - return callback(null, histogramSql); -}; - -Histogram.prototype._shouldOverride = function (override) { - return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); -}; - -Histogram.prototype._shouldOverrideBins = function (override) { - return override && _.has(override, 'bins'); -}; - -var DATE_AGGREGATIONS = { - 'auto': true, - 'minute': true, - 'hour': true, - 'day': true, - 'week': true, - 'month': true, - 'quarter': true, - 'year': true -}; - -Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { - var _column = this.column; - var _query = this.query; - var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - - if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { - return callback(new Error('Invalid aggregation value. Valid ones: ' + - Object.keys(DATE_AGGREGATIONS).join(', ') - )); - } - - if (_aggregation === 'auto') { - this.getAutomaticAggregation(psql, function (err, aggregation) { - if (err || aggregation === 'none') { - this.aggregation = 'day'; - } else { - this.aggregation = aggregation; - } - override.aggregation = this.aggregation; - this._buildDateHistogramQuery(psql, override, callback); - }.bind(this)); - return null; - } - - var dateBasicsQuery; - - if (override && _.has(override, 'start') && _.has(override, 'end')) { - dateBasicsQuery = dateOverrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _start: getBinStart(override), - _end: getBinEnd(override), - _offset: parseOffset(_offset, _aggregation) - }); - } else { - dateBasicsQuery = dateBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) - }); - } - - var dateBinsQuery = [ - dateBinsQueryTpl({ - _aggregation: _aggregation - }) - ].join(',\n'); - - var nullsQuery = nullsQueryTpl({ - _query: _query, - _column: _column - }); - - var dateHistogramQuery = dateHistogramQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) - }); - - var histogramSql = [ - "WITH", - [ - dateBasicsQuery, - dateBinsQuery, - nullsQuery - ].join(',\n'), - dateHistogramQuery - ].join('\n'); - - debug(histogramSql); - - return callback(null, histogramSql); -}; - -Histogram.prototype.getAutomaticAggregation = function (psql, callback) { - var dateIntervalQuery = dateIntervalQueryTpl({ - query: this.query, - column: this.column - }); - - debug(dateIntervalQuery); - - psql.query(dateIntervalQuery, function (err, result) { - if (err) { - return callback(err); + switch (this._getHistogramSubtype(override)) { + case DATE_HISTOGRAM: + debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override); + implementation = new DateHistogram(this.query, this.options, this.queries); + break; + case NUMERIC_HISTOGRAM: + debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override); + implementation = new NumericHistogram(this.query, this.options, this.queries); + break; + default: + throw new Error('Unsupported Histogram type'); } - var aggegations = result.rows[0]; - var aggregation = Object.keys(aggegations) - .map(function (key) { - return { - name: key, - value: aggegations[key] - }; - }) - .reduce(function (closer, current) { - if (current.value > MAX_INTERVAL_VALUE) { - return closer; - } + return implementation; + } - var closerDiff = MAX_INTERVAL_VALUE - closer.value; - var currentDiff = MAX_INTERVAL_VALUE - current.value; - - if (Number.isFinite(current.value) && closerDiff > currentDiff) { - return current; - } - - return closer; - }, { name: 'none', value: -1 }); - - callback(null, aggregation.name); - }); -}; - -Histogram.prototype.format = function(result, override) { - override = override || {}; - var buckets = []; - - var binsCount = getBinsCount(override); - var width = getWidth(override); - var binsStart = getBinStart(override); - var nulls = 0; - var infinities = 0; - var nans = 0; - var avg; - var timestampStart; - var aggregation; - var offset; - - if (result.rows.length) { - var firstRow = result.rows[0]; - binsCount = firstRow.bins_number; - width = firstRow.bin_width || width; - avg = firstRow.avg_val; - nulls = firstRow.nulls_count; - timestampStart = firstRow.timestamp_start; - infinities = firstRow.infinities_count; - nans = firstRow.nans_count; - binsStart = populateBinStart(override, firstRow); - - if (Number.isFinite(timestampStart)) { - aggregation = getAggregation(override, this.aggregation); - offset = getOffset(override, this.offset); + _getHistogramSubtype (override) { + if(this._isDateHistogram(override)) { + return DATE_HISTOGRAM; } - buckets = result.rows.map(function(row) { - return _.omit( - row, - 'bins_number', - 'bin_width', - 'nulls_count', - 'infinities_count', - 'nans_count', - 'avg_val', - 'timestamp_start' - ); - }); + return NUMERIC_HISTOGRAM; } - return { - aggregation: aggregation, - offset: offset, - timestamp_start: timestampStart, - bin_width: width, - bins_count: binsCount, - bins_start: binsStart, - nulls: nulls, - infinities: infinities, - nans: nans, - avg: avg, - bins: buckets - }; -}; - -function getAggregation(override, aggregation) { - return override && override.aggregation ? override.aggregation : aggregation; -} - -function getOffset(override, offset) { - if (override && override.offset) { - return override.offset; + _isDateHistogram (override = {}) { + return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation')); } - if (offset) { - return offset; - } - - return 0; -} -function getBinStart(override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.min(override.start, override.end); + getResult (psql, override, callback) { + this.histogramImplementation = this._getHistogramImplementation(override); + this.histogramImplementation.getResult(psql, override, callback); } - return override.start || 0; -} - -function getBinEnd(override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; -} - -function getBinsCount(override) { - return override.bins || 0; -} -function getWidth(override) { - var width = 0; - var binsCount = override.bins; + // In order to keep previous behaviour with overviews, + // we have to expose the following methods to bypass + // the concrete overview implementation - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; + sql (psql, override, callback) { + this.histogramImplementation.sql(psql, override, callback); } - return width; -} - -function parseOffset(offset, aggregation) { - if (!offset) { - return '0'; - } - if (aggregation === 'hour' || aggregation === 'minute') { - return '0'; + format (result, override) { + return this.histogramImplementation.format(result, override); } - - var offsetInHours = Math.ceil(offset / 3600); - return '' + offsetInHours; -} - -function populateBinStart(override, firstRow) { - var binStart; - if (firstRow.hasOwnProperty('timestamp')) { - binStart = firstRow.timestamp; - } else if (override.hasOwnProperty('start')) { - binStart = getBinStart(override); - } else { - binStart = firstRow.min; + getType () { + return this.histogramImplementation.getType(); } - return binStart; -} - -Histogram.prototype.getType = function() { - return TYPE; -}; - -Histogram.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); + toString () { + return this.histogramImplementation.toString(); + } }; diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js new file mode 100644 index 00000000..ec251d09 --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -0,0 +1,85 @@ +const BaseDataview = require('../base'); + +const TYPE = 'histogram'; + +module.exports = class BaseHistogram extends BaseDataview { + constructor (query, options, queries) { + super(); + + if (typeof options.column !== 'string') { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; + } + + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { + // assume numeric, will fail later + this._columnType = 'numeric'; + if (!err && !!type) { + this._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + this.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + return this._buildQuery(psql, override, callback); + } + + format (result, override) { + const histogram = this._getSummary(result, override); + histogram.bins = this._getBuckets(result); + return histogram; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } + + _hasOverridenRange (override) { + return override && override.hasOwnProperty('start') && override.hasOwnProperty('end'); + } + + _getBinStart (override = {}) { + if (this._hasOverridenRange(override)) { + return Math.min(override.start, override.end); + } + + return override.start || 0; + } + + _getBinEnd (override = {}) { + if (this._hasOverridenRange(override)) { + return Math.max(override.start, override.end); + } + + return override.end || 0; + } + + _getBinsCount (override = {}) { + return override.bins || 0; + } +}; diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js new file mode 100644 index 00000000..34a42f9a --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -0,0 +1,302 @@ +const BaseHistogram = require('./base-histogram'); +const debug = require('debug')('windshaft:dataview:date-histogram'); + +const dateIntervalQueryTpl = ctx => ` + WITH + __cdb_dates AS ( + SELECT + MAX(${ctx.column}::timestamp) AS __cdb_end, + MIN(${ctx.column}::timestamp) AS __cdb_start + FROM (${ctx.query}) __cdb_source + ), + __cdb_interval_in_days AS ( + SELECT + DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days + FROM __cdb_dates + ), + __cdb_interval_in_hours AS ( + SELECT + __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours + FROM __cdb_interval_in_days, __cdb_dates + ), + __cdb_interval_in_minutes AS ( + SELECT + __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes + FROM __cdb_interval_in_hours, __cdb_dates + ), + __cdb_interval_in_seconds AS ( + SELECT + __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds + FROM __cdb_interval_in_minutes, __cdb_dates + ) + SELECT + ROUND(__cdb_days / 365) AS year, + ROUND(__cdb_days / 90) AS quarter, + ROUND(__cdb_days / 30) AS month, + ROUND(__cdb_days / 7) AS week, + __cdb_days AS day, + __cdb_hours AS hour, + __cdb_minutes AS minute, + __cdb_seconds AS second + FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds +`; + +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${ctx.query}) __cdb_histogram_nulls + WHERE ${ctx.column} IS NULL + ) +`; + +const dateBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(date_part('epoch', ${ctx.column})) AS __cdb_max_val, + min(date_part('epoch', ${ctx.column})) AS __cdb_min_val, + avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' + ) + ) AS __cdb_start_date, + max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx.query}) __cdb_basics_query + ) +`; + +const dateOverrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx.end})::float AS __cdb_max_val, + min(${ctx.start})::float AS __cdb_min_val, + avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx.aggregation}', + TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}' + ) + ) AS __cdb_start_date, + max( + TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}' + ) AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx.query}) __cdb_basics_query + ) +`; + +const dateBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + __cdb_bins_array, + ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number + FROM ( + SELECT + ARRAY( + SELECT GENERATE_SERIES( + __cdb_start_date::timestamptz, + __cdb_end_date::timestamptz, + ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`} + ) + ) AS __cdb_bins_array + FROM __cdb_basics + ) __cdb_bins_array_query + ) +`; + +const dateHistogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET( + ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}', + __cdb_bins_array + ), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min( + date_part( + 'epoch', + date_trunc( + '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' + ) AT TIME ZONE '${ctx.offset}' + ) + )::numeric AS timestamp, + date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, + min(date_part('epoch', ${ctx.column}))::numeric AS min, + max(date_part('epoch', ${ctx.column}))::numeric AS max, + avg(date_part('epoch', ${ctx.column}))::numeric AS avg, + count(*) AS freq + FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls + WHERE date_part('epoch', ${ctx.column}) IS NOT NULL + GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start + ORDER BY bin +`; + +const MAX_INTERVAL_VALUE = 366; + +const DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + +/** + date_histogram: { + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // MANDATORY + offset: -7200 // OPTIONAL (UTC offset in seconds) + } + } +*/ +module.exports = class DateHistogram extends BaseHistogram { + constructor (query, options, queries) { + super(query, options, queries); + + this.aggregation = options.aggregation; + this.offset = options.offset; + } + + _buildQueryTpl (ctx) { + return ` + WITH + ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, + ${dateBinsQueryTpl(ctx)}, + ${nullsQueryTpl(ctx)} + ${dateHistogramQueryTpl(ctx)} + `; + } + + _buildQuery (psql, override, callback) { + if (!this._isValidAggregation(override)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (this._getAggregation(override) === 'auto') { + this._getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildQuery(psql, override, callback); + }.bind(this)); + return null; + } + + const histogramSql = this._buildQueryTpl({ + override: override, + query: this.query, + column: this.column, + aggregation: this._getAggregation(override), + start: this._getBinStart(override), + end: this._getBinEnd(override), + offset: this._parseOffset(override) + }); + + debug(histogramSql); + + return callback(null, histogramSql); + } + + _isValidAggregation (override) { + return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override)); + } + + _getAutomaticAggregation (psql, callback) { + const dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + const aggegations = result.rows[0]; + const aggregation = Object.keys(aggegations) + .map(key => ({ name: key, value: aggegations[key] })) + .reduce((closer, current) => { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + const closerDiff = MAX_INTERVAL_VALUE - closer.value; + const currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); + } + + _getSummary (result, override) { + const firstRow = result.rows[0] || {}; + + return { + aggregation: this._getAggregation(override), + offset: this._getOffset(override), + timestamp_start: firstRow.timestamp_start, + + bin_width: firstRow.bin_width, + bins_count: firstRow.bins_number, + bins_start: firstRow.timestamp, + nulls: firstRow.nulls_count, + infinities: firstRow.infinities_count, + nans: firstRow.nans_count, + avg: firstRow.avg_val + }; + } + + _getBuckets (result) { + return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp })); + } + + _getAggregation (override = {}) { + return override.aggregation ? override.aggregation : this.aggregation; + } + + _getOffset (override = {}) { + return Number.isFinite(override.offset) ? override.offset : (this.offset || 0); + } + + _parseOffset (override) { + if (this._shouldIgnoreOffset(override)) { + return '0'; + } + + const offsetInHours = Math.ceil(this._getOffset(override) / 3600); + + return '' + offsetInHours; + } + + _shouldIgnoreOffset (override) { + return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute'); + } +}; diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js new file mode 100644 index 00000000..ea191bea --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -0,0 +1,234 @@ +const BaseHistogram = require('./base-histogram'); +const debug = require('debug')('windshaft:dataview:numeric-histogram'); + +const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; + +const filterOutSpecialNumericValues = ctx => ` + ${ctx.column} != 'infinity'::float + AND + ${ctx.column} != '-infinity'::float + AND + ${ctx.column} != 'NaN'::float +`; + +const filteredQueryTpl = ctx => ` + __cdb_filtered_source AS ( + SELECT * + FROM (${ctx.query}) __cdb_filtered_source_query + WHERE ${ctx.column} IS NOT NULL + ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} + ) +`; + +const basicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val, + avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; + +const overrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val, + avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; + +const iqrQueryTpl = ctx => ` + __cdb_iqrange AS ( + SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr + FROM ( + SELECT quartile, max(_cdb_iqr_column) AS quartile_max from ( + SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column} + ) AS quartile + FROM __cdb_filtered_source) _cdb_quartiles + WHERE quartile = 1 or quartile = 3 + GROUP BY quartile + ) __cdb_iqr + ) +`; + +const binsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 + THEN 1 + ELSE GREATEST( + LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)), + LEAST( + CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT), + ${ctx.maxBins} + ) + ) + END AS __cdb_bins_number + FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source + LIMIT 1 + ) +`; + +const overrideBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT ${ctx.override.bins} AS __cdb_bins_number + ) +`; + +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${ctx.query}) __cdb_histogram_nulls + WHERE ${ctx.column} IS NULL + ) +`; + +const infinitiesQueryTpl = ctx => ` + __cdb_infinities AS ( + SELECT + count(*) AS __cdb_infinities_count + FROM (${ctx.query}) __cdb_infinities_query + WHERE + ${ctx.column} = 'infinity'::float + OR + ${ctx.column} = '-infinity'::float + ) +`; + +const nansQueryTpl = ctx => ` + __cdb_nans AS ( + SELECT + count(*) AS __cdb_nans_count + FROM (${ctx.query}) __cdb_nans_query + WHERE ${ctx.column} = 'NaN'::float + ) +`; + +const specialNumericValuesColumnDefinitionTpl = () => ` + __cdb_infinities_count AS infinities_count, + __cdb_nans_count AS nans_count +`; + +const specialNumericValuesCTETpl = () => ` + __cdb_infinities, __cdb_nans +`; + +const specialNumericValuesColumnTpl = () => ` + infinities_count, nans_count +`; + +const histogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} + __cdb_avg_val AS avg_val, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min(${ctx.column})::numeric AS min, + max(${ctx.column})::numeric AS max, + avg(${ctx.column})::numeric AS avg, + count(*) AS freq + FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins + ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} + GROUP BY bin, bins_number, bin_width, nulls_count, avg_val + ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} + ORDER BY bin +`; + +const BIN_MIN_NUMBER = 6; +const BIN_MAX_NUMBER = 48; + +/** +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} +*/ +module.exports = class NumericHistogram extends BaseHistogram { + constructor (query, options, queries) { + super(query, options, queries); + } + + _buildQuery (psql, override, callback) { + const histogramSql = this._buildQueryTpl({ + override: override, + column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, + isFloatColumn: this._columnType === 'float', + query: this.query, + start: this._getBinStart(override), + end: this._getBinEnd(override), + minBins: BIN_MIN_NUMBER, + maxBins: BIN_MAX_NUMBER, + }); + + debug(histogramSql); + + return callback(null, histogramSql); + } + + _buildQueryTpl (ctx) { + return ` + WITH + ${filteredQueryTpl(ctx)}, + ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._hasOverridenBins(ctx.override) ? + overrideBinsQueryTpl(ctx) : + `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` + }, + ${nullsQueryTpl(ctx)} + ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} + ${histogramQueryTpl(ctx)} + `; + } + + _hasOverridenBins (override) { + return override && override.hasOwnProperty('bins'); + } + + _getSummary (result, override) { + const firstRow = result.rows[0] || {}; + + return { + bin_width: firstRow.bin_width, + bins_count: firstRow.bins_number, + bins_start: this._populateBinStart(firstRow, override), + nulls: firstRow.nulls_count, + infinities: firstRow.infinities_count, + nans: firstRow.nans_count, + avg: firstRow.avg_val, + }; + } + + _getBuckets (result) { + return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq })); + } + + _populateBinStart (firstRow, override = {}) { + let binStart; + + if (override.hasOwnProperty('start')) { + binStart = this._getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; + } + +}; diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index 33993777..a54c660f 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -1,11 +1,9 @@ -var dot = require('dot'); -dot.templateSettings.strip = false; +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:list'); -var BaseWidget = require('./base'); +const TYPE = 'list'; -var TYPE = 'list'; - -var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); +const listSqlTpl = ctx => `select ${ctx.columns} from (${ctx.query}) as _cdb_list`; /** { @@ -15,52 +13,52 @@ var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as } } */ +module.exports = class List extends BaseDataview { + constructor (query, options = {}) { + super(); -function List(query, options) { - options = options || {}; + this._checkOptions(options); - if (!Array.isArray(options.columns)) { - throw new Error('List expects `columns` array in widget options'); + this.query = query; + this.columns = options.columns; } - BaseWidget.apply(this); - - this.query = query; - this.columns = options.columns; -} - -List.prototype = new BaseWidget(); -List.prototype.constructor = List; - -module.exports = List; - -List.prototype.sql = function(psql, override, callback) { - if (!callback) { - callback = override; + _checkOptions (options) { + if (!Array.isArray(options.columns)) { + throw new Error('List expects `columns` array in dataview options'); + } } - var listSql = listSqlTpl({ - _query: this.query, - _columns: this.columns.join(', ') - }); + sql (psql, override, callback) { + if (!callback) { + callback = override; + } - return callback(null, listSql); -}; + const listSql = listSqlTpl({ + query: this.query, + columns: this.columns.join(', ') + }); -List.prototype.format = function(result) { - return { - rows: result.rows - }; -}; + debug(listSql); -List.prototype.getType = function() { - return TYPE; -}; + return callback(null, listSql); + } -List.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _columns: this.columns.join(', ') - }); + format (result) { + return { + rows: result.rows + }; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _columns: this.columns.join(', ') + }); + } }; diff --git a/package.json b/package.json index 39aedfa6..8bf90aaf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.13.0", + "version": "3.13.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -62,6 +62,6 @@ }, "engines": { "node": ">=6.9", - "yarn": "^0.21.3" + "yarn": ">=0.27.5 <1.0.0" } }