diff --git a/INSTALL.md b/INSTALL.md index 2de2cb09..45654b0b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are - Core - Node.js >=6.9.x - - yarn >=0.21.3 + - yarn >=0.27.5 <1.0.0 - PostgreSQL >8.3.x, PostGIS >1.5.x - Redis >2.4.0 (http://www.redis.io) - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik). diff --git a/NEWS.md b/NEWS.md index 15720ece..139c9a3f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,23 @@ # Changelog -## 3.12.10 +## 3.13.1 Released 2017-mm-dd + - Upgrades yarn minimum version requirement to v0.27.5 + +Bugfixes: + - + +## 3.13.0 +Released 2017-10-02 + - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables. + +Bugfixes: + - Bounding box parameter ignored in static named maps #735. + - camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis + +## 3.12.10 +Released 2017-09-18 + - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2). ## 3.12.9 Released 2017-09-07 diff --git a/app.js b/app.js index 1220e8f6..9270cac0 100755 --- a/app.js +++ b/app.js @@ -2,14 +2,20 @@ var http = require('http'); var https = require('https'); var path = require('path'); var fs = require('fs'); - var _ = require('underscore'); +var semver = require('semver'); // jshint undef:false var log = console.log.bind(console); var logError = console.error.bind(console); // jshint undef:true +var nodejsVersion = process.versions.node; +if (!semver.satisfies(nodejsVersion, '>=6.9.0')) { + logError(`Node version ${nodejsVersion} is not supported, please use Node.js 6.9 or higher.`); + process.exit(1); +} + var argv = require('yargs') .usage('Usage: $0 [options]') .help('h') diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index f74702e9..d743a158 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -4,9 +4,6 @@ var _ = require('underscore'); var step = require('step'); var debug = require('debug')('windshaft:cartodb'); -var LZMA = require('lzma').LZMA; -var lzmaWorker = new LZMA(); - // Whitelist query parameters and attach format var REQUEST_QUERY_PARAMS_WHITELIST = [ 'config', @@ -28,7 +25,7 @@ function BaseController(authApi, pgConnection) { module.exports = BaseController; -// jshint maxcomplexity:10 +// jshint maxcomplexity:8 /** * Whitelist input and get database name & default geometry type from * subdomain/user metadata held in CartoDB Redis @@ -38,35 +35,6 @@ module.exports = BaseController; BaseController.prototype.req2params = function(req, callback){ var self = this; - if ( req.query.lzma ) { - - // Decode (from base64) - var lzma = new Buffer(req.query.lzma, 'base64') - .toString('binary') - .split('') - .map(function(c) { - return c.charCodeAt(0) - 128; - }); - - - // Decompress - lzmaWorker.decompress( - lzma, - function(result) { - req.profiler.done('lzma'); - try { - delete req.query.lzma; - _.extend(req.query, JSON.parse(result)); - self.req2params(req, callback); - } catch (err) { - req.profiler.done('req2params'); - callback(new Error('Error parsing lzma as JSON: ' + err)); - } - } - ); - return; - } - var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST; if (Array.isArray(req.context.allowedQueryParams)) { allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams); diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index bde3827d..b6aa8bdf 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -32,7 +32,8 @@ NamedMapsController.prototype.register = function(app) { this.tile.bind(this)); app.get(app.base_url_mapconfig + - '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']), + '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, + allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), this.staticMap.bind(this)); }; diff --git a/lib/cartodb/middleware/lzma.js b/lib/cartodb/middleware/lzma.js new file mode 100644 index 00000000..d58f16cc --- /dev/null +++ b/lib/cartodb/middleware/lzma.js @@ -0,0 +1,30 @@ +'use strict'; + +var LZMA = require('lzma').LZMA; + +var lzmaWorker = new LZMA(); + +module.exports = function lzmaMiddleware(req, res, next) { + if (!req.query.hasOwnProperty('lzma')) { + return next(); + } + + // Decode (from base64) + var lzma = new Buffer(req.query.lzma, 'base64') + .toString('binary') + .split('') + .map(function(c) { + return c.charCodeAt(0) - 128; + }); + + // Decompress + lzmaWorker.decompress(lzma, function(result) { + try { + delete req.query.lzma; + Object.assign(req.query, JSON.parse(result)); + next(); + } catch (err) { + next(new Error('Error parsing lzma as JSON: ' + err)); + } + }); +}; diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index 77f37723..b4c59af2 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -1,95 +1,178 @@ -var _ = require('underscore'); -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:widget:aggregation'); +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:aggregation'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const filteredQueryTpl = ctx => ` + filtered_source AS ( + SELECT * + FROM (${ctx.query}) _cdb_filtered_source + ${ctx.aggregationColumn && ctx.isFloatColumn ? ` + WHERE + ${ctx.aggregationColumn} != 'infinity'::float + AND + ${ctx.aggregationColumn} != '-infinity'::float + AND + ${ctx.aggregationColumn} != 'NaN'::float` : + '' + } + ) +`; -var filteredQueryTpl = dot.template([ - 'filtered_source AS (', - ' SELECT *', - ' FROM ({{=it._query}}) _cdb_filtered_source', - ' {{?it._aggregationColumn && it._isFloatColumn}}WHERE', - ' {{=it._aggregationColumn}} != \'infinity\'::float', - ' AND', - ' {{=it._aggregationColumn}} != \'-infinity\'::float', - ' AND', - ' {{=it._aggregationColumn}} != \'NaN\'::float{{?}}', - ')' -].join(' \n')); +const summaryQueryTpl = ctx => ` + summary AS ( + SELECT + count(1) AS count, + sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count + ${ctx.isFloatColumn ? `, + sum( + CASE + WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float + THEN 1 + ELSE 0 + END + ) AS infinities_count, + sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` : + '' + } + FROM (${ctx.query}) _cdb_aggregation_nulls + ) +`; -var summaryQueryTpl = dot.template([ - 'summary AS (', - ' SELECT', - ' count(1) AS count,', - ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', - ' {{?it._isFloatColumn}},sum(', - ' CASE', - ' WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float', - ' THEN 1', - ' ELSE 0', - ' END', - ' ) AS infinities_count,', - ' sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}', - ' FROM ({{=it._query}}) _cdb_aggregation_nulls', - ')' -].join('\n')); +const rankedCategoriesQueryTpl = ctx => ` + categories AS( + SELECT + ${ctx.column} AS category, + ${ctx.aggregationFn} AS value, + row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank + FROM filtered_source + ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''} + GROUP BY ${ctx.column} + ORDER BY 2 DESC + ) +`; -var rankedCategoriesQueryTpl = dot.template([ - 'categories AS(', - ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', - ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', - ' FROM filtered_source', - ' {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}', - ' GROUP BY {{=it._column}}', - ' ORDER BY 2 DESC', - ')' -].join('\n')); +const categoriesSummaryMinMaxQueryTpl = () => ` + categories_summary_min_max AS( + SELECT + max(value) max_val, + min(value) min_val + FROM categories + ) +`; -var categoriesSummaryMinMaxQueryTpl = dot.template([ - 'categories_summary_min_max AS(', - ' SELECT max(value) max_val, min(value) min_val', - ' FROM categories', - ')' -].join('\n')); +const categoriesSummaryCountQueryTpl = ctx => ` + categories_summary_count AS( + SELECT count(1) AS categories_count + FROM ( + SELECT ${ctx.column} AS category + FROM (${ctx.query}) _cdb_categories + GROUP BY ${ctx.column} + ) _cdb_categories_count + ) +`; -var categoriesSummaryCountQueryTpl = dot.template([ - 'categories_summary_count AS(', - ' SELECT count(1) AS categories_count', - ' FROM (', - ' SELECT {{=it._column}} AS category', - ' FROM ({{=it._query}}) _cdb_categories', - ' GROUP BY {{=it._column}}', - ' ) _cdb_categories_count', - ')' -].join('\n')); +const specialNumericValuesColumns = () => `, nans_count, infinities_count`; -var rankedAggregationQueryTpl = dot.template([ - 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,', - ' count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - ' FROM categories, summary, categories_summary_min_max, categories_summary_count', - ' WHERE rank < {{=it._limit}}', - 'UNION ALL', - 'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,', - ' min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - ' FROM categories, summary, categories_summary_min_max, categories_summary_count', - ' WHERE rank >= {{=it._limit}}', - 'GROUP BY nulls_count, min_val, max_val, count,', - ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}' -].join('\n')); +const rankedAggregationQueryTpl = ctx => ` + SELECT + CAST(category AS text), + value, + false as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM categories, summary, categories_summary_min_max, categories_summary_count + WHERE rank < ${ctx.limit} + UNION ALL + SELECT + 'Other' category, + ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value, + true as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM categories, summary, categories_summary_min_max, categories_summary_count + WHERE rank >= ${ctx.limit} + GROUP BY + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } +`; -var aggregationQueryTpl = dot.template([ - 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', - ' nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count', - 'GROUP BY category, nulls_count, min_val, max_val, count,', - ' categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}', - 'ORDER BY value DESC' -].join('\n')); +const aggregationQueryTpl = ctx => ` + SELECT + CAST(${ctx.column} AS text) AS category, + ${ctx.aggregationFn} AS value, + false as agg, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count + GROUP BY + category, + nulls_count, + min_val, + max_val, + count, + categories_count + ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' } + ORDER BY value DESC +`; -var CATEGORIES_LIMIT = 6; +const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`; -var VALID_OPERATIONS = { +const aggregationDataviewQueryTpl = ctx => ` + WITH + ${filteredQueryTpl(ctx)}, + ${summaryQueryTpl(ctx)}, + ${rankedCategoriesQueryTpl(ctx)}, + ${categoriesSummaryMinMaxQueryTpl(ctx)}, + ${categoriesSummaryCountQueryTpl(ctx)} + ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`} +`; + +const filterCategoriesQueryTpl = ctx => ` + SELECT + ${ctx.column} AS category, + ${ctx.value} AS value + FROM (${ctx.query}) _cdb_aggregation_search + WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery} + GROUP BY ${ctx.column} +`; + +const searchQueryTpl = ctx => ` + WITH + search_unfiltered AS ( + ${ctx.searchUnfiltered} + ), + search_filtered AS ( + ${ctx.searchFiltered} + ), + search_union AS ( + SELECT * FROM search_unfiltered + UNION ALL + SELECT * FROM search_filtered + ) + SELECT category, sum(value) AS value + FROM search_union + GROUP BY category + ORDER BY value desc +`; + +const CATEGORIES_LIMIT = 6; + +const VALID_OPERATIONS = { count: [], sum: ['aggregationColumn'], avg: ['aggregationColumn'], @@ -97,7 +180,7 @@ var VALID_OPERATIONS = { max: ['aggregationColumn'] }; -var TYPE = 'aggregation'; +const TYPE = 'aggregation'; /** { @@ -108,256 +191,150 @@ var TYPE = 'aggregation'; } } */ -function Aggregation(query, options, queries) { - if (!_.isString(options.column)) { - throw new Error('Aggregation expects `column` in widget options'); +module.exports = class Aggregation extends BaseDataview { + constructor (query, options = {}, queries = {}) { + super(); + + this._checkOptions(options); + + this.query = query; + this.queries = queries; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; + this._isFloatColumn = null; } - if (!_.isString(options.aggregation)) { - throw new Error('Aggregation expects `aggregation` operation in widget options'); - } - - if (!VALID_OPERATIONS[options.aggregation]) { - throw new Error("Aggregation does not support '" + options.aggregation + "' operation"); - } - - var requiredOptions = VALID_OPERATIONS[options.aggregation]; - var missingOptions = _.difference(requiredOptions, Object.keys(options)); - if (missingOptions.length > 0) { - throw new Error( - "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') - ); - } - - BaseWidget.apply(this); - - this.query = query; - this.queries = queries; - this.column = options.column; - this.aggregation = options.aggregation; - this.aggregationColumn = options.aggregationColumn; - this._isFloatColumn = null; -} - -Aggregation.prototype = new BaseWidget(); -Aggregation.prototype.constructor = Aggregation; - -module.exports = Aggregation; - -Aggregation.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this.aggregationColumn && this._isFloatColumn === null) { - this._isFloatColumn = false; - this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) { - if (!err && !!type) { - self._isFloatColumn = type.float; - } - self.sql(psql, override, callback); - }); - return null; - } - - var _query = this.query; - - var aggregationSql; - - if (!!override.ownFilter) { - aggregationSql = [ - this.getCategoriesCTESql( - _query, - this.column, - this.aggregation, - this.aggregationColumn, - this._isFloatColumn - ), - aggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: _query, - _column: this.column, - _aggregation: this.getAggregationSql(), - _limit: CATEGORIES_LIMIT - }) - ].join('\n'); - } else { - aggregationSql = [ - this.getCategoriesCTESql( - _query, - this.column, - this.aggregation, - this.aggregationColumn, - this._isFloatColumn - ), - rankedAggregationQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: _query, - _column: this.column, - _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum', - _limit: CATEGORIES_LIMIT - }) - ].join('\n'); - } - - debug(aggregationSql); - - return callback(null, aggregationSql); -}; - -Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) { - return [ - "WITH", - [ - filteredQueryTpl({ - _isFloatColumn: isFloatColumn, - _query: this.query, - _column: this.column, - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - summaryQueryTpl({ - _isFloatColumn: isFloatColumn, - _query: query, - _column: column, - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - rankedCategoriesQueryTpl({ - _query: query, - _column: column, - _aggregation: this.getAggregationSql(), - _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null - }), - categoriesSummaryMinMaxQueryTpl({ - _query: query, - _column: column - }), - categoriesSummaryCountQueryTpl({ - _query: query, - _column: column - }) - ].join(',\n') - ].join('\n'); -}; - -var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); -Aggregation.prototype.getAggregationSql = function() { - return aggregationFnQueryTpl({ - _aggregationFn: this.aggregation, - _aggregationColumn: this.aggregationColumn || 1 - }); -}; - -Aggregation.prototype.format = function(result) { - var categories = []; - var count = 0; - var nulls = 0; - var nans = 0; - var infinities = 0; - var minValue = 0; - var maxValue = 0; - var categoriesCount = 0; - - - if (result.rows.length) { - var firstRow = result.rows[0]; - count = firstRow.count; - nulls = firstRow.nulls_count; - nans = firstRow.nans_count; - infinities = firstRow.infinities_count; - minValue = firstRow.min_val; - maxValue = firstRow.max_val; - categoriesCount = firstRow.categories_count; - - result.rows.forEach(function(row) { - categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', - 'max_val', 'categories_count', 'nans_count', 'infinities_count')); - }); - } - - return { - aggregation: this.aggregation, - count: count, - nulls: nulls, - nans: nans, - infinities: infinities, - min: minValue, - max: maxValue, - categoriesCount: categoriesCount, - categories: categories - }; -}; - -var filterCategoriesQueryTpl = dot.template([ - 'SELECT {{=it._column}} AS category, {{=it._value}} AS value', - 'FROM ({{=it._query}}) _cdb_aggregation_search', - 'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}', - 'GROUP BY {{=it._column}}' -].join('\n')); - -var searchQueryTpl = dot.template([ - 'WITH', - 'search_unfiltered AS (', - ' {{=it._searchUnfiltered}}', - '),', - 'search_filtered AS (', - ' {{=it._searchFiltered}}', - '),', - 'search_union AS (', - ' SELECT * FROM search_unfiltered', - ' UNION ALL', - ' SELECT * FROM search_filtered', - ')', - 'SELECT category, sum(value) AS value', - 'FROM search_union', - 'GROUP BY category', - 'ORDER BY value desc' -].join('\n')); - - -Aggregation.prototype.search = function(psql, userQuery, callback) { - var self = this; - - var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); - var _value = this.aggregation !== 'count' && this.aggregationColumn ? - this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)'; - - // TODO unfiltered will be wrong as filters are already applied at this point - var query = searchQueryTpl({ - _searchUnfiltered: filterCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _value: '0', - _userQuery: _userQuery - }), - _searchFiltered: filterCategoriesQueryTpl({ - _query: this.query, - _column: this.column, - _value: _value, - _userQuery: _userQuery - }) - }); - - psql.query(query, function(err, result) { - if (err) { - return callback(err, result); + _checkOptions (options) { + if (typeof options.column !== 'string') { + throw new Error(`Aggregation expects 'column' in dataview options`); } - return callback(null, {type: self.getType(), categories: result.rows }); - }, true); // use read-only transaction -}; + if (typeof options.aggregation !== 'string') { + throw new Error(`Aggregation expects 'aggregation' operation in dataview options`); + } -Aggregation.prototype.getType = function() { - return TYPE; -}; + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error(`Aggregation does not support '${options.aggregation}' operation`); + } -Aggregation.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _column: this.column, - _aggregation: this.aggregation - }); + const requiredOptions = VALID_OPERATIONS[options.aggregation]; + const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption)); + + if (missingOptions.length > 0) { + throw new Error( + `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}` + ); + } + } + + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + if (this._shouldCheckColumnType()) { + this._isFloatColumn = false; + this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => { + if (!err && !!type) { + this._isFloatColumn = type.float; + } + this.sql(psql, override, callback); + }); + return null; + } + + const aggregationSql = aggregationDataviewQueryTpl({ + override: override, + query: this.query, + column: this.column, + aggregation: this.aggregation, + aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null, + aggregationFn: aggregationFnQueryTpl({ + aggregation: this.aggregation, + aggregationColumn: this.aggregationColumn || 1 + }), + isFloatColumn: this._isFloatColumn, + limit: CATEGORIES_LIMIT + }); + + debug(aggregationSql); + + return callback(null, aggregationSql); + } + + _shouldCheckColumnType () { + return this.aggregationColumn && this._isFloatColumn === null; + } + + format (result) { + const { + count = 0, + nulls_count = 0, + nans_count = 0, + infinities_count = 0, + min_val = 0, + max_val = 0, + categories_count = 0 + } = result.rows[0] || {}; + + return { + aggregation: this.aggregation, + count: count, + nulls: nulls_count, + nans: nans_count, + infinities: infinities_count, + min: min_val, + max: max_val, + categoriesCount: categories_count, + categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg })) + }; + } + + search (psql, userQuery, callback) { + const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`); + const value = this.aggregation !== 'count' && this.aggregationColumn ? + `${this.aggregation}(${this.aggregationColumn})` : + 'count(1)'; + + // TODO unfiltered will be wrong as filters are already applied at this point + const query = searchQueryTpl({ + searchUnfiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: '0', + userQuery: escapedUserQuery + }), + searchFiltered: filterCategoriesQueryTpl({ + query: this.query, + column: this.column, + value: value, + userQuery: escapedUserQuery + }) + }); + + debug(query); + + psql.query(query, (err, result) => { + if (err) { + return callback(err, result); + } + + return callback(null, {type: this.getType(), categories: result.rows }); + }, true); // use read-only transaction + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _aggregation: this.aggregation + }); + } }; diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index 7ec356b7..daaeaf0c 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,28 +1,36 @@ -var _ = require('underscore'); -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:widget:formula'); +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:formula'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const countInfinitiesQueryTpl = ctx => ` + SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities + WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float +`; -var formulaQueryTpl = dot.template([ - 'SELECT', - ' {{=it._operation}}({{=it._column}}) AS result,', - ' (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - ' {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ' ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' -].join('\n')); +const countNansQueryTpl = ctx => ` + SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans + WHERE ${ctx.column} = 'NaN'::float +`; -var VALID_OPERATIONS = { +const filterOutSpecialNumericValuesTpl = ctx => ` + WHERE + ${ctx.column} != 'infinity'::float + AND + ${ctx.column} != '-infinity'::float + AND + ${ctx.column} != 'NaN'::float +`; + +const formulaQueryTpl = ctx => ` + SELECT + ${ctx.operation}(${ctx.column}) AS result, + (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count + ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''} + ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''} + FROM (${ctx.query}) __cdb_formula + ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''} +`; + +const VALID_OPERATIONS = { count: true, avg: true, sum: true, @@ -30,7 +38,7 @@ var VALID_OPERATIONS = { max: true }; -var TYPE = 'formula'; +const TYPE = 'formula'; /** { @@ -41,93 +49,90 @@ var TYPE = 'formula'; } } */ -function Formula(query, options, queries) { - if (!_.isString(options.operation)) { - throw new Error('Formula expects `operation` in widget options'); +module.exports = class Formula extends BaseDataview { + constructor (query, options = {}, queries = {}) { + super(); + + this._checkOptions(options); + + this.query = query; + this.queries = queries; + this.column = options.column || '1'; + this.operation = options.operation; + this._isFloatColumn = null; } - if (!VALID_OPERATIONS[options.operation]) { - throw new Error("Formula does not support '" + options.operation + "' operation"); + _checkOptions (options) { + if (typeof options.operation !== 'string') { + throw new Error(`Formula expects 'operation' in dataview options`); + } + + if (!VALID_OPERATIONS[options.operation]) { + throw new Error(`Formula does not support '${options.operation}' operation`); + } + + if (options.operation !== 'count' && typeof options.column !== 'string') { + throw new Error(`Formula expects 'column' in dataview options`); + } } - if (options.operation !== 'count' && !_.isString(options.column)) { - throw new Error('Formula expects `column` in widget options'); - } - BaseWidget.apply(this); + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } - this.query = query; - this.queries = queries; - this.column = options.column || '1'; - this.operation = options.operation; - this._isFloatColumn = null; -} + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { + if (!err && !!type) { + this._isFloatColumn = type.float; + } + this.sql(psql, override, callback); + }); + return null; + } -Formula.prototype = new BaseWidget(); -Formula.prototype.constructor = Formula; - -module.exports = Formula; - -Formula.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this._isFloatColumn === null) { - this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - if (!err && !!type) { - self._isFloatColumn = type.float; - } - self.sql(psql, override, callback); + const formulaSql = formulaQueryTpl({ + isFloatColumn: this._isFloatColumn, + query: this.query, + operation: this.operation, + column: this.column }); - return null; + + debug(formulaSql); + + return callback(null, formulaSql); } - var formulaSql = formulaQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.query, - _operation: this.operation, - _column: this.column - }); + format (res) { + const { + result = 0, + nulls_count = 0, + nans_count, + infinities_count + } = res.rows[0] || {}; - debug(formulaSql); - - return callback(null, formulaSql); -}; - -Formula.prototype.format = function(result) { - var formattedResult = { - operation: this.operation, - result: 0, - nulls: 0, - nans: 0, - infinities: 0 - }; - - if (result.rows.length) { - formattedResult.operation = this.operation; - formattedResult.result = result.rows[0].result; - formattedResult.nulls = result.rows[0].nulls_count; - formattedResult.nans = result.rows[0].nans_count; - formattedResult.infinities = result.rows[0].infinities_count; + return { + operation: this.operation, + result, + nulls: nulls_count, + nans: nans_count, + infinities: infinities_count + }; } - return formattedResult; -}; + getType () { + return TYPE; + } -Formula.prototype.getType = function() { - return TYPE; -}; - -Formula.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _column: this.column, - _operation: this.operation - }); + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _operation: this.operation + }); + } }; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js index 6e62ad9d..affbbd25 100644 --- a/lib/cartodb/models/dataview/histogram.js +++ b/lib/cartodb/models/dataview/histogram.js @@ -1,717 +1,72 @@ -var _ = require('underscore'); -var BaseWidget = require('./base'); -var debug = require('debug')('windshaft:dataview:histogram'); +const debug = require('debug')('windshaft:dataview:histogram'); +const NumericHistogram = require('./histograms/numeric-histogram'); +const DateHistogram = require('./histograms/date-histogram'); -var dot = require('dot'); -dot.templateSettings.strip = false; +const DATE_HISTOGRAM = 'DateHistogram'; +const NUMERIC_HISTOGRAM = 'NumericHistogram'; -var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); +module.exports = class Histogram { + constructor (query, options, queries) { + this.query = query; + this.options = options || {}; + this.queries = queries; -var dateIntervalQueryTpl = dot.template([ - 'WITH', - '__cdb_dates AS (', - ' SELECT', - ' MAX({{=it.column}}::timestamp) AS __cdb_end,', - ' MIN({{=it.column}}::timestamp) AS __cdb_start', - ' FROM ({{=it.query}}) __cdb_source', - '),', - '__cdb_interval_in_days AS (', - ' SELECT' , - ' DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days', - ' FROM __cdb_dates', - '),', - '__cdb_interval_in_hours AS (', - ' SELECT', - ' __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours', - ' FROM __cdb_interval_in_days, __cdb_dates', - '),', - '__cdb_interval_in_minutes AS (', - ' SELECT', - ' __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes', - ' FROM __cdb_interval_in_hours, __cdb_dates', - '),', - '__cdb_interval_in_seconds AS (', - ' SELECT', - ' __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds', - ' FROM __cdb_interval_in_minutes, __cdb_dates', - ')', - 'SELECT', - ' ROUND(__cdb_days / 365) AS year,', - ' ROUND(__cdb_days / 90) AS quarter,', - ' ROUND(__cdb_days / 30) AS month,', - ' ROUND(__cdb_days / 7) AS week,', - ' __cdb_days AS day,', - ' __cdb_hours AS hour,', - ' __cdb_minutes AS minute,', - ' __cdb_seconds AS second', - 'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds' -].join('\n')); - -var MAX_INTERVAL_VALUE = 366; -var BIN_MIN_NUMBER = 6; -var BIN_MAX_NUMBER = 48; - -var filteredQueryTpl = dot.template([ - '__cdb_filtered_source AS (', - ' SELECT *', - ' FROM ({{=it._query}}) __cdb_filtered_source_query', - ' WHERE', - ' {{=it._column}} IS NOT NULL', - ' {{?it._isFloatColumn}}AND', - ' {{=it._column}} != \'infinity\'::float', - ' AND', - ' {{=it._column}} != \'-infinity\'::float', - ' AND', - ' {{=it._column}} != \'NaN\'::float{{?}}', - ')' -].join(' \n')); - -var basicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,', - ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', - ' FROM __cdb_filtered_source', - ')' -].join(' \n')); - -var overrideBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,', - ' avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows', - ' FROM __cdb_filtered_source', - ')' -].join('\n')); - -var iqrQueryTpl = dot.template([ - '__cdb_iqrange AS (', - ' SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr', - ' FROM (', - ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', - ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', - ' ) AS quartile', - ' FROM __cdb_filtered_source) _cdb_quartiles', - ' WHERE quartile = 1 or quartile = 3', - ' GROUP BY quartile', - ' ) __cdb_iqr', - ')' -].join('\n')); - -var binsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0', - ' THEN 1', - ' ELSE GREATEST(', - ' LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),', - ' LEAST(', - ' CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),', - ' {{=it._maxBins}}', - ' )', - ' )', - ' END AS __cdb_bins_number', - ' FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source', - ' LIMIT 1', - ')' -].join('\n')); - -var overrideBinsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT {{=it._bins}} AS __cdb_bins_number', - ')' -].join('\n')); - -var nullsQueryTpl = dot.template([ - '__cdb_nulls AS (', - ' SELECT', - ' count(*) AS __cdb_nulls_count', - ' FROM ({{=it._query}}) __cdb_histogram_nulls', - ' WHERE {{=it._column}} IS NULL', - ')' -].join('\n')); - -var infinitiesQueryTpl = dot.template([ - '__cdb_infinities AS (', - ' SELECT', - ' count(*) AS __cdb_infinities_count', - ' FROM ({{=it._query}}) __cdb_infinities_query', - ' WHERE', - ' {{=it._column}} = \'infinity\'::float', - ' OR', - ' {{=it._column}} = \'-infinity\'::float', - ')' -].join('\n')); - -var nansQueryTpl = dot.template([ - '__cdb_nans AS (', - ' SELECT', - ' count(*) AS __cdb_nans_count', - ' FROM ({{=it._query}}) __cdb_nans_query', - ' WHERE {{=it._column}} = \'NaN\'::float', - ')' -].join('\n')); - -var histogramQueryTpl = dot.template([ - 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', - ' __cdb_bins_number AS bins_number,', - ' __cdb_nulls_count AS nulls_count,', - ' {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,', - ' __cdb_nans_count AS nans_count,{{?}}', - ' __cdb_avg_val AS avg_val,', - ' CASE WHEN __cdb_min_val = __cdb_max_val', - ' THEN 0', - ' ELSE GREATEST(', - ' 1,', - ' LEAST(', - ' WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),', - ' __cdb_bins_number', - ' )', - ' ) - 1', - ' END AS bin,', - ' min({{=it._column}})::numeric AS min,', - ' max({{=it._column}})::numeric AS max,', - ' avg({{=it._column}})::numeric AS avg,', - ' count(*) AS freq', - 'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,', - ' __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}', - 'GROUP BY bin, bins_number, bin_width, nulls_count,', - ' avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}', - 'ORDER BY bin' -].join('\n')); - -var dateBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,', - ' min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', - ' min(date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )) AS __cdb_start_date,', - ' max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,', - ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) __cdb_basics_query', - ')' -].join(' \n')); - -var dateOverrideBasicsQueryTpl = dot.template([ - '__cdb_basics AS (', - ' SELECT', - ' max({{=it._end}})::float AS __cdb_max_val,', - ' min({{=it._start}})::float AS __cdb_min_val,', - ' avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,', - ' min(', - ' date_trunc(', - ' \'{{=it._aggregation}}\',', - ' TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' )', - ' ) AS __cdb_start_date,', - ' max(', - ' TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AS __cdb_end_date,', - ' count(1) AS __cdb_total_rows', - ' FROM ({{=it._query}}) __cdb_basics_query', - ')' -].join(' \n')); - -var dateBinsQueryTpl = dot.template([ - '__cdb_bins AS (', - ' SELECT', - ' __cdb_bins_array,', - ' ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number', - ' FROM (', - ' SELECT', - ' ARRAY(', - ' SELECT GENERATE_SERIES(', - ' __cdb_start_date::timestamptz,', - ' __cdb_end_date::timestamptz,', - ' {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval', - ' )', - ' ) AS __cdb_bins_array', - ' FROM __cdb_basics', - ' ) __cdb_bins_array_query', - ')' -].join('\n')); - -var dateHistogramQueryTpl = dot.template([ - 'SELECT', - ' (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,', - ' __cdb_bins_number AS bins_number,', - ' __cdb_nulls_count AS nulls_count,', - ' CASE WHEN __cdb_min_val = __cdb_max_val', - ' THEN 0', - ' ELSE GREATEST(1, LEAST(', - ' WIDTH_BUCKET(', - ' {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',', - ' __cdb_bins_array', - ' ),', - ' __cdb_bins_number', - ' )) - 1', - ' END AS bin,', - ' min(', - ' date_part(', - ' \'epoch\', ', - ' date_trunc(', - ' \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'', - ' ) AT TIME ZONE \'{{=it._offset}}\'', - ' )', - ' )::numeric AS timestamp,', - ' date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,', - ' min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,', - ' max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,', - ' avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,', - ' count(*) AS freq', - 'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls', - 'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL', - 'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start', - 'ORDER BY bin' -].join('\n')); - -var TYPE = 'histogram'; - -/** -Numeric histogram: -{ - type: 'histogram', - options: { - column: 'name', // column data type: numeric - bins: 10 // OPTIONAL - } -} - -Time series: -{ - type: 'histogram', - options: { - column: 'date', // column data type: date - aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric) - offset: -7200 // OPTIONAL (UTC offset in seconds) - } - } - */ -function Histogram(query, options, queries) { - if (!_.isString(options.column)) { - throw new Error('Histogram expects `column` in widget options'); + this.histogramImplementation = this._getHistogramImplementation(); } - this.query = query; - this.queries = queries; - this.column = options.column; - this.bins = options.bins; - this.aggregation = options.aggregation; - this.offset = options.offset; + _getHistogramImplementation (override) { + let implementation = null; - this._columnType = null; -} - -Histogram.prototype = new BaseWidget(); -Histogram.prototype.constructor = Histogram; - -module.exports = Histogram; - -Histogram.prototype.sql = function(psql, override, callback) { - var self = this; - - if (!callback) { - callback = override; - override = {}; - } - - if (this._columnType === null) { - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - // assume numeric, will fail later - self._columnType = 'numeric'; - if (!err && !!type) { - self._columnType = Object.keys(type).find(function (key) { - return type[key]; - }); - } - self.sql(psql, override, callback); - }, true); // use read-only transaction - return null; - } - - this._buildQuery(psql, override, callback); -}; - -Histogram.prototype.isDateHistogram = function (override) { - return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined); -}; - -Histogram.prototype._buildQuery = function (psql, override, callback) { - var filteredQuery, basicsQuery, binsQuery; - var _column = this.column; - var _query = this.query; - - if (this.isDateHistogram(override)) { - return this._buildDateHistogramQuery(psql, override, callback); - } - - if (this._columnType === 'date') { - _column = columnCastTpl({column: _column}); - } - - filteredQuery = filteredQueryTpl({ - _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column - }); - - if (this._shouldOverride(override)) { - debug('overriding with %j', override); - basicsQuery = overrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _start: getBinStart(override), - _end: getBinEnd(override) - }); - - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - basicsQuery = basicsQueryTpl({ - _query: _query, - _column: _column - }); - - if (this._shouldOverrideBins(override)) { - binsQuery = [ - overrideBinsQueryTpl({ - _bins: override.bins - }) - ].join(',\n'); - } else { - binsQuery = [ - iqrQueryTpl({ - _query: _query, - _column: _column - }), - binsQueryTpl({ - _query: _query, - _minBins: BIN_MIN_NUMBER, - _maxBins: BIN_MAX_NUMBER - }) - ].join(',\n'); - } - } - - var cteSql = [ - filteredQuery, - basicsQuery, - binsQuery, - nullsQueryTpl({ - _query: _query, - _column: _column - }) - ]; - - if (this._columnType === 'float') { - cteSql.push( - infinitiesQueryTpl({ - _query: _query, - _column: _column - }), - nansQueryTpl({ - _query: _query, - _column: _column - }) - ); - } - - var histogramSql = [ - "WITH", - cteSql.join(',\n'), - histogramQueryTpl({ - _isFloatColumn: this._columnType === 'float', - _query: _query, - _column: _column - }) - ].join('\n'); - - debug(histogramSql); - - return callback(null, histogramSql); -}; - -Histogram.prototype._shouldOverride = function (override) { - return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins'); -}; - -Histogram.prototype._shouldOverrideBins = function (override) { - return override && _.has(override, 'bins'); -}; - -var DATE_AGGREGATIONS = { - 'auto': true, - 'minute': true, - 'hour': true, - 'day': true, - 'week': true, - 'month': true, - 'quarter': true, - 'year': true -}; - -Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) { - var _column = this.column; - var _query = this.query; - var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation; - var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset; - - if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) { - return callback(new Error('Invalid aggregation value. Valid ones: ' + - Object.keys(DATE_AGGREGATIONS).join(', ') - )); - } - - if (_aggregation === 'auto') { - this.getAutomaticAggregation(psql, function (err, aggregation) { - if (err || aggregation === 'none') { - this.aggregation = 'day'; - } else { - this.aggregation = aggregation; - } - override.aggregation = this.aggregation; - this._buildDateHistogramQuery(psql, override, callback); - }.bind(this)); - return null; - } - - var dateBasicsQuery; - - if (override && _.has(override, 'start') && _.has(override, 'end')) { - dateBasicsQuery = dateOverrideBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _start: getBinStart(override), - _end: getBinEnd(override), - _offset: parseOffset(_offset, _aggregation) - }); - } else { - dateBasicsQuery = dateBasicsQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) - }); - } - - var dateBinsQuery = [ - dateBinsQueryTpl({ - _aggregation: _aggregation - }) - ].join(',\n'); - - var nullsQuery = nullsQueryTpl({ - _query: _query, - _column: _column - }); - - var dateHistogramQuery = dateHistogramQueryTpl({ - _query: _query, - _column: _column, - _aggregation: _aggregation, - _offset: parseOffset(_offset, _aggregation) - }); - - var histogramSql = [ - "WITH", - [ - dateBasicsQuery, - dateBinsQuery, - nullsQuery - ].join(',\n'), - dateHistogramQuery - ].join('\n'); - - debug(histogramSql); - - return callback(null, histogramSql); -}; - -Histogram.prototype.getAutomaticAggregation = function (psql, callback) { - var dateIntervalQuery = dateIntervalQueryTpl({ - query: this.query, - column: this.column - }); - - debug(dateIntervalQuery); - - psql.query(dateIntervalQuery, function (err, result) { - if (err) { - return callback(err); + switch (this._getHistogramSubtype(override)) { + case DATE_HISTOGRAM: + debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override); + implementation = new DateHistogram(this.query, this.options, this.queries); + break; + case NUMERIC_HISTOGRAM: + debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override); + implementation = new NumericHistogram(this.query, this.options, this.queries); + break; + default: + throw new Error('Unsupported Histogram type'); } - var aggegations = result.rows[0]; - var aggregation = Object.keys(aggegations) - .map(function (key) { - return { - name: key, - value: aggegations[key] - }; - }) - .reduce(function (closer, current) { - if (current.value > MAX_INTERVAL_VALUE) { - return closer; - } + return implementation; + } - var closerDiff = MAX_INTERVAL_VALUE - closer.value; - var currentDiff = MAX_INTERVAL_VALUE - current.value; - - if (Number.isFinite(current.value) && closerDiff > currentDiff) { - return current; - } - - return closer; - }, { name: 'none', value: -1 }); - - callback(null, aggregation.name); - }); -}; - -Histogram.prototype.format = function(result, override) { - override = override || {}; - var buckets = []; - - var binsCount = getBinsCount(override); - var width = getWidth(override); - var binsStart = getBinStart(override); - var nulls = 0; - var infinities = 0; - var nans = 0; - var avg; - var timestampStart; - var aggregation; - var offset; - - if (result.rows.length) { - var firstRow = result.rows[0]; - binsCount = firstRow.bins_number; - width = firstRow.bin_width || width; - avg = firstRow.avg_val; - nulls = firstRow.nulls_count; - timestampStart = firstRow.timestamp_start; - infinities = firstRow.infinities_count; - nans = firstRow.nans_count; - binsStart = populateBinStart(override, firstRow); - - if (Number.isFinite(timestampStart)) { - aggregation = getAggregation(override, this.aggregation); - offset = getOffset(override, this.offset); + _getHistogramSubtype (override) { + if(this._isDateHistogram(override)) { + return DATE_HISTOGRAM; } - buckets = result.rows.map(function(row) { - return _.omit( - row, - 'bins_number', - 'bin_width', - 'nulls_count', - 'infinities_count', - 'nans_count', - 'avg_val', - 'timestamp_start' - ); - }); + return NUMERIC_HISTOGRAM; } - return { - aggregation: aggregation, - offset: offset, - timestamp_start: timestampStart, - bin_width: width, - bins_count: binsCount, - bins_start: binsStart, - nulls: nulls, - infinities: infinities, - nans: nans, - avg: avg, - bins: buckets - }; -}; - -function getAggregation(override, aggregation) { - return override && override.aggregation ? override.aggregation : aggregation; -} - -function getOffset(override, offset) { - if (override && override.offset) { - return override.offset; - } - if (offset) { - return offset; - } - - return 0; -} - -function getBinStart(override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.min(override.start, override.end); - } - return override.start || 0; -} - -function getBinEnd(override) { - if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) { - return Math.max(override.start, override.end); - } - return override.end || 0; -} - -function getBinsCount(override) { - return override.bins || 0; -} - -function getWidth(override) { - var width = 0; - var binsCount = override.bins; - - if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { - width = (override.end - override.start) / binsCount; - } - - return width; -} - -function parseOffset(offset, aggregation) { - if (!offset) { - return '0'; - } - if (aggregation === 'hour' || aggregation === 'minute') { - return '0'; - } - - var offsetInHours = Math.ceil(offset / 3600); - return '' + offsetInHours; -} - -function populateBinStart(override, firstRow) { - var binStart; - - if (firstRow.hasOwnProperty('timestamp')) { - binStart = firstRow.timestamp; - } else if (override.hasOwnProperty('start')) { - binStart = getBinStart(override); - } else { - binStart = firstRow.min; - } - - return binStart; -} - -Histogram.prototype.getType = function() { - return TYPE; -}; - -Histogram.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _column: this.column, - _query: this.query - }); + _isDateHistogram (override = {}) { + return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation')); + } + + getResult (psql, override, callback) { + this.histogramImplementation = this._getHistogramImplementation(override); + this.histogramImplementation.getResult(psql, override, callback); + } + + // In order to keep previous behaviour with overviews, + // we have to expose the following methods to bypass + // the concrete overview implementation + + sql (psql, override, callback) { + this.histogramImplementation.sql(psql, override, callback); + } + + format (result, override) { + return this.histogramImplementation.format(result, override); + } + + getType () { + return this.histogramImplementation.getType(); + } + + toString () { + return this.histogramImplementation.toString(); + } }; diff --git a/lib/cartodb/models/dataview/histograms/base-histogram.js b/lib/cartodb/models/dataview/histograms/base-histogram.js new file mode 100644 index 00000000..ec251d09 --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/base-histogram.js @@ -0,0 +1,85 @@ +const BaseDataview = require('../base'); + +const TYPE = 'histogram'; + +module.exports = class BaseHistogram extends BaseDataview { + constructor (query, options, queries) { + super(); + + if (typeof options.column !== 'string') { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.queries = queries; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; + } + + sql (psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + if (this._columnType === null) { + this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => { + // assume numeric, will fail later + this._columnType = 'numeric'; + if (!err && !!type) { + this._columnType = Object.keys(type).find(function (key) { + return type[key]; + }); + } + this.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + return this._buildQuery(psql, override, callback); + } + + format (result, override) { + const histogram = this._getSummary(result, override); + histogram.bins = this._getBuckets(result); + return histogram; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); + } + + _hasOverridenRange (override) { + return override && override.hasOwnProperty('start') && override.hasOwnProperty('end'); + } + + _getBinStart (override = {}) { + if (this._hasOverridenRange(override)) { + return Math.min(override.start, override.end); + } + + return override.start || 0; + } + + _getBinEnd (override = {}) { + if (this._hasOverridenRange(override)) { + return Math.max(override.start, override.end); + } + + return override.end || 0; + } + + _getBinsCount (override = {}) { + return override.bins || 0; + } +}; diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js new file mode 100644 index 00000000..34a42f9a --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -0,0 +1,302 @@ +const BaseHistogram = require('./base-histogram'); +const debug = require('debug')('windshaft:dataview:date-histogram'); + +const dateIntervalQueryTpl = ctx => ` + WITH + __cdb_dates AS ( + SELECT + MAX(${ctx.column}::timestamp) AS __cdb_end, + MIN(${ctx.column}::timestamp) AS __cdb_start + FROM (${ctx.query}) __cdb_source + ), + __cdb_interval_in_days AS ( + SELECT + DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days + FROM __cdb_dates + ), + __cdb_interval_in_hours AS ( + SELECT + __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours + FROM __cdb_interval_in_days, __cdb_dates + ), + __cdb_interval_in_minutes AS ( + SELECT + __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes + FROM __cdb_interval_in_hours, __cdb_dates + ), + __cdb_interval_in_seconds AS ( + SELECT + __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds + FROM __cdb_interval_in_minutes, __cdb_dates + ) + SELECT + ROUND(__cdb_days / 365) AS year, + ROUND(__cdb_days / 90) AS quarter, + ROUND(__cdb_days / 30) AS month, + ROUND(__cdb_days / 7) AS week, + __cdb_days AS day, + __cdb_hours AS hour, + __cdb_minutes AS minute, + __cdb_seconds AS second + FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds +`; + +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${ctx.query}) __cdb_histogram_nulls + WHERE ${ctx.column} IS NULL + ) +`; + +const dateBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(date_part('epoch', ${ctx.column})) AS __cdb_max_val, + min(date_part('epoch', ${ctx.column})) AS __cdb_min_val, + avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' + ) + ) AS __cdb_start_date, + max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx.query}) __cdb_basics_query + ) +`; + +const dateOverrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx.end})::float AS __cdb_max_val, + min(${ctx.start})::float AS __cdb_min_val, + avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, + min( + date_trunc( + '${ctx.aggregation}', + TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}' + ) + ) AS __cdb_start_date, + max( + TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}' + ) AS __cdb_end_date, + count(1) AS __cdb_total_rows + FROM (${ctx.query}) __cdb_basics_query + ) +`; + +const dateBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + __cdb_bins_array, + ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number + FROM ( + SELECT + ARRAY( + SELECT GENERATE_SERIES( + __cdb_start_date::timestamptz, + __cdb_end_date::timestamptz, + ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`} + ) + ) AS __cdb_bins_array + FROM __cdb_basics + ) __cdb_bins_array_query + ) +`; + +const dateHistogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET( + ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}', + __cdb_bins_array + ), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min( + date_part( + 'epoch', + date_trunc( + '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' + ) AT TIME ZONE '${ctx.offset}' + ) + )::numeric AS timestamp, + date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, + min(date_part('epoch', ${ctx.column}))::numeric AS min, + max(date_part('epoch', ${ctx.column}))::numeric AS max, + avg(date_part('epoch', ${ctx.column}))::numeric AS avg, + count(*) AS freq + FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls + WHERE date_part('epoch', ${ctx.column}) IS NOT NULL + GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start + ORDER BY bin +`; + +const MAX_INTERVAL_VALUE = 366; + +const DATE_AGGREGATIONS = { + 'auto': true, + 'minute': true, + 'hour': true, + 'day': true, + 'week': true, + 'month': true, + 'quarter': true, + 'year': true +}; + +/** + date_histogram: { + type: 'histogram', + options: { + column: 'date', // column data type: date + aggregation: 'day' // MANDATORY + offset: -7200 // OPTIONAL (UTC offset in seconds) + } + } +*/ +module.exports = class DateHistogram extends BaseHistogram { + constructor (query, options, queries) { + super(query, options, queries); + + this.aggregation = options.aggregation; + this.offset = options.offset; + } + + _buildQueryTpl (ctx) { + return ` + WITH + ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, + ${dateBinsQueryTpl(ctx)}, + ${nullsQueryTpl(ctx)} + ${dateHistogramQueryTpl(ctx)} + `; + } + + _buildQuery (psql, override, callback) { + if (!this._isValidAggregation(override)) { + return callback(new Error('Invalid aggregation value. Valid ones: ' + + Object.keys(DATE_AGGREGATIONS).join(', ') + )); + } + + if (this._getAggregation(override) === 'auto') { + this._getAutomaticAggregation(psql, function (err, aggregation) { + if (err || aggregation === 'none') { + this.aggregation = 'day'; + } else { + this.aggregation = aggregation; + } + override.aggregation = this.aggregation; + this._buildQuery(psql, override, callback); + }.bind(this)); + return null; + } + + const histogramSql = this._buildQueryTpl({ + override: override, + query: this.query, + column: this.column, + aggregation: this._getAggregation(override), + start: this._getBinStart(override), + end: this._getBinEnd(override), + offset: this._parseOffset(override) + }); + + debug(histogramSql); + + return callback(null, histogramSql); + } + + _isValidAggregation (override) { + return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override)); + } + + _getAutomaticAggregation (psql, callback) { + const dateIntervalQuery = dateIntervalQueryTpl({ + query: this.query, + column: this.column + }); + + psql.query(dateIntervalQuery, function (err, result) { + if (err) { + return callback(err); + } + + const aggegations = result.rows[0]; + const aggregation = Object.keys(aggegations) + .map(key => ({ name: key, value: aggegations[key] })) + .reduce((closer, current) => { + if (current.value > MAX_INTERVAL_VALUE) { + return closer; + } + + const closerDiff = MAX_INTERVAL_VALUE - closer.value; + const currentDiff = MAX_INTERVAL_VALUE - current.value; + + if (Number.isFinite(current.value) && closerDiff > currentDiff) { + return current; + } + + return closer; + }, { name: 'none', value: -1 }); + + callback(null, aggregation.name); + }); + } + + _getSummary (result, override) { + const firstRow = result.rows[0] || {}; + + return { + aggregation: this._getAggregation(override), + offset: this._getOffset(override), + timestamp_start: firstRow.timestamp_start, + + bin_width: firstRow.bin_width, + bins_count: firstRow.bins_number, + bins_start: firstRow.timestamp, + nulls: firstRow.nulls_count, + infinities: firstRow.infinities_count, + nans: firstRow.nans_count, + avg: firstRow.avg_val + }; + } + + _getBuckets (result) { + return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp })); + } + + _getAggregation (override = {}) { + return override.aggregation ? override.aggregation : this.aggregation; + } + + _getOffset (override = {}) { + return Number.isFinite(override.offset) ? override.offset : (this.offset || 0); + } + + _parseOffset (override) { + if (this._shouldIgnoreOffset(override)) { + return '0'; + } + + const offsetInHours = Math.ceil(this._getOffset(override) / 3600); + + return '' + offsetInHours; + } + + _shouldIgnoreOffset (override) { + return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute'); + } +}; diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js new file mode 100644 index 00000000..ea191bea --- /dev/null +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -0,0 +1,234 @@ +const BaseHistogram = require('./base-histogram'); +const debug = require('debug')('windshaft:dataview:numeric-histogram'); + +const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; + +const filterOutSpecialNumericValues = ctx => ` + ${ctx.column} != 'infinity'::float + AND + ${ctx.column} != '-infinity'::float + AND + ${ctx.column} != 'NaN'::float +`; + +const filteredQueryTpl = ctx => ` + __cdb_filtered_source AS ( + SELECT * + FROM (${ctx.query}) __cdb_filtered_source_query + WHERE ${ctx.column} IS NOT NULL + ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} + ) +`; + +const basicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val, + avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; + +const overrideBasicsQueryTpl = ctx => ` + __cdb_basics AS ( + SELECT + max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val, + avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + FROM __cdb_filtered_source + ) +`; + +const iqrQueryTpl = ctx => ` + __cdb_iqrange AS ( + SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr + FROM ( + SELECT quartile, max(_cdb_iqr_column) AS quartile_max from ( + SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column} + ) AS quartile + FROM __cdb_filtered_source) _cdb_quartiles + WHERE quartile = 1 or quartile = 3 + GROUP BY quartile + ) __cdb_iqr + ) +`; + +const binsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT + CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 + THEN 1 + ELSE GREATEST( + LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)), + LEAST( + CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT), + ${ctx.maxBins} + ) + ) + END AS __cdb_bins_number + FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source + LIMIT 1 + ) +`; + +const overrideBinsQueryTpl = ctx => ` + __cdb_bins AS ( + SELECT ${ctx.override.bins} AS __cdb_bins_number + ) +`; + +const nullsQueryTpl = ctx => ` + __cdb_nulls AS ( + SELECT + count(*) AS __cdb_nulls_count + FROM (${ctx.query}) __cdb_histogram_nulls + WHERE ${ctx.column} IS NULL + ) +`; + +const infinitiesQueryTpl = ctx => ` + __cdb_infinities AS ( + SELECT + count(*) AS __cdb_infinities_count + FROM (${ctx.query}) __cdb_infinities_query + WHERE + ${ctx.column} = 'infinity'::float + OR + ${ctx.column} = '-infinity'::float + ) +`; + +const nansQueryTpl = ctx => ` + __cdb_nans AS ( + SELECT + count(*) AS __cdb_nans_count + FROM (${ctx.query}) __cdb_nans_query + WHERE ${ctx.column} = 'NaN'::float + ) +`; + +const specialNumericValuesColumnDefinitionTpl = () => ` + __cdb_infinities_count AS infinities_count, + __cdb_nans_count AS nans_count +`; + +const specialNumericValuesCTETpl = () => ` + __cdb_infinities, __cdb_nans +`; + +const specialNumericValuesColumnTpl = () => ` + infinities_count, nans_count +`; + +const histogramQueryTpl = ctx => ` + SELECT + (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, + __cdb_bins_number AS bins_number, + __cdb_nulls_count AS nulls_count, + ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} + __cdb_avg_val AS avg_val, + CASE WHEN __cdb_min_val = __cdb_max_val + THEN 0 + ELSE GREATEST( + 1, + LEAST( + WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), + __cdb_bins_number + ) + ) - 1 + END AS bin, + min(${ctx.column})::numeric AS min, + max(${ctx.column})::numeric AS max, + avg(${ctx.column})::numeric AS avg, + count(*) AS freq + FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins + ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} + GROUP BY bin, bins_number, bin_width, nulls_count, avg_val + ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} + ORDER BY bin +`; + +const BIN_MIN_NUMBER = 6; +const BIN_MAX_NUMBER = 48; + +/** +Numeric histogram: +{ + type: 'histogram', + options: { + column: 'name', // column data type: numeric + bins: 10 // OPTIONAL + } +} +*/ +module.exports = class NumericHistogram extends BaseHistogram { + constructor (query, options, queries) { + super(query, options, queries); + } + + _buildQuery (psql, override, callback) { + const histogramSql = this._buildQueryTpl({ + override: override, + column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, + isFloatColumn: this._columnType === 'float', + query: this.query, + start: this._getBinStart(override), + end: this._getBinEnd(override), + minBins: BIN_MIN_NUMBER, + maxBins: BIN_MAX_NUMBER, + }); + + debug(histogramSql); + + return callback(null, histogramSql); + } + + _buildQueryTpl (ctx) { + return ` + WITH + ${filteredQueryTpl(ctx)}, + ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, + ${this._hasOverridenBins(ctx.override) ? + overrideBinsQueryTpl(ctx) : + `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` + }, + ${nullsQueryTpl(ctx)} + ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} + ${histogramQueryTpl(ctx)} + `; + } + + _hasOverridenBins (override) { + return override && override.hasOwnProperty('bins'); + } + + _getSummary (result, override) { + const firstRow = result.rows[0] || {}; + + return { + bin_width: firstRow.bin_width, + bins_count: firstRow.bins_number, + bins_start: this._populateBinStart(firstRow, override), + nulls: firstRow.nulls_count, + infinities: firstRow.infinities_count, + nans: firstRow.nans_count, + avg: firstRow.avg_val, + }; + } + + _getBuckets (result) { + return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq })); + } + + _populateBinStart (firstRow, override = {}) { + let binStart; + + if (override.hasOwnProperty('start')) { + binStart = this._getBinStart(override); + } else { + binStart = firstRow.min; + } + + return binStart; + } + +}; diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js index 33993777..a54c660f 100644 --- a/lib/cartodb/models/dataview/list.js +++ b/lib/cartodb/models/dataview/list.js @@ -1,11 +1,9 @@ -var dot = require('dot'); -dot.templateSettings.strip = false; +const BaseDataview = require('./base'); +const debug = require('debug')('windshaft:dataview:list'); -var BaseWidget = require('./base'); +const TYPE = 'list'; -var TYPE = 'list'; - -var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); +const listSqlTpl = ctx => `select ${ctx.columns} from (${ctx.query}) as _cdb_list`; /** { @@ -15,52 +13,52 @@ var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as } } */ +module.exports = class List extends BaseDataview { + constructor (query, options = {}) { + super(); -function List(query, options) { - options = options || {}; + this._checkOptions(options); - if (!Array.isArray(options.columns)) { - throw new Error('List expects `columns` array in widget options'); + this.query = query; + this.columns = options.columns; } - BaseWidget.apply(this); - - this.query = query; - this.columns = options.columns; -} - -List.prototype = new BaseWidget(); -List.prototype.constructor = List; - -module.exports = List; - -List.prototype.sql = function(psql, override, callback) { - if (!callback) { - callback = override; + _checkOptions (options) { + if (!Array.isArray(options.columns)) { + throw new Error('List expects `columns` array in dataview options'); + } } - var listSql = listSqlTpl({ - _query: this.query, - _columns: this.columns.join(', ') - }); + sql (psql, override, callback) { + if (!callback) { + callback = override; + } - return callback(null, listSql); -}; + const listSql = listSqlTpl({ + query: this.query, + columns: this.columns.join(', ') + }); -List.prototype.format = function(result) { - return { - rows: result.rows - }; -}; + debug(listSql); -List.prototype.getType = function() { - return TYPE; -}; + return callback(null, listSql); + } -List.prototype.toString = function() { - return JSON.stringify({ - _type: TYPE, - _query: this.query, - _columns: this.columns.join(', ') - }); + format (result) { + return { + rows: result.rows + }; + } + + getType () { + return TYPE; + } + + toString () { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _columns: this.columns.join(', ') + }); + } }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index d9302276..350e52a0 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -4,6 +4,8 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var _ = require('underscore'); +var lzmaMiddleware = require('./middleware/lzma'); + var controller = require('./controllers'); var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); @@ -364,6 +366,8 @@ function bootstrap(opts) { next(); }); + app.use(lzmaMiddleware); + // temporary measure until we upgrade to newer version expressjs so we can check err.status app.use(function(err, req, res, next) { if (err) { diff --git a/package.json b/package.json index 12430490..fd857e96 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.12.10", + "version": "3.13.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -23,9 +23,9 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.58.1", + "camshaft": "0.59.1", "cartodb-psql": "0.10.1", - "cartodb-query-tables": "0.2.0", + "cartodb-query-tables": "0.3.0", "cartodb-redis": "0.14.0", "debug": "~2.2.0", "dot": "~1.0.2", @@ -38,11 +38,12 @@ "queue-async": "~1.0.7", "redis-mpool": "0.4.1", "request": "~2.79.0", + "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.19.2", + "turbo-carto": "0.20.0", "underscore": "~1.6.0", - "windshaft": "3.3.1", + "windshaft": "3.3.2", "yargs": "~5.0.0" }, "devDependencies": { @@ -52,7 +53,6 @@ "moment": "~2.18.1", "nock": "~2.11.0", "redis": "~0.12.1", - "semver": "~1.1.4", "strftime": "~0.8.2" }, "scripts": { @@ -62,6 +62,6 @@ }, "engines": { "node": ">=6.9", - "yarn": "^0.21.3" + "yarn": ">=0.27.5 <1.0.0" } } diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js index 83c2f535..2f087626 100644 --- a/test/acceptance/analysis/analysis-layers-use-cases.js +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client'); var dot = require('dot'); var debug = require('debug')('windshaft:cartodb:test'); -describe('analysis-layers use cases', function() { +describe('analysis-layers use cases', function () { - var multitypeStyleTemplate = dot.template([ - "#points['mapnik::geometry_type'=1] {", - " marker-fill-opacity: {{=it._opacity}};", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: {{=it._opacity}};", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: {{=it._color}};", - " marker-allow-overlap: true;", - "}", - "#lines['mapnik::geometry_type'=2] {", - " line-color: {{=it._color}};", - " line-width: 2;", - " line-opacity: {{=it._opacity}};", - "}", - "#polygons['mapnik::geometry_type'=3] {", - " polygon-fill: {{=it._color}};", - " polygon-opacity: {{=it._opacity}};", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: {{=it._opacity}};", - "}" - ].join('\n')); + var multitypeStyleTemplate = dot.template( + `#points['mapnik::geometry_type'=1] { + marker-fill-opacity: {{=it._opacity}}; + marker-line-color: #FFF; + marker-line-width: 0.5; + marker-line-opacity: {{=it._opacity}}; + marker-placement: point; + marker-type: ellipse; + marker-width: 8; + marker-fill: {{=it._color}}; + marker-allow-overlap: true; + } + #lines['mapnik::geometry_type'=2] { + line-color: {{=it._color}}; + line-width: 2; + line-opacity: {{=it._opacity}}; + } + #polygons['mapnik::geometry_type'=3] { + polygon-fill: {{=it._color}}; + polygon-opacity: {{=it._opacity}}; + line-color: #FFF; + line-width: 0.5; + line-opacity: {{=it._opacity}}; + }` + ); function cartocss(color, opacity) { @@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() { version: '1.5.0', layers: layers, dataviews: dataviews || {}, - analysis: analysis || [] + analyses: analysis || [] }; } - function analysisDef(analysis) { - return JSON.stringify(analysis); - } - var DEFAULT_MULTITYPE_STYLE = cartocss(); var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + var pointInPolygonDef = { + id: 'a1', + type: 'point-in-polygon', + params: { + points_source: { + type: 'source', + params: { + query: 'select * from analysis_rent_listings' + } + }, + polygons_source: { + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks' + } + }, + radius: 250 + } + } + } + }; + + var bufferDef = { + id: 'b1', + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks' + } + }, + radius: 250 + } + }; + var useCases = [ { desc: '1 mapnik layer', @@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } @@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_banks", + sql: 'select * from analysis_banks', cartocss: cartocss('#2167AB'), cartocss_version: '2.3.0' } @@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } @@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } }, { - type: 'analysis', + type: 'cartodb', options: { - def: analysisDef({ - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - }), - cartocss: cartocss('black', 0.5) + source: { + id: 'b1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } - ]) + ], + {}, + [ + bufferDef + ] + ) }, { @@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() { { type: 'cartodb', options: { - sql: "select * from analysis_rent_listings", + sql: 'select * from analysis_rent_listings', cartocss: DEFAULT_MULTITYPE_STYLE, cartocss_version: '2.3.0' } }, { - type: 'analysis', + type: 'cartodb', options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - } - } - }), - cartocss: cartocss('green', 1.0) + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } - ]) + ], + {}, + [ + pointInPolygonDef + ] + ) }, { desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 250 - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - }, - { - type: 'cartodb', - options: { - sql: "select * from analysis_rent_listings", - cartocss: DEFAULT_MULTITYPE_STYLE, - cartocss_version: '2.3.0' - } - } - ]) - }, - - { - desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'cartodb', - options: { - sql: "select * from analysis_rent_listings", - cartocss: DEFAULT_MULTITYPE_STYLE, - cartocss_version: '2.3.0' - } - }, - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 300 - } - }), - cartocss: cartocss('magenta', 0.5) - } - }, - { - type: 'analysis', - options: { - def: analysisDef({ - "type": "point-in-polygon", - "params": { - "pointsSource": { - "type": "source", - "params": { - "query": "select * from analysis_rent_listings" - } - }, - "polygonsSource": { - "type": "buffer", - "params": { - "source": { - "type": "source", - "params": { - "query": "select * from analysis_banks" - } - }, - "radius": 300 - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - } - ]) - }, - - { - skip: true, - desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', - mapConfig: mapConfig([ - { - type: 'cartodb', - options: { - "source": { id: "a" }, - "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "b1" }, - "cartocss": cartocss('green', 1.0), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "b2" }, - "cartocss": cartocss('magenta', 0.5), - "cartocss_version": "2.3.0" - } - } - ], - [ - { - id: "b2", - options: { - def: analysisDef({ - "type": "count-in-polygon", - "id": "a0", - "params": { - "columnName": 'count_airbnb', - "pointsSource": { - "type": "source", - "params": { - query: "select * from analysis_rent_listings" - }, - dataviews: { - price_histogram: { - type: 'histogram', - options: { - column: 'price' - } - } - } - }, - "polygonsSource": { - "id": "b1", - "type": "buffer", - "params": { - "source": { - "id": "b0", - "type": "source", - "params": { - query: "select * from analysis_banks" - } - }, - "radius": 250 - }, - dataviews: { - bank_category: { - type: 'aggregation', - options: { - column: 'bank' - } - } - } - } - }, - dataviews: { - count_histogram: { - type: 'histogram', - options: { - column: 'count_airbnb' - } - } - } - }), - cartocss: cartocss('green', 1.0) - } - } - ]) - }, - - { - skip: true, - desc: 'I. Distribution centers', mapConfig: mapConfig( - // layers [ { type: 'cartodb', options: { - "source": { id: "b0" }, - "cartocss": [ - "#distribution_centers {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 0.7;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: blue;", - " marker-allow-overlap: true;", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } }, { type: 'cartodb', options: { - "source": { id: "a0" }, - "cartocss": [ - "#shops {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 0.7;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "a1" }, - "cartocss": [ - "#routing {", - " line-color: ramp([routing_time], colorbrewer(Reds));", - " line-width: ramp([routing_time], 2, 8);", - " line-opacity: 1.0;", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + sql: 'select * from analysis_rent_listings', + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } ], - // dataviews - { - distribution_center_name_category: { - source: { id: 'b0' }, - type: 'aggregation', - options: { - column: 'name' - } - }, - time_histogram: { - source: { id: 'a1' }, - type: 'histogram', - options: { - column: 'routing_time' - } - }, - distance_histogram: { - source: { id: 'a1' }, - type: 'histogram', - options: { - column: 'routing_distance' - } - } - }, - // analysis + {}, [ - { - id: 'a1', - type: 'routing-n-to-n', - params: { - // distanceColumn: 'routing_distance', - // timeColumn: 'routing_time', - originSource: { - id: 'b0', - type: 'source', - params: { - query: 'select * from distribution_centers' - } - }, - destinationSource: { - id: 'a0', - type: 'source', - params: { - query: 'select * from shops' - } - } - } - } + pointInPolygonDef ] ) }, { - skip: true, - desc: 'II. Population analysis', + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', mapConfig: mapConfig( - // layers - [ - { - type: 'cartodb', - options: { - "source": { id: "a2" }, - "cartocss": [ - "#count_in_polygon {", - " polygon-opacity: 1.0", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: 0.7", - " polygon-fill: ramp([estimated_people], colorbrewer(Reds));", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" - } - }, - { - type: 'cartodb', - options: { - "source": { id: "a0" }, - "cartocss": DEFAULT_MULTITYPE_STYLE, - "cartocss_version": "2.3.0" - } - } - ], - // dataviews - { - total_population_formula: { - "source": { id: "a3" }, - type: 'formula', - options: { - column: 'total_population', - operation: 'sum' - } - }, - people_histogram: { // this injects a range filter at `a2` node output - "source": { id: "a2" }, - type: 'histogram', - options: { - column: 'estimated_people' - } - }, - subway_line_category: { // this injects a category filter at `a0` node output - "source": { id: "a0" }, - type: 'aggregation', - options: { - column: 'subway_line' - } - } - }, - // analysis - [ - { - id: 'a3', - // this will union the polygons, produce just one polygon, and calculate the total population for it - type: 'total-population', - params: { - columnName: 'total_population', - source: { - id: 'a2', - type: 'estimated-population', - params: { - columnName: 'estimated_people', - source: { - id: 'a1', - type: 'trade-area', - params: { - source: { - "id": "a0", - "type": "source", - "params": { - query: "select * from subway_stops" - } - }, - kind: 'walk', - time: 300 - } - } - } - } - } - } - ]) - }, - - { - skip: true, - desc: 'III. Point in polygon', - mapConfig: mapConfig( - // layers [ { type: 'cartodb', options: { - "source": { id: "a1" }, - "cartocss": [ - "#count_in_polygon {", - " polygon-opacity: 1.0", - " line-color: #FFF;", - " line-width: 0.5;", - " line-opacity: 0.7", - " polygon-fill: ramp([count_people], colorbrewer(Reds));", - "}" - ].join('\n'), - "cartocss_version": "2.3.0" + sql: 'select * from analysis_rent_listings', + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + source: { + id: 'a1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + source: { + id: 'b1' + }, + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' } } ], - // dataviews - { - age_histogram: { - "source": { id: "a0" }, - type: 'histogram', - options: { - column: 'age' - } - }, - income_histogram: { - "source": { id: "a0" }, - type: 'histogram', - options: { - column: 'income' - } - }, - gender_category: { - "source": { id: "a0" }, - type: 'aggregation', - options: { - column: 'gender' - } - } - }, - // analysis + {}, [ - { - "id": "a1", - "type": "count-in-polygon", - "params": { - "columnName": 'count_people', - "pointsSource": { - "id": 'a0', - "type": "source", - "params": { - query: "select the_geom, age, gender, income from people" - } - }, - "polygonsSource": { - "id": "b0", - "type": "source", - "params": { - query: "select * from postal_codes" - } - } - } - } + bufferDef, + pointInPolygonDef ] ) } - ]; - useCases.forEach(function(useCase, imageIdx) { + useCases.forEach(function (useCase) { if (!!useCase.skip) { - debug(JSON.stringify(useCase.mapConfig, null, 4)); + return debug(JSON.stringify(useCase.mapConfig, null, 4)); } - it.skip('should implement use case: "' + useCase.desc + '"', function(done) { + it(`should implement use case: '${useCase.desc}'`, function (done) { var testClient = new TestClient(useCase.mapConfig, 1234); var tile = useCase.tile || TILE_ANALYSIS_TABLES; - testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) { assert.ok(!err, err); - image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); assert.equal(image.width(), 256); diff --git a/test/acceptance/named_maps_static_view.js b/test/acceptance/named_maps_static_view.js index c7ddff2f..b7af4197 100644 --- a/test/acceptance/named_maps_static_view.js +++ b/test/acceptance/named_maps_static_view.js @@ -192,12 +192,36 @@ describe('named maps static view', function() { } getStaticMap({ zoom: 3 }, function(err, img) { assert.ok(!err); - img.save('/tmp/static.png'); assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done); }); }); }); + it('should return override bbox', function (done) { + var view = { + bounds: { + west: 0, + south: 0, + east: 45, + north: 45 + }, + zoom: 4, + center: { + lng: 40, + lat: 20 + } + }; + templateMaps.addTemplate(username, createTemplate(view), function (err) { + if (err) { + return done(err); + } + getStaticMap({ bbox: '0,45,90,45' }, function(err, img) { + assert.ok(!err); + assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done); + }); + }); + }); + it('should allow to select the layers to render', function (done) { var view = { bounds: { diff --git a/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png b/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png new file mode 100644 index 00000000..f7fee54c Binary files /dev/null and b/test/fixtures/previews/populated_places_simple_reduced-override-bbox.png differ diff --git a/test/unit/cartodb/lzmaMiddleware.test.js b/test/unit/cartodb/lzmaMiddleware.test.js new file mode 100644 index 00000000..9a41030a --- /dev/null +++ b/test/unit/cartodb/lzmaMiddleware.test.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var testHelper = require('../../support/test_helper'); + +var lzmaMiddleware = require('../../../lib/cartodb/middleware/lzma'); + +describe('lzma-middleware', function() { + + it('it should extend params with decoded lzma', function(done) { + var qo = { + config: { + version: '1.3.0' + } + }; + testHelper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { + var req = { + headers: { + host:'localhost' + }, + query: { + api_key: 'test', + lzma: data + } + }; + lzmaMiddleware(req, {}, function(err) { + if ( err ) { + return done(err); + } + var query = req.query; + assert.deepEqual(qo.config, query.config); + assert.equal('test', query.api_key); + done(); + }); + }); + }); + +}); diff --git a/test/unit/cartodb/req2params.test.js b/test/unit/cartodb/req2params.test.js index 6293199e..7b055e99 100644 --- a/test/unit/cartodb/req2params.test.js +++ b/test/unit/cartodb/req2params.test.js @@ -1,6 +1,6 @@ var assert = require('assert'); var _ = require('underscore'); -var test_helper = require('../../support/test_helper'); +require('../../support/test_helper'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); @@ -98,34 +98,31 @@ describe('req2params', function() { }); }); - it('it should extend params with decoded lzma', function(done) { - var qo = { - config: { - version: '1.3.0' + it('it should remove invalid params', function(done) { + var config = { + version: '1.3.0' + }; + var req = { + headers: { + host:'localhost' + }, + query: { + non_included: 'toberemoved', + api_key: 'test', + style: 'override', + config: config } }; - test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { - var req = { - headers: { - host:'localhost' - }, - query: { - non_included: 'toberemoved', - api_key: 'test', - style: 'override', - lzma: data - } - }; - baseController.req2params(prepareRequest(req), function(err, req) { - if ( err ) { - return done(err); - } - var query = req.params; - assert.deepEqual(qo.config, query.config); - assert.equal('test', query.api_key); - assert.equal(undefined, query.non_included); - done(); - }); + baseController.req2params(prepareRequest(req), function(err, req) { + if (err) { + return done(err); + } + var query = req.params; + assert.deepEqual(config, query.config); + assert.equal('test', query.api_key); + assert.equal(undefined, query.non_included); + assert.equal(undefined, query.style); + done(); }); }); diff --git a/yarn.lock b/yarn.lock index 8270d25a..62fc9315 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -28,6 +32,15 @@ ajv@^4.9.1: co "^4.6.0" json-stable-stringify "^1.0.1" +ajv@^5.1.0: + version "5.2.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.2.tgz#47c68d69e86f5d953103b0074a9430dc63da5e39" + dependencies: + co "^4.6.0" + fast-deep-equal "^1.0.0" + json-schema-traverse "^0.3.0" + json-stable-stringify "^1.0.1" + align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -48,13 +61,9 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" -ap@~0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/ap/-/ap-0.2.0.tgz#ae0942600b29912f0d2b14ec60c45e8f330b6110" - aproba@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1" + version "1.2.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" are-we-there-yet@~1.1.2: version "1.1.4" @@ -105,7 +114,11 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws4@^1.2.1: +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + +aws4@^1.2.1, aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -150,6 +163,18 @@ boom@2.x.x: dependencies: hoek "2.x.x" +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + dependencies: + hoek "4.x.x" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -194,9 +219,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.58.1: - version "0.58.1" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.58.1.tgz#e4156580683f624212ea3020e59790ad006f24cc" +camshaft@0.59.1: + version "0.59.1" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.1.tgz#a2e87fa4a0236655160cd1a6a9e966cbbf86dd43" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -253,9 +278,9 @@ cartodb-psql@0.10.1, cartodb-psql@^0.10.1: pg cartodb/node-postgres#6.1.6-cdb1 underscore "~1.6.0" -cartodb-query-tables@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" +cartodb-query-tables@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756" cartodb-redis@0.14.0: version "0.14.0" @@ -343,12 +368,16 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0, commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" +commander@^2.9.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -368,8 +397,8 @@ content-disposition@0.5.1: resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.1.tgz#87476c6a67c8daa87e32e87616df883ba7fb071b" content-type@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.2.tgz#b7d113aee7a8dd27bd21133c4dc2529df1721eed" + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" cookie-signature@1.0.6: version "1.0.6" @@ -389,6 +418,12 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" +cryptiles@3.x.x: + version "3.1.2" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" + dependencies: + boom "5.x.x" + d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -403,7 +438,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@^2.2.0, debug@~2.2.0: +debug@2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -421,6 +456,12 @@ debug@^1.0.4: dependencies: ms "2.0.0" +debug@^2.2.0: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + dependencies: + ms "2.0.0" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -491,7 +532,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -602,7 +647,7 @@ express@~4.13.3: utils-merge "1.0.0" vary "~1.0.1" -extend@~3.0.0: +extend@~3.0.0, extend@~3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -610,6 +655,10 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" +fast-deep-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" + fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -648,9 +697,17 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" +form-data@~2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.5" + mime-types "^2.1.12" + forwarded@~0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363" + version "0.1.2" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" fresh@0.3.0: version "0.3.0" @@ -691,11 +748,11 @@ gauge@~2.7.3: wide-align "^1.1.0" gdal@~0.9.2: - version "0.9.4" - resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.4.tgz#6dad4abb8ffb3e0d51150fb7935ad7c622c81818" + version "0.9.6" + resolved "https://registry.yarnpkg.com/gdal/-/gdal-0.9.6.tgz#0cf75d830d35847b4274368b10e04a925321a0ba" dependencies: - nan "~2.5.0" - node-pre-gyp "~0.6.27" + nan "~2.6.2" + node-pre-gyp "~0.6.36" generate-function@^2.0.0: version "2.0.0" @@ -733,7 +790,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -764,6 +821,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -804,6 +872,10 @@ har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + har-validator@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" @@ -820,6 +892,13 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" +har-validator@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" + dependencies: + ajv "^5.1.0" + har-schema "^2.0.0" + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -834,7 +913,7 @@ has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" -hawk@~3.1.3: +hawk@3.1.3, hawk@~3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" dependencies: @@ -843,6 +922,15 @@ hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" +hawk@~6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -854,6 +942,10 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" +hoek@4.x.x: + version "4.2.0" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" + hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -883,6 +975,14 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -935,8 +1035,8 @@ is-fullwidth-code-point@^1.0.0: number-is-nan "^1.0.0" is-my-json-valid@^2.12.4: - version "2.16.0" - resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.0.tgz#f079dd9bfdae65ee2038aae8acbc86ab109e3693" + version "2.16.1" + resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.16.1.tgz#5a846777e2c2620d1e69104e5d3a03b1f6088f11" dependencies: generate-function "^2.0.0" generate-object-property "^1.1.0" @@ -991,16 +1091,16 @@ istanbul@~0.4.3: wordwrap "^1.0.0" js-base64@^2.1.9: - version "2.1.9" - resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce" + version "2.3.2" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.3.2.tgz#a79a923666372b580f8e27f51845c6f7e8fbfbaf" js-string-escape@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" js-yaml@3.x, js-yaml@^3.4.6: - version "3.9.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.9.1.tgz#08775cebdfdd359209f0d2acd383c8f86a6904a0" + version "3.10.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" dependencies: argparse "^1.0.7" esprima "^4.0.0" @@ -1022,6 +1122,10 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" +json-schema-traverse@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" + json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1231,17 +1335,17 @@ millstone@0.6.17: underscore "~1.6.0" zipfile "~0.5.5" -mime-db@~1.29.0: - version "1.29.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.29.0.tgz#48d26d235589651704ac5916ca06001914266878" +mime-db@~1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: - version "2.1.16" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.16.tgz#2b858a52e5ecd516db897ac2be87487830698e23" +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, mime-types@~2.1.7: + version "2.1.17" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: - mime-db "~1.29.0" + mime-db "~1.30.0" -mime@1.3.4, mime@~1.3.4: +mime@1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1249,13 +1353,17 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" + +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1263,6 +1371,10 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1313,17 +1425,17 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" +nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" -nan@~2.5.0: - version "2.5.1" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nan@~2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45" ncp@~2.0.0: version "2.0.0" @@ -1344,15 +1456,16 @@ nock@~2.11.0: mkdirp "^0.5.0" propagate "0.3.x" -node-pre-gyp@~0.6.27, node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36: - version "0.6.36" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" +node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: + version "0.6.38" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.38.tgz#e92a20f83416415bb4086f6d1fb78b3da73d113d" dependencies: + hawk "3.1.3" mkdirp "^0.5.1" nopt "^4.0.1" npmlog "^4.0.2" rc "^1.1.7" - request "^2.81.0" + request "2.81.0" rimraf "^2.6.1" semver "^5.3.0" tar "^2.2.1" @@ -1397,7 +1510,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1: +oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1475,8 +1588,8 @@ parse-json@^2.2.0: error-ex "^1.2.0" parseurl@~1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56" + version "1.3.2" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3" path-exists@^2.0.0: version "2.1.0" @@ -1504,6 +1617,10 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1516,10 +1633,9 @@ pg-pool@1.*: object-assign "4.1.0" pg-types@1.*: - version "1.12.0" - resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.0.tgz#8ad3b7b897e3fd463e62de241ad5fc640b4a66f0" + version "1.12.1" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2" dependencies: - ap "~0.2.0" postgres-array "~1.0.0" postgres-bytea "~1.0.0" postgres-date "~1.0.0" @@ -1660,6 +1776,10 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +qs@~6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" @@ -1700,7 +1820,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1730,6 +1850,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1747,32 +1876,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: - version "2.79.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" - dependencies: - aws-sign2 "~0.6.0" - aws4 "^1.2.1" - caseless "~0.11.0" - combined-stream "~1.0.5" - extend "~3.0.0" - forever-agent "~0.6.1" - form-data "~2.1.1" - har-validator "~2.0.6" - hawk "~3.1.3" - http-signature "~1.1.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.7" - oauth-sign "~0.8.1" - qs "~6.3.0" - stringstream "~0.0.4" - tough-cookie "~2.3.0" - tunnel-agent "~0.4.1" - uuid "^3.0.0" - -request@^2.69.0, request@^2.81.0: +request@2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -1799,6 +1903,58 @@ request@^2.69.0, request@^2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +request@2.x, request@^2.55.0, request@^2.69.0: + version "2.82.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.82.0.tgz#2ba8a92cd7ac45660ea2b10a53ae67cd247516ea" + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.6.0" + caseless "~0.12.0" + combined-stream "~1.0.5" + extend "~3.0.1" + forever-agent "~0.6.1" + form-data "~2.3.1" + har-validator "~5.0.3" + hawk "~6.0.2" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.17" + oauth-sign "~0.8.2" + performance-now "^2.1.0" + qs "~6.5.1" + safe-buffer "^5.1.1" + stringstream "~0.0.5" + tough-cookie "~2.3.2" + tunnel-agent "^0.6.0" + uuid "^3.1.0" + +request@~2.79.0: + version "2.79.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" + dependencies: + aws-sign2 "~0.6.0" + aws4 "^1.2.1" + caseless "~0.11.0" + combined-stream "~1.0.5" + extend "~3.0.0" + forever-agent "~0.6.1" + form-data "~2.1.1" + har-validator "~2.0.6" + hawk "~3.1.3" + http-signature "~1.1.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.7" + oauth-sign "~0.8.1" + qs "~6.3.0" + stringstream "~0.0.4" + tough-cookie "~2.3.0" + tunnel-agent "~0.4.1" + uuid "^3.0.0" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -1818,8 +1974,8 @@ right-align@^0.1.1: align-text "^0.1.1" rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" + version "2.6.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" dependencies: glob "^7.0.5" @@ -1829,7 +1985,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" @@ -1845,10 +2001,6 @@ semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" -semver@~1.1.4: - version "1.1.4" - resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540" - semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" @@ -1857,6 +2009,10 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -1921,6 +2077,12 @@ sntp@1.x.x: dependencies: hoek "2.x.x" +sntp@2.x.x: + version "2.0.2" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" + dependencies: + hoek "4.x.x" + source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -1928,8 +2090,8 @@ source-map@^0.4.4: amdefine ">=0.0.4" source-map@^0.5.1, source-map@^0.5.6, source-map@~0.5.1: - version "0.5.6" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412" + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" source-map@~0.2.0: version "0.2.0" @@ -1955,10 +2117,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -1970,11 +2136,11 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" "sqlite3@2.x || 3.x": - version "3.1.9" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.9.tgz#b2e7fbaa348380318d3834323918c3c351b8bf18" + version "3.1.12" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.12.tgz#2b3a14b17162e39e8aa6e1e2487a41d0795396d8" dependencies: - nan "~2.6.2" - node-pre-gyp "~0.6.36" + nan "~2.7.0" + node-pre-gyp "~0.6.38" srs@1.x: version "1.2.0" @@ -2036,7 +2202,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4: +stringstream@~0.0.4, stringstream@~0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2141,9 +2307,9 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0: - version "2.3.2" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.2.tgz#f081f76e4c85720e6c37a5faced737150d84072a" +tough-cookie@~2.3.0, tough-cookie@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: punycode "^1.4.1" @@ -2157,9 +2323,9 @@ tunnel-agent@~0.4.1: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" -turbo-carto@0.19.2: - version "0.19.2" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.19.2.tgz#062d68e59f89377f0cfa69a2717c047fe95e32fd" +turbo-carto@0.20.0: + version "0.20.0" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.0.tgz#ebeb4620f3e26ed3cce53a9ff30f087f46a2e3fd" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" @@ -2234,7 +2400,7 @@ utils-merge@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" -uuid@^3.0.0: +uuid@^3.0.0, uuid@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" @@ -2281,9 +2447,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.1.tgz#b5557fa6b0cfa13920904f57206b86f7aa054f74" +windshaft@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.2.tgz#72efe0dbc0d8d4bcba4211fdabd15dd2e0799df9" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2