diff --git a/NEWS.md b/NEWS.md index ae1749ec..f5070f22 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ Released 2016-mm-dd +Enhancements: + - Dataviews use mapconfig to store/retrieve their queries instead of instantiating analyses again. + ## 2.61.0 diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 15a56e1d..0be51713 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -2,11 +2,8 @@ var assert = require('assert'); var _ = require('underscore'); var PSQL = require('cartodb-psql'); -var camshaft = require('camshaft'); var step = require('step'); -var Timer = require('../stats/timer'); - var BBoxFilter = require('../models/filter/bbox'); var DataviewFactory = require('../models/dataview/factory'); @@ -26,110 +23,36 @@ function DataviewBackend(analysisBackend) { module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { - var self = this; - - var timer = new Timer(); var dataviewName = params.dataviewName; - - var mapConfig; - var dataviewDefinition; step( function getMapConfig() { mapConfigProvider.getMapConfig(this); }, - function _getDataviewDefinition(err, _mapConfig) { + function runDataviewQuery(err, mapConfig) { assert.ifError(err); - mapConfig = _mapConfig; - - var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); - if (!_dataviewDefinition) { + var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!dataviewDefinition) { throw new Error("Dataview '" + dataviewName + "' does not exists"); } - dataviewDefinition = _dataviewDefinition; - - return dataviewDefinition; - }, - function loadAnalysis(err) { - assert.ifError(err); - - var analysisConfiguration = { - user: user, - db: { - host: params.dbhost, - port: params.dbport, - dbname: params.dbname, - user: params.dbuser, - pass: params.dbpassword - }, - batch: { - username: user, - apiKey: params.api_key - } - }; - - var sourceId = dataviewDefinition.source.id; - var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId); - - var next = this; - - self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) { - if (err) { - return next(err); - } - - var sourceId2Node = {}; - var rootNode = analysis.getRoot(); - if (rootNode.params && rootNode.params.id) { - sourceId2Node[rootNode.params.id] = rootNode; - } - - analysis.getNodes().forEach(function(node) { - if (node.params && node.params.id) { - sourceId2Node[node.params.id] = node; - } - }); - - var node = sourceId2Node[sourceId]; - - if (!node) { - return next(new Error('Analysis node not found for dataview')); - } - - return next(null, node); - }); - }, - function runDataviewQuery(err, node) { - assert.ifError(err); - var pg = new PSQL(dbParamsFromReqParams(params)); var ownFilter = +params.own_filter; ownFilter = !!ownFilter; - var query = layerQuery(node, dataviewName, ownFilter); - + var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; var sourceId = dataviewDefinition.source.id; // node.id - var layer = _.find( - mapConfig.obj().layers, - function(l){ return l.options.source && (l.options.source.id === sourceId); } - ); + var layer = _.find(mapConfig.obj().layers, function(l) { + return l.options.source && (l.options.source.id === sourceId); + }); var queryRewriteData = layer && layer.options.query_rewrite_data; - if ( queryRewriteData ) { - if ( node.type === 'source' ) { - var filters = node.getFilters(); - var filters_disabler = Object.keys(filters).reduce( - function(disabler, filter_id){ disabler[filter_id] = false; return disabler; }, - {} - ); - var unfiltered_query = node.getQuery(filters_disabler); - queryRewriteData = _.extend( - {}, - queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query } - ); - } + if (queryRewriteData && dataviewDefinition.node.type === 'source') { + queryRewriteData = _.extend({}, queryRewriteData, { + filters: dataviewDefinition.node.filters, + unfiltered_query: dataviewDefinition.sql.own_filter_on + }); } if (params.bbox) { @@ -166,98 +89,32 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param dataview.getResult(pg, overrideParams, this); }, function returnCallback(err, result) { - return callback(err, result, timer.getTimes()); + return callback(err, result); } ); }; DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { - var self = this; - - var timer = new Timer(); - var dataviewName = params.dataviewName; - var mapConfig; - var dataviewDefinition; step( function getMapConfig() { mapConfigProvider.getMapConfig(this); }, - function _getDataviewDefinition(err, _mapConfig) { + function runDataviewSearchQuery(err, mapConfig) { assert.ifError(err); - mapConfig = _mapConfig; - - var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); - if (!_dataviewDefinition) { + var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!dataviewDefinition) { throw new Error("Dataview '" + dataviewName + "' does not exists"); } - dataviewDefinition = _dataviewDefinition; - - return dataviewDefinition; - }, - function loadAnalysis(err) { - assert.ifError(err); - - var analysisConfiguration = { - user: user, - db: { - host: params.dbhost, - port: params.dbport, - dbname: params.dbname, - user: params.dbuser, - pass: params.dbpassword - }, - batch: { - // TODO load this from configuration - endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', - username: user, - apiKey: params.api_key - } - }; - - var sourceId = dataviewDefinition.source.id; - var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId); - - var next = this; - - self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) { - if (err) { - return next(err); - } - - var sourceId2Node = {}; - var rootNode = analysis.getRoot(); - if (rootNode.params && rootNode.params.id) { - sourceId2Node[rootNode.params.id] = rootNode; - } - - analysis.getNodes().forEach(function(node) { - if (node.params && node.params.id) { - sourceId2Node[node.params.id] = node; - } - }); - - var node = sourceId2Node[sourceId]; - - if (!node) { - return next(new Error('Analysis node not found for dataview')); - } - - return next(null, node); - }); - }, - function runDataviewQuery(err, node) { - assert.ifError(err); - var pg = new PSQL(dbParamsFromReqParams(params)); var ownFilter = +params.own_filter; ownFilter = !!ownFilter; - var query = layerQuery(node, dataviewName, ownFilter); + var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off; if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); @@ -270,23 +127,11 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca dataview.search(pg, userQuery, this); }, function returnCallback(err, result) { - return callback(err, result, timer.getTimes()); + return callback(err, result); } ); }; -function getAnalysisDefinition(mapConfigAnalyses, sourceId) { - mapConfigAnalyses = mapConfigAnalyses || []; - for (var i = 0; i < mapConfigAnalyses.length; i++) { - var analysisGraph = new camshaft.reference.AnalysisGraph(mapConfigAnalyses[i]); - var nodes = analysisGraph.getNodesWithId(); - if (nodes.hasOwnProperty(sourceId)) { - return mapConfigAnalyses[i]; - } - } - throw new Error('There is no associated analysis for the dataview source id'); -} - function getDataviewDefinition(mapConfig, dataviewName) { var dataviews = mapConfig.dataviews || {}; return dataviews[dataviewName]; @@ -311,31 +156,3 @@ function dbParamsFromReqParams(params) { } return dbParams; } - -var SKIP_COLUMNS = { - 'the_geom': true, - 'the_geom_webmercator': true -}; - -function skipColumns(columnNames) { - return columnNames - .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); -} - -var layerQueryTemplate = dot.template([ - 'SELECT {{=it._columns}}', - 'FROM ({{=it._query}}) _cdb_analysis_query' -].join('\n')); - -function layerQuery(node, dataviewName, ownFilter) { - var applyFilters = {}; - if (!ownFilter) { - applyFilters[dataviewName] = false; - } - - if (node.type === 'source') { - return node.getQuery(applyFilters); - } - var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); - return layerQueryTemplate({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') }); -} diff --git a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index f69c22db..6040631d 100644 --- a/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -123,14 +123,37 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi return layer; }); - - debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); - var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node); if (missingNodesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) { return callback(missingNodesErrors.concat(missingDataviewsNodesErrors)); } + // Augment dataviews with sql from analyses + Object.keys(dataviews).forEach(function(dataviewName) { + var dataview = requestMapConfig.dataviews[dataviewName]; + var dataviewSourceId = dataview.source.id; + var dataviewNode = sourceId2Node[dataviewSourceId]; + dataview.node = { + type: dataviewNode.type, + filters: dataviewNode.getFilters() + }; + dataview.sql = { + own_filter_on: dataviewQuery(dataviewNode, dataviewName, true), + own_filter_off: dataviewQuery(dataviewNode, dataviewName, false), + no_filters: dataviewNode.getQuery(Object.keys(dataviewNode.getFilters()) + .reduce(function(applyFilters, filterId) { + applyFilters[filterId] = false; + return applyFilters; + }, {}) + ) + }; + }); + if (Object.keys(dataviews).length > 0) { + requestMapConfig.dataviews = dataviews; + } + + debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); + context.analysesResults = analysesResults; return callback(null, requestMapConfig); @@ -147,7 +170,7 @@ function skipColumns(columnNames) { .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); } -var layerQueryTemplate = dot.template([ +var wrappedQueryTpl = dot.template([ 'SELECT {{=it._columns}}', 'FROM ({{=it._query}}) _cdb_analysis_query' ].join('\n')); @@ -157,7 +180,20 @@ function layerQuery(node) { return node.getQuery(); } var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); - return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') }); + return wrappedQueryTpl({ _query: node.getQuery(), _columns: _columns.join(', ') }); +} + +function dataviewQuery(node, dataviewName, ownFilter) { + var applyFilters = {}; + if (!ownFilter) { + applyFilters[dataviewName] = false; + } + + if (node.type === 'source') { + return node.getQuery(applyFilters); + } + var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); + return wrappedQueryTpl({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') }); } function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {