cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use mapconfig to to store/retrieve dataviews queries

Browse Source
This commit is contained in:
Raul Ochoa
2016-07-06 22:30:24 +02:00
parent c539d4fbbd
commit 57bb8dbbe3
3 changed files with 62 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
NEWS.md
View File

@@ -4,6 +4,9 @@
Released 2016-mm-dd
Enhancements:
- Dataviews use mapconfig to store/retrieve their queries instead of instantiating analyses again.
## 2.61.0

219
lib/cartodb/backends/dataview.js
View File

@@ -2,11 +2,8 @@ var assert = require('assert');
var _ = require('underscore');
var PSQL = require('cartodb-psql');
var camshaft = require('camshaft');
var step = require('step');
var Timer = require('../stats/timer');
var BBoxFilter = require('../models/filter/bbox');
var DataviewFactory = require('../models/dataview/factory');
@@ -26,110 +23,36 @@ function DataviewBackend(analysisBackend) {
module.exports = DataviewBackend;
DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
var self = this;
var timer = new Timer();
var dataviewName = params.dataviewName;
var mapConfig;
var dataviewDefinition;
step(
function getMapConfig() {
mapConfigProvider.getMapConfig(this);
},
function _getDataviewDefinition(err, _mapConfig) {
function runDataviewQuery(err, mapConfig) {
assert.ifError(err);
mapConfig = _mapConfig;
var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
if (!_dataviewDefinition) {
var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
if (!dataviewDefinition) {
throw new Error("Dataview '" + dataviewName + "' does not exists");
}
dataviewDefinition = _dataviewDefinition;
return dataviewDefinition;
},
function loadAnalysis(err) {
assert.ifError(err);
var analysisConfiguration = {
user: user,
db: {
host: params.dbhost,
port: params.dbport,
dbname: params.dbname,
user: params.dbuser,
pass: params.dbpassword
},
batch: {
username: user,
apiKey: params.api_key
}
};
var sourceId = dataviewDefinition.source.id;
var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId);
var next = this;
self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) {
if (err) {
return next(err);
}
var sourceId2Node = {};
var rootNode = analysis.getRoot();
if (rootNode.params && rootNode.params.id) {
sourceId2Node[rootNode.params.id] = rootNode;
}
analysis.getNodes().forEach(function(node) {
if (node.params && node.params.id) {
sourceId2Node[node.params.id] = node;
}
});
var node = sourceId2Node[sourceId];
if (!node) {
return next(new Error('Analysis node not found for dataview'));
}
return next(null, node);
});
},
function runDataviewQuery(err, node) {
assert.ifError(err);
var pg = new PSQL(dbParamsFromReqParams(params));
var ownFilter = +params.own_filter;
ownFilter = !!ownFilter;
var query = layerQuery(node, dataviewName, ownFilter);
var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(
mapConfig.obj().layers,
function(l){ return l.options.source && (l.options.source.id === sourceId); }
);
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if ( queryRewriteData ) {
if ( node.type === 'source' ) {
var filters = node.getFilters();
var filters_disabler = Object.keys(filters).reduce(
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
{}
);
var unfiltered_query = node.getQuery(filters_disabler);
queryRewriteData = _.extend(
{},
queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query }
);
}
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox) {
@@ -166,98 +89,32 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
dataview.getResult(pg, overrideParams, this);
},
function returnCallback(err, result) {
return callback(err, result, timer.getTimes());
return callback(err, result);
}
);
};
DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
var self = this;
var timer = new Timer();
var dataviewName = params.dataviewName;
var mapConfig;
var dataviewDefinition;
step(
function getMapConfig() {
mapConfigProvider.getMapConfig(this);
},
function _getDataviewDefinition(err, _mapConfig) {
function runDataviewSearchQuery(err, mapConfig) {
assert.ifError(err);
mapConfig = _mapConfig;
var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
if (!_dataviewDefinition) {
var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
if (!dataviewDefinition) {
throw new Error("Dataview '" + dataviewName + "' does not exists");
}
dataviewDefinition = _dataviewDefinition;
return dataviewDefinition;
},
function loadAnalysis(err) {
assert.ifError(err);
var analysisConfiguration = {
user: user,
db: {
host: params.dbhost,
port: params.dbport,
dbname: params.dbname,
user: params.dbuser,
pass: params.dbpassword
},
batch: {
// TODO load this from configuration
endpoint: 'http://127.0.0.1:8080/api/v1/sql/job',
username: user,
apiKey: params.api_key
}
};
var sourceId = dataviewDefinition.source.id;
var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId);
var next = this;
self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) {
if (err) {
return next(err);
}
var sourceId2Node = {};
var rootNode = analysis.getRoot();
if (rootNode.params && rootNode.params.id) {
sourceId2Node[rootNode.params.id] = rootNode;
}
analysis.getNodes().forEach(function(node) {
if (node.params && node.params.id) {
sourceId2Node[node.params.id] = node;
}
});
var node = sourceId2Node[sourceId];
if (!node) {
return next(new Error('Analysis node not found for dataview'));
}
return next(null, node);
});
},
function runDataviewQuery(err, node) {
assert.ifError(err);
var pg = new PSQL(dbParamsFromReqParams(params));
var ownFilter = +params.own_filter;
ownFilter = !!ownFilter;
var query = layerQuery(node, dataviewName, ownFilter);
var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
@@ -270,23 +127,11 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca
dataview.search(pg, userQuery, this);
},
function returnCallback(err, result) {
return callback(err, result, timer.getTimes());
return callback(err, result);
}
);
};
function getAnalysisDefinition(mapConfigAnalyses, sourceId) {
mapConfigAnalyses = mapConfigAnalyses || [];
for (var i = 0; i < mapConfigAnalyses.length; i++) {
var analysisGraph = new camshaft.reference.AnalysisGraph(mapConfigAnalyses[i]);
var nodes = analysisGraph.getNodesWithId();
if (nodes.hasOwnProperty(sourceId)) {
return mapConfigAnalyses[i];
}
}
throw new Error('There is no associated analysis for the dataview source id');
}
function getDataviewDefinition(mapConfig, dataviewName) {
var dataviews = mapConfig.dataviews || {};
return dataviews[dataviewName];
@@ -311,31 +156,3 @@ function dbParamsFromReqParams(params) {
}
return dbParams;
}
var SKIP_COLUMNS = {
'the_geom': true,
'the_geom_webmercator': true
};
function skipColumns(columnNames) {
return columnNames
.filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
}
var layerQueryTemplate = dot.template([
'SELECT {{=it._columns}}',
'FROM ({{=it._query}}) _cdb_analysis_query'
].join('\n'));
function layerQuery(node, dataviewName, ownFilter) {
var applyFilters = {};
if (!ownFilter) {
applyFilters[dataviewName] = false;
}
if (node.type === 'source') {
return node.getQuery(applyFilters);
}
var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
return layerQueryTemplate({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') });
}

46
lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js
View File

@@ -123,14 +123,37 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
return layer;
});
debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node);
if (missingNodesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) {
return callback(missingNodesErrors.concat(missingDataviewsNodesErrors));
}
// Augment dataviews with sql from analyses
Object.keys(dataviews).forEach(function(dataviewName) {
var dataview = requestMapConfig.dataviews[dataviewName];
var dataviewSourceId = dataview.source.id;
var dataviewNode = sourceId2Node[dataviewSourceId];
dataview.node = {
type: dataviewNode.type,
filters: dataviewNode.getFilters()
};
dataview.sql = {
own_filter_on: dataviewQuery(dataviewNode, dataviewName, true),
own_filter_off: dataviewQuery(dataviewNode, dataviewName, false),
no_filters: dataviewNode.getQuery(Object.keys(dataviewNode.getFilters())
.reduce(function(applyFilters, filterId) {
applyFilters[filterId] = false;
return applyFilters;
}, {})
)
};
});
if (Object.keys(dataviews).length > 0) {
requestMapConfig.dataviews = dataviews;
}
debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
context.analysesResults = analysesResults;
return callback(null, requestMapConfig);
@@ -147,7 +170,7 @@ function skipColumns(columnNames) {
.filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
}
var layerQueryTemplate = dot.template([
var wrappedQueryTpl = dot.template([
'SELECT {{=it._columns}}',
'FROM ({{=it._query}}) _cdb_analysis_query'
].join('\n'));
@@ -157,7 +180,20 @@ function layerQuery(node) {
return node.getQuery();
}
var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') });
return wrappedQueryTpl({ _query: node.getQuery(), _columns: _columns.join(', ') });
}
function dataviewQuery(node, dataviewName, ownFilter) {
var applyFilters = {};
if (!ownFilter) {
applyFilters[dataviewName] = false;
}
if (node.type === 'source') {
return node.getQuery(applyFilters);
}
var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
return wrappedQueryTpl({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') });
}
function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {