cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Configurable QueryTablesAPI to call directly postgresql using cartodb-psql

Browse Source 
or to keep using a request to the SQL API
This commit is contained in:
Raul Ochoa
2014-08-06 21:48:08 +02:00
parent d1373bec66
commit 5cf79c82bb
7 changed files with 123 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
lib/cartodb/api/query_tables_api.js
View File

@@ -1,4 +1,5 @@
var sqlApi = require('../sql/sql_api');
var sqlApi = require('../sql/sql_api'),
PSQL = require('cartodb-psql');
function QueryTablesApi() {
}
@@ -33,7 +34,7 @@ QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, table
});
};
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql, callback) {
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
// Replace mapnik tokens
sql = sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
@@ -45,20 +46,32 @@ QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key,
sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
sqlApi.query(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
callback(null, tableNames);
});
if (shouldQueryPostgresDirectly()) {
var psql = new PSQL(options);
psql.query(sql, function(err, resultSet) {
var rows = resultSet.rows || [];
handleAffectedTablesInQueryRows(err, rows, callback);
});
} else {
sqlApi.query(username, options.api_key, sql, function(err, rows) {
handleAffectedTablesInQueryRows(err, rows, callback);
});
}
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, api_key, sql, callback) {
function handleAffectedTablesInQueryRows(err, rows, callback) {
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
callback(null, tableNames);
}
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
sql = sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.pixel_width, '1')
@@ -72,23 +85,42 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
sqlApi.query(username, api_key, query, function(err, rows){
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg));
return;
}
var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
var lastUpdatedTime = result.max || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
if (shouldQueryPostgresDirectly()) {
var psql = new PSQL(options);
psql.query(query, function(err, resultSet) {
var rows = resultSet.rows || [];
handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback);
});
});
} else {
sqlApi.query(username, options.api_key, query, function(err, rows) {
handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback);
});
}
};
function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg));
return;
}
var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
var lastUpdatedTime = result.max || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
});
}
function shouldQueryPostgresDirectly() {
return global.environment
&& global.environment.enabledFeatures
&& global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
}

18
lib/cartodb/server_options.js
View File

@@ -191,7 +191,14 @@ module.exports = function(){
if ( req.profiler ) req.profiler.done('getSignerMapKey');
key = data;
}
queryTablesApi.getAffectedTablesInQuery(user, key, sql, this); // in addCacheChannel
queryTablesApi.getAffectedTablesInQuery(user, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this); // in addCacheChannel
},
function finish(err, data) {
next(err,data);
@@ -316,7 +323,14 @@ module.exports = function(){
Step(
function getAffectedTablesAndLastUpdatedTime() {
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, key, sql, this);
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');

23
npm-shrinkwrap.json generated
View File

@@ -937,6 +937,29 @@
"version": "0.5.0",
"from": "git://github.com/CartoDB/node-cartodb-redis.git#0.5.0"
},
"cartodb-psql": {
"version": "0.2.0",
"from": "git://github.com/CartoDB/node-cartodb-psql.git#0.2.0",
"dependencies": {
"pg": {
"version": "2.6.2",
"dependencies": {
"generic-pool": {
"version": "2.0.3"
},
"buffer-writer": {
"version": "1.0.0"
}
}
},
"step": {
"version": "0.0.5"
},
"underscore": {
"version": "1.6.0"
}
}
},
"redis-mpool": {
"version": "0.0.4",
"from": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",

1
package.json
View File

@@ -28,6 +28,7 @@
"step": "0.0.x",
"request": "2.9.202",
"cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.5.0",
"cartodb-psql": "git://github.com/CartoDB/node-cartodb-psql.git#0.2.0",
"redis-mpool": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
"mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1",
"lzma": "~1.2.3",

7
test/acceptance/multilayer.js
View File

@@ -23,7 +23,11 @@ serverOptions = ServerOptions();
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('multilayer', function() {
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@@ -1342,3 +1346,4 @@ suite('multilayer', function() {
});
});

8
test/acceptance/server.js
View File

@@ -19,7 +19,12 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('server', function() {
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@@ -1389,3 +1394,4 @@ suite('server', function() {
});
});

7
test/acceptance/templates.js
View File

@@ -25,7 +25,11 @@ var serverOptions = ServerOptions();
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('template_api', function() {
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@@ -1948,3 +1952,4 @@ suite('template_api', function() {
});
});