diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js index 52e3a989..61ef8d92 100644 --- a/lib/cartodb/api/query_tables_api.js +++ b/lib/cartodb/api/query_tables_api.js @@ -1,4 +1,5 @@ -var sqlApi = require('../sql/sql_api'); +var sqlApi = require('../sql/sql_api'), + PSQL = require('cartodb-psql'); function QueryTablesApi() { } @@ -33,7 +34,7 @@ QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, table }); }; -QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql, callback) { +QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) { // Replace mapnik tokens sql = sql .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') @@ -45,20 +46,32 @@ QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; // call sql api - sqlApi.query(username, api_key, sql, function(err, rows){ - if (err){ - var msg = err.message ? err.message : err; - callback(new Error('could not fetch source tables: ' + msg)); - return; - } - var qtables = rows[0].cdb_querytables; - var tableNames = qtables.split(/^\{(.*)\}$/)[1]; - tableNames = tableNames ? tableNames.split(',') : []; - callback(null, tableNames); - }); + if (shouldQueryPostgresDirectly()) { + var psql = new PSQL(options); + psql.query(sql, function(err, resultSet) { + var rows = resultSet.rows || []; + handleAffectedTablesInQueryRows(err, rows, callback); + }); + } else { + sqlApi.query(username, options.api_key, sql, function(err, rows) { + handleAffectedTablesInQueryRows(err, rows, callback); + }); + } }; -QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, api_key, sql, callback) { +function handleAffectedTablesInQueryRows(err, rows, callback) { + if (err){ + var msg = err.message ? err.message : err; + callback(new Error('could not fetch source tables: ' + msg)); + return; + } + var qtables = rows[0].cdb_querytables; + var tableNames = qtables.split(/^\{(.*)\}$/)[1]; + tableNames = tableNames ? tableNames.split(',') : []; + callback(null, tableNames); +} + +QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) { sql = sql .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') .replace(affectedTableRegexCache.pixel_width, '1') @@ -72,23 +85,42 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam 'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])' ].join(' '); - sqlApi.query(username, api_key, query, function(err, rows){ - if (err || rows.length === 0) { - var msg = err.message ? err.message : err; - callback(new Error('could not fetch affected tables and last updated time: ' + msg)); - return; - } - - var result = rows[0]; - - var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1]; - tableNames = tableNames ? tableNames.split(',') : []; - - var lastUpdatedTime = result.max || 0; - - callback(null, { - affectedTables: tableNames, - lastUpdatedTime: lastUpdatedTime * 1000 + if (shouldQueryPostgresDirectly()) { + var psql = new PSQL(options); + psql.query(query, function(err, resultSet) { + var rows = resultSet.rows || []; + handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback); }); - }); + } else { + sqlApi.query(username, options.api_key, query, function(err, rows) { + handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback); + }); + } }; + +function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { + if (err || rows.length === 0) { + var msg = err.message ? err.message : err; + callback(new Error('could not fetch affected tables and last updated time: ' + msg)); + return; + } + + var result = rows[0]; + + var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1]; + tableNames = tableNames ? tableNames.split(',') : []; + + var lastUpdatedTime = result.max || 0; + + callback(null, { + affectedTables: tableNames, + lastUpdatedTime: lastUpdatedTime * 1000 + }); +} + + +function shouldQueryPostgresDirectly() { + return global.environment + && global.environment.enabledFeatures + && global.environment.enabledFeatures.cdbQueryTablesFromPostgres; +} diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index cd8098ad..751a8218 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -191,7 +191,14 @@ module.exports = function(){ if ( req.profiler ) req.profiler.done('getSignerMapKey'); key = data; } - queryTablesApi.getAffectedTablesInQuery(user, key, sql, this); // in addCacheChannel + queryTablesApi.getAffectedTablesInQuery(user, { + user: req.params.dbuser, + pass: req.params.dbpass, + host: req.params.dbhost, + port: req.params.dbport, + dbname: req.params.dbname, + api_key: key + }, sql, this); // in addCacheChannel }, function finish(err, data) { next(err,data); @@ -316,7 +323,14 @@ module.exports = function(){ Step( function getAffectedTablesAndLastUpdatedTime() { - queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, key, sql, this); + queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, { + user: req.params.dbuser, + pass: req.params.dbpass, + host: req.params.dbhost, + port: req.params.dbport, + dbname: req.params.dbname, + api_key: key + }, sql, this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { if (req.profiler) req.profiler.done('queryTablesAndLastUpdated'); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index a331beb6..42e552a6 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -937,6 +937,29 @@ "version": "0.5.0", "from": "git://github.com/CartoDB/node-cartodb-redis.git#0.5.0" }, + "cartodb-psql": { + "version": "0.2.0", + "from": "git://github.com/CartoDB/node-cartodb-psql.git#0.2.0", + "dependencies": { + "pg": { + "version": "2.6.2", + "dependencies": { + "generic-pool": { + "version": "2.0.3" + }, + "buffer-writer": { + "version": "1.0.0" + } + } + }, + "step": { + "version": "0.0.5" + }, + "underscore": { + "version": "1.6.0" + } + } + }, "redis-mpool": { "version": "0.0.4", "from": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4", diff --git a/package.json b/package.json index 74daefb7..58f98aa8 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "step": "0.0.x", "request": "2.9.202", "cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.5.0", + "cartodb-psql": "git://github.com/CartoDB/node-cartodb-psql.git#0.2.0", "redis-mpool": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4", "mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1", "lzma": "~1.2.3", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index fd95edd7..f685dad6 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -23,7 +23,11 @@ serverOptions = ServerOptions(); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -suite('multilayer', function() { +[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) { + +global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue}; + +suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { var redis_client = redis.createClient(global.environment.redis.port); var sqlapi_server; @@ -1342,3 +1346,4 @@ suite('multilayer', function() { }); +}); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 0a430dd1..e98a68d4 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -19,7 +19,12 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')(); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -suite('server', function() { +[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) { + +global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue}; + +suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { + var redis_client = redis.createClient(global.environment.redis.port); var sqlapi_server; @@ -1389,3 +1394,4 @@ suite('server', function() { }); +}); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 085afad1..f169a914 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -25,7 +25,11 @@ var serverOptions = ServerOptions(); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -suite('template_api', function() { +[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) { + +global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue}; + +suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { var redis_client = redis.createClient(global.environment.redis.port); var sqlapi_server; @@ -1948,3 +1952,4 @@ suite('template_api', function() { }); +});