cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use node-cartodb-query-tables library

Browse Source
This commit is contained in:
Alejandro Martínez
2016-02-22 11:40:25 +01:00
parent e1732076fc
commit cf06ff86c2
10 changed files with 92 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/api/query_tables_api.js
-26
View File
@@ -53,32 +53,6 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
});
};
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
});
};
function prepareSql(sql) {
return sql
lib/cartodb/backends/pg_connection.js
+35
View File
@@ -1,5 +1,6 @@
var assert = require('assert');
var step = require('step');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
function PgConnection(metadataBackend) {
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
}
);
};
/**
* Returns a `cartodb-psql` object for a given username.
* @param {String} username
* @param {Function} callback function({Error}, {PSQL})
*/
PgConnection.prototype.getConnection = function(username, callback) {
var self = this;
var params = {};
require('debug')('cachechan')("getConn1");
step(
function setAuth() {
self.setDBAuth(username, params, this);
},
function setConn(err) {
assert.ifError(err);
self.setDBConn(username, params, this);
},
function openConnection(err) {
assert.ifError(err);
return callback(err, new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
}));
}
);
};
lib/cartodb/cache/model/database_tables_entry.js
Vendored
-26
View File
@@ -1,26 +0,0 @@
var crypto = require('crypto');
function DatabaseTables(tables) {
this.namespace = 't';
this.tables = tables;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tables.map(function(table) {
return this.namespace + ':' + shortHashKey(table.dbname + ':' + table.table_name + '.' + table.schema_name);
}.bind(this)).sort();
};
DatabaseTables.prototype.getCacheChannel = function() {
var key = this.tables.map(function(table) {
return table.dbname + ':' + table.schema_name + "." + table.table_name;
}).sort().join(";;");
return key;
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}
lib/cartodb/controllers/layergroup.js
+18 -7
View File
@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var QueryTables = require('node-cartodb-query-tables');
/**
* @param {AuthApi} authApi
@@ -28,6 +29,7 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
@@ -320,9 +322,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
self.surrogateKeysCache.tag(res, affectedTables);
}
self.send(req, res, body, status, headers);
}
@@ -366,11 +367,21 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
step(
function getConnection() {
self.pgConnection.getConnection(user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
return tables;
},
@@ -378,7 +389,7 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
if(tables === undefined){
callback(err);
}else{
callback(err, tables.affectedTables);
callback(err, tables);
}
}
);
lib/cartodb/controllers/map.js
+18 -10
View File
@@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var step = require('step');
var windshaft = require('windshaft');
var QueryTables = require('node-cartodb-query-tables');
var util = require('util');
var BaseController = require('./base');
@@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
@@ -329,31 +329,39 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
// return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
// });
// } else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
// self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
//}
step(
function getConnection() {
self.pgConnection.getConnection(username, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, result);
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString());
res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables && result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
lib/cartodb/controllers/named_maps.js
+6 -9
View File
@@ -9,8 +9,6 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
BaseController.call(this, authApi, pgConnection);
@@ -53,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
if (err) {
global.logger.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
if (!result || !!result.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
lastModifiedDate = new Date(result.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
self.send(req, res, resource, 200);
@@ -230,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
return next(null);
}
var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
var affectedTables = affectedTablesAndLastUpdate.tables || [];
if (affectedTables.length === 0) {
return next(null);