cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'new_querytables_library' into analysis-layers

Browse Source
This commit is contained in:
Raul Ochoa
2016-03-04 12:08:57 +01:00
parent 10a602a4f3 16a7c4fa3d
commit c7bdabfc65
23 changed files with 325 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/api/query_tables_api.js
-90
View File
@@ -1,90 +0,0 @@
function QueryTablesApi(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
// This is an Array, so no need to split into parts
var tableNames = rows[0].cdb_querytablestext;
return callback(null, tableNames);
});
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
var query =
'SELECT * FROM CDB_QueryTables_Updated_At($windshaft$' + prepareSql(sql) + '$windshaft$)';
this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables or last updated time: ' + msg));
return;
}
var affectedTables = rows;
var updatedTimes = affectedTables.map(function getUpdateDate(table) {
return table.updated_at;
});
var lastUpdatedTime = (affectedTables.length === 0 ? 0 : Math.max.apply(null, updatedTimes)) || 0;
callback(null, {
affectedTables: affectedTables,
lastUpdatedTime: lastUpdatedTime
});
});
};
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
if (!Array.isArray(tableNames) || tableNames.length === 0) {
return callback(null, 0);
}
var query = [
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
'])'
].join(' ');
this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
if (err) {
var msg = err.message ? err.message : err;
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var lastUpdated = 0;
if (rows.length !== 0) {
lastUpdated = rows[0].max || 0;
}
return callback(null, lastUpdated*1000);
});
};
function prepareSql(sql) {
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1')
;
}
lib/cartodb/backends/pg_connection.js
+35
View File
@@ -1,5 +1,6 @@
var assert = require('assert');
var step = require('step');
var PSQL = require('cartodb-psql');
var _ = require('underscore');
function PgConnection(metadataBackend) {
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
}
);
};
/**
* Returns a `cartodb-psql` object for a given username.
* @param {String} username
* @param {Function} callback function({Error}, {PSQL})
*/
PgConnection.prototype.getConnection = function(username, callback) {
var self = this;
var params = {};
require('debug')('cachechan')("getConn1");
step(
function setAuth() {
self.setDBAuth(username, params, this);
},
function setConn(err) {
assert.ifError(err);
self.setDBConn(username, params, this);
},
function openConnection(err) {
assert.ifError(err);
return callback(err, new PSQL({
user: params.dbuser,
pass: params.dbpass,
host: params.dbhost,
port: params.dbport,
dbname: params.dbname
}));
}
);
};
lib/cartodb/cache/model/database_tables_entry.js
Vendored
-26
View File
@@ -1,26 +0,0 @@
var crypto = require('crypto');
function DatabaseTables(tables) {
this.namespace = 't';
this.tables = tables;
}
module.exports = DatabaseTables;
DatabaseTables.prototype.key = function() {
return this.tables.map(function(table) {
return this.namespace + ':' + shortHashKey(table.dbname + ':' + table.table_name + '.' + table.schema_name);
}.bind(this));
};
DatabaseTables.prototype.getCacheChannel = function() {
var key = this.tables.map(function(table) {
return table.dbname + ':' + table.schema_name + "." + table.table_name;
}).join(";;");
return key;
};
function shortHashKey(target) {
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
}
lib/cartodb/cache/named_map_provider_cache.js
Vendored
+1 -3
View File
@@ -7,11 +7,10 @@ var queue = require('queue-async');
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
@@ -30,7 +29,6 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.templateMaps,
this.pgConnection,
this.userLimitsApi,
this.queryTablesApi,
this.namedLayersAdapter,
user,
templateId,
lib/cartodb/controllers/layergroup.js
+19 -16
View File
@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var QueryTables = require('cartodb-query-tables');
/**
* @param {AuthApi} authApi
@@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @param {WidgetBackend} widgetBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @constructor
*/
function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
@@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
this.widgetBackend = widgetBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.layergroupAffectedTables = layergroupAffectedTables;
}
@@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
global.logger.warn('ERROR generating cache channel: ' + err);
}
if (!!affectedTables) {
var tablesCacheEntry = new TablesCacheEntry(affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
self.surrogateKeysCache.tag(res, affectedTables);
}
self.send(req, res, body, status, headers);
}
@@ -366,20 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(user, sql, this); // in addCacheChannel
step(
function getConnection() {
self.pgConnection.getConnection(user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function buildCacheChannel(err, tables) {
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, tables.affectedTables);
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
return tables;
},
function finish(err, tables) {
if(tables === undefined){
callback(err);
}else{
callback(err, tables.affectedTables);
}
}
callback
);
};
lib/cartodb/controllers/map.js
+15 -27
View File
@@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var step = require('step');
var windshaft = require('windshaft');
var QueryTables = require('cartodb-query-tables');
var util = require('util');
var BaseController = require('./base');
@@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
var Datasource = windshaft.model.Datasource;
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
var MapConfigAnalysisLayersAdapter = require('../models/mapconfig_analysis_layers_adapter');
@@ -27,7 +27,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
* @param {TemplateMaps} templateMaps
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {QueryTablesApi} queryTablesApi
* @param {OverviewsMetadataApi} overviewsMetadataApi
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
@@ -35,7 +34,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
queryTablesApi, overviewsMetadataApi,
overviewsMetadataApi,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
@@ -44,7 +43,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.queryTablesApi = queryTablesApi;
this.overviewsMetadataApi = overviewsMetadataApi;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
@@ -237,7 +235,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
self.templateMaps,
self.pgConnection,
self.userLimitsApi,
self.queryTablesApi,
self.namedLayersAdapter,
cdbuser,
req.params.template_id,
@@ -339,43 +336,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
var layergroupId = layergroup.layergroupid;
step(
function checkCachedAffectedTables() {
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
function getPgConnection() {
self.pgConnection.getConnection(username, this);
},
function getAffectedTablesAndLastUpdatedTime(err) {
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
// if (hasCache) {
// var next = this;
// var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
// self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
// if (err) {
// return next(err);
// }
// return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
// });
// } else {
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
//}
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
// feed affected tables cache so it can be reused from, for instance, layergroup controller
self.layergroupAffectedTables.set(dbName, layergroupId, result);
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.set('Last-Modified', (new Date()).toUTCString());
res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
if (result.affectedTables && result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tableCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables && result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
lib/cartodb/controllers/named_maps.js
+6 -9
View File
@@ -9,8 +9,6 @@ var BaseController = require('./base');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var TablesCacheEntry = require('../cache/model/database_tables_entry');
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
BaseController.call(this, authApi, pgConnection);
@@ -53,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
if (err) {
global.logger.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.affectedTables) {
if (!result || !!result.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.lastUpdatedTime);
lastModifiedDate = new Date(result.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
var tablesCacheEntry = new TablesCacheEntry(result.affectedTables);
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
if (result.affectedTables.length > 0) {
self.surrogateKeysCache.tag(res, tablesCacheEntry);
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
self.send(req, res, resource, 200);
@@ -230,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
return next(null);
}
var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
var affectedTables = affectedTablesAndLastUpdate.tables || [];
if (affectedTables.length === 0) {
return next(null);
lib/cartodb/models/mapconfig/named_map_provider.js
+12 -3
View File
@@ -5,17 +5,17 @@ var dot = require('dot');
var step = require('step');
var MapConfig = require('windshaft').model.MapConfig;
var templateName = require('../../backends/template_maps').templateName;
var QueryTables = require('cartodb-query-tables');
/**
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter,
owner, templateId, config, authToken, params) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
this.namedLayersAdapter = namedLayersAdapter;
this.owner = owner;
@@ -256,7 +256,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
},
function getAffectedTables(err, sql) {
assert.ifError(err);
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
step(
function getConnection() {
self.pgConnection.getConnection(self.owner, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;
lib/cartodb/server.js
+1 -5
View File
@@ -19,7 +19,6 @@ var windshaft = require('windshaft');
var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
@@ -52,7 +51,6 @@ module.exports = function(serverOptions) {
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var queryTablesApi = new QueryTablesApi(pgQueryRunner);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
@@ -142,7 +140,7 @@ module.exports = function(serverOptions) {
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
@@ -166,7 +164,6 @@ module.exports = function(serverOptions) {
new windshaft.backend.Widget(),
surrogateKeysCache,
userLimitsApi,
queryTablesApi,
layergroupAffectedTablesCache
).register(app);
@@ -176,7 +173,6 @@ module.exports = function(serverOptions) {
templateMaps,
mapBackend,
metadataBackend,
queryTablesApi,
overviewsMetadataApi,
surrogateKeysCache,
userLimitsApi,
lib/cartodb/utils/overviews_query_rewriter.js
+5 -9
View File
@@ -74,14 +74,9 @@ function overviews_view_name(table) {
}
// replace a table name in a query by anoter name
function replace_table_in_query(sql, old_table_name, new_table_name) {
function replace_table_in_query(sql, old_table_name, replacement) {
var old_table = TableNameParser.parse(old_table_name);
var new_table = TableNameParser.parse(new_table_name);
var old_table_ident = TableNameParser.table_identifier(old_table);
var new_table_ident = TableNameParser.table_identifier(new_table);
// text that will be substituted by the table name pattern
var replacement = new_table_ident;
// regular expression prefix (beginning) to match a table name
function pattern_prefix(schema, identifier) {
@@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
function overviews_query(query, overviews, zoom_level_expression) {
var replaced_query = query;
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
var replacement;
for ( var table in overviews ) {
if (overviews.hasOwnProperty(table)) {
var table_overviews = overviews[table];
var table_view = overviews_view_name(table);
replaced_query = replace_table_in_query(replaced_query, table, table_view);
sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )";
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
replaced_query = replace_table_in_query(replaced_query, table, replacement);
}
}
if ( replaced_query !== query ) {
@@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
return !!sql.match(
/^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
/^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
);
};