cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make generic middlewares to calculate surrogate key and cache channel headers:

Browse Source 
- In controllers: all reference to map config are now camelized, for instance: mapconfig -> mapConfig or mapconfigProvider -> mapConfigProvider
 - In controllers: all map config providers created in req/res cycle are saved into `res.locals` and `mapConfigProvider` as key.
 - In map-config-providers: all of them implement `.getAffectedTables()`, in order to calculate the tables involved for a given map-config. For that, `pgConnection` and `affectedTablesCache` are injected as constructor argument.
 - Named Map Provider: rename references from `affectedTablesAndLastUpdate` to `affectedTables`.
 - Named Map Provider Cache: In order to create new named map providers, needs affectedTablesCache.
 - Extract locals middlewares (surrogate-key and cache-channel) from controllers and create an unified version of them.
 - Extract last-modified middleware from named maps controller (draft).
This commit is contained in:
Daniel García Aubert
2018-03-21 14:11:54 +01:00
parent af8ed99ae7
commit a142620b70
11 changed files with 538 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/cache/named_map_provider_cache.js
Vendored
+10 -1
View File
@@ -6,12 +6,20 @@ var queue = require('queue-async');
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
function NamedMapProviderCache(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.userLimitsApi = userLimitsApi;
this.mapConfigAdapter = mapConfigAdapter;
this.affectedTablesCache = affectedTablesCache;
this.providerCache = new LruCache({ max: 2000 });
}
@@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter,
this.affectedTablesCache,
user,
templateId,
config,
lib/cartodb/controllers/layergroup.js
+113 -139
View File
@@ -9,11 +9,12 @@ const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const sendResponse = require('../middleware/send-response');
const DataviewBackend = require('../backends/dataview');
const AnalysisStatusBackend = require('../backends/analysis-status');
const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
const QueryTables = require('cartodb-query-tables');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
@@ -44,7 +45,7 @@ function LayergroupController(
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
layergroupAffectedTablesCache,
analysisBackend,
authApi
) {
@@ -55,7 +56,7 @@ function LayergroupController(
this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.dataviewBackend = new DataviewBackend(analysisBackend);
this.analysisStatusBackend = new AnalysisStatusBackend();
@@ -65,10 +66,10 @@ function LayergroupController(
module.exports = LayergroupController;
LayergroupController.prototype.register = function(app) {
const { base_url_mapconfig: mapconfigBasePath } = app;
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapconfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
`${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
cleanUpQueryParams(),
locals(),
@@ -78,13 +79,17 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
@@ -93,7 +98,7 @@ LayergroupController.prototype.register = function(app) {
);
app.get(
`${mapconfigBasePath}/:token/:z/:x/:y.:format`,
`${mapConfigBasePath}/:token/:z/:x/:y.:format`,
cors(),
cleanUpQueryParams(),
locals(),
@@ -103,13 +108,17 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
@@ -118,7 +127,7 @@ LayergroupController.prototype.register = function(app) {
);
app.get(
`${mapconfigBasePath}/:token/:layer/:z/:x/:y.(:format)`,
`${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`,
distinguishLayergroupFromStaticRoute(),
cors(),
cleanUpQueryParams(),
@@ -129,13 +138,17 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'maplayer_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
@@ -144,7 +157,7 @@ LayergroupController.prototype.register = function(app) {
);
app.get(
`${mapconfigBasePath}/:token/:layer/attributes/:fid`,
`${mapConfigBasePath}/:token/:layer/attributes/:fid`,
cors(),
cleanUpQueryParams(),
locals(),
@@ -154,20 +167,24 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getFeatureAttributes(this.attributesBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
const forcedFormat = 'png';
app.get(
`${mapconfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
`${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
cleanUpQueryParams(['layer']),
locals(),
@@ -177,18 +194,23 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByCenter(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
app.get(
`${mapconfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
`${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
cleanUpQueryParams(['layer']),
locals(),
@@ -198,13 +220,18 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByBoundingBox(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
@@ -227,7 +254,7 @@ LayergroupController.prototype.register = function(app) {
];
app.get(
`${mapconfigBasePath}/:token/dataview/:dataviewName`,
`${mapConfigBasePath}/:token/dataview/:dataviewName`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@@ -237,18 +264,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/:layer/widget/:dataviewName`,
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@@ -258,18 +289,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/dataview/:dataviewName/search`,
`${mapConfigBasePath}/:token/dataview/:dataviewName/search`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@@ -279,18 +314,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/:layer/widget/:dataviewName/search`,
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
@@ -300,18 +339,22 @@ LayergroupController.prototype.register = function(app) {
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
sendResponse()
);
app.get(
`${mapconfigBasePath}/:token/analysis/node/:nodeId`,
`${mapConfigBasePath}/:token/analysis/node/:nodeId`,
cors(),
cleanUpQueryParams(),
locals(),
@@ -367,7 +410,13 @@ function getRequestParams(locals) {
return params;
}
function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) {
function createMapStoreMapConfigProvider (
mapStore,
userLimitsApi,
pgConnection,
affectedTablesCache,
forcedFormat = null
) {
return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
const { user } = res.locals;
@@ -378,7 +427,14 @@ function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat
params.layer = params.layer || 'all';
}
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params);
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(
mapStore,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
params
);
next();
};
@@ -575,88 +631,6 @@ function setCacheControlHeader () {
};
}
function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) {
return function getAffectedTablesMiddleware (req, res, next) {
const { user, dbname, token } = res.locals;
if (layergroupAffectedTables.hasAffectedTables(dbname, token)) {
res.locals.affectedTables = layergroupAffectedTables.get(dbname, token);
return next();
}
mapStore.load(token, (err, mapconfig) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
const queries = [];
mapconfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
global.logger.warn('ERROR generating cache channel:' +
' this request doesn\'t need an X-Cache-Channel generated');
return next();
}
pgConnection.getConnection(user, (err, connection) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
global.logger.warn('ERROR generating cache channel: ', err);
return next();
}
// feed affected tables cache so it can be reused from, for instance, map controller
layergroupAffectedTables.set(dbname, token, affectedTables);
res.locals.affectedTables = affectedTables;
next();
});
});
});
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
surrogateKeysCache.tag(res, affectedTables);
}
next();
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
lib/cartodb/controllers/map.js
+94 -119
View File
@@ -2,7 +2,6 @@ const _ = require('underscore');
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const QueryTables = require('cartodb-query-tables');
const ResourceLocator = require('../models/resource-locator');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
@@ -12,8 +11,9 @@ const layergroupToken = require('../middleware/layergroup-token');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const sendResponse = require('../middleware/send-response');
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
@@ -64,15 +64,15 @@ function MapController (
module.exports = MapController;
MapController.prototype.register = function(app) {
const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app;
const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app;
app.get(
`${mapconfigBasePath}`,
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
`${mapconfigBasePath}`,
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
@@ -88,7 +88,7 @@ MapController.prototype.register = function(app) {
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.options(app.base_url_mapconfig, cors('Content-Type'));
app.options(`${mapConfigBasePath}`, cors('Content-Type'));
};
MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
@@ -113,9 +113,8 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us
this.getCreateMapMiddlewares(useTemplate),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
getAffectedTables(this.pgConnection, this.layergroupAffectedTables),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setLastModified(),
setLastUpdatedTimeToLayergroup(),
setCacheControl(),
@@ -140,16 +139,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) {
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter
this.mapConfigAdapter,
this.layergroupAffectedTables
),
instantiateLayergroup(this.mapBackend, this.userLimitsApi)
instantiateLayergroup(
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
}
return [
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
createLayergroup (this.mapBackend, this.userLimitsApi)
createLayergroup (
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
};
@@ -220,17 +230,25 @@ function checkCreateLayergroup () {
};
}
function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
function getTemplate (
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
return function getTemplateMiddleware (req, res, next) {
const templateParams = req.body;
const { user } = res.locals;
const mapconfigProvider = new NamedMapMapConfigProvider(
const mapConfigProvider = new NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
user,
req.params.template_id,
templateParams,
@@ -238,15 +256,15 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi
res.locals
);
mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => {
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => {
req.profiler.done('named.getMapConfig');
if (err) {
return next(err);
}
res.locals.mapconfig = mapconfig;
res.locals.mapConfig = mapConfig;
res.locals.rendererParams = rendererParams;
res.locals.mapconfigProvider = mapconfigProvider;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
@@ -289,38 +307,51 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
};
}
function createLayergroup (mapBackend, userLimitsApi) {
function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function createLayergroupMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { context, user } = res.locals;
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapconfig = new MapConfig(requestMapConfig, datasource);
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals);
const mapConfig = new MapConfig(requestMapConfig, datasource);
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
res.locals
);
res.locals.mapconfig = mapconfig;
res.locals.mapConfig = mapConfig;
res.locals.analysesResults = context.analysesResults;
mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapConfig, res.locals, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.body = layergroup;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}
function instantiateLayergroup (mapBackend, userLimitsApi) {
function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function instantiateLayergroupMiddleware (req, res, next) {
const { user, mapconfig, rendererParams } = res.locals;
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams);
const { user, mapConfig, rendererParams } = res.locals;
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
rendererParams
);
mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
@@ -328,12 +359,11 @@ function instantiateLayergroup (mapBackend, userLimitsApi) {
res.body = layergroup;
const { mapconfigProvider } = res.locals;
const { mapConfigProvider } = res.locals;
res.locals.analysesResults = mapconfigProvider.analysesResults;
res.locals.template = mapconfigProvider.template;
res.locals.templateName = mapconfigProvider.getTemplateName();
res.locals.context = mapconfigProvider.context;
res.locals.analysesResults = mapConfigProvider.analysesResults;
res.locals.template = mapConfigProvider.template;
res.locals.context = mapConfigProvider.context;
next();
});
@@ -342,10 +372,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) {
function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware(req, res, next) {
const { mapconfig, user } = res.locals;
const { mapConfig, user } = res.locals;
// Error won't blow up, just be logged.
metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
@@ -370,71 +400,6 @@ function augmentLayergroupData () {
};
}
function getAffectedTables (pgConnection, layergroupAffectedTables) {
return function getAffectedTablesMiddleware (req, res, next) {
const { dbname, user, mapconfig } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
const sql = [];
mapconfig.getLayers().forEach(function(layer) {
sql.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
sql.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
return next(err);
}
// feed affected tables cache so it can be reused from, for instance, layergroup controller
layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
res.locals.affectedTables = affectedTables;
next();
});
});
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (req.method === 'GET') {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { affectedTables, user, templateName } = res.locals;
if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) {
surrogateKeysCache.tag(res, affectedTables);
}
if (templateName) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
}
next();
};
}
function setLastModified () {
return function setLastModifiedMiddleware (req, res, next) {
if (req.method === 'GET') {
@@ -447,18 +412,28 @@ function setLastModified () {
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { affectedTables, analysesResults } = res.locals;
const { mapConfigProvider, analysesResults } = res.locals;
const layergroup = res.body;
var lastUpdateTime = affectedTables.getLastUpdatedAt();
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
return next(err);
}
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
if (!affectedTables) {
return next();
}
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
var lastUpdateTime = affectedTables.getLastUpdatedAt();
next();
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
});
};
}
@@ -488,7 +463,7 @@ function setCacheControl () {
function setLayerStats (pgConnection, statsBackend) {
return function setLayerStatsMiddleware(req, res, next) {
const { user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
@@ -496,7 +471,7 @@ function setLayerStats (pgConnection, statsBackend) {
return next(err);
}
statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
statsBackend.getStats(mapConfig, connection, function(err, layersStats) {
if (err) {
return next(err);
}
@@ -531,10 +506,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) {
function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
const { user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj());
next();
};
@@ -553,10 +528,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
const { mapconfig, context } = res.locals;
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
@@ -564,10 +539,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
function setAggregationMetadataToLayergroup (layergroupMetadata) {
return function setAggregationMetadataToLayergroupMiddleware (req, res, next) {
const { mapconfig, context } = res.locals;
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
@@ -575,10 +550,10 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) {
function setTilejsonMetadataToLayergroup (layergroupMetadata) {
return function augmentLayergroupTilejsonMiddleware (req, res, next) {
const { user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig);
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig);
next();
};
@@ -589,10 +564,10 @@ function augmentError (options) {
return function augmentErrorMiddleware (err, req, res, next) {
req.profiler.done('error');
const { mapconfig } = res.locals;
const { mapConfig } = res.locals;
if (addContext) {
err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err;
err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err;
}
err.label = label;
lib/cartodb/controllers/named_maps.js
+29 -93
View File
@@ -1,4 +1,3 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const locals = require('../middleware/locals');
@@ -7,6 +6,9 @@ const layergroupToken = require('../middleware/layergroup-token');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
@@ -28,8 +30,8 @@ function getRequestParams(locals) {
const params = Object.assign({}, locals);
delete params.template;
delete params.affectedTablesAndLastUpdate;
delete params.namedMapProvider;
delete params.affectedTables;
delete params.mapConfigProvider;
delete params.allowedQueryParams;
return params;
@@ -77,14 +79,13 @@ NamedMapsController.prototype.register = function(app) {
namedMapProviderCache: this.namedMapProviderCache,
label: 'NAMED_MAP_TILE'
}),
getAffectedTables(),
getTile({
tileBackend: this.tileBackend,
label: 'NAMED_MAP_TILE'
}),
setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setCacheChannelHeader(),
setLastModifiedHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
setCacheControlHeader(),
setContentTypeHeader(),
sendResponse(),
@@ -106,7 +107,6 @@ NamedMapsController.prototype.register = function(app) {
namedMapProviderCache: this.namedMapProviderCache,
label: 'STATIC_VIZ_MAP', forcedFormat: 'png'
}),
getAffectedTables(),
getTemplate({ label: 'STATIC_VIZ_MAP' }),
prepareLayerFilterFromPreviewLayers({
namedMapProviderCache: this.namedMapProviderCache,
@@ -115,9 +115,9 @@ NamedMapsController.prototype.register = function(app) {
getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }),
getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }),
incrementMapViews({ metadataBackend: this.metadataBackend }),
setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setCacheChannelHeader(),
setLastModifiedHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
cacheChannelHeader(),
lastModifiedHeader(),
setCacheControlHeader(),
setContentTypeHeader(),
sendResponse()
@@ -143,25 +143,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul
return next(err);
}
res.locals.namedMapProvider = namedMapProvider;
next();
});
};
}
function getAffectedTables () {
return function getAffectedTables (req, res, next) {
const { namedMapProvider } = res.locals;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
req.profiler.done('affectedTables');
if (err) {
return next(err);
}
res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
res.locals.mapConfigProvider = namedMapProvider;
next();
});
@@ -170,9 +152,9 @@ function getAffectedTables () {
function getTemplate ({ label }) {
return function getTemplateMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
const { mapConfigProvider } = res.locals;
namedMapProvider.getTemplate((err, template) => {
mapConfigProvider.getTemplate((err, template) => {
if (err) {
err.label = label;
return next(err);
@@ -220,7 +202,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
return next(err);
}
res.locals.namedMapProvider = provider;
res.locals.mapConfigProvider = provider;
next();
});
@@ -229,9 +211,9 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
function getTile ({ tileBackend, label }) {
return function getTileMiddleware (req, res, next) {
const { namedMapProvider, format } = res.locals;
const { mapConfigProvider, format } = res.locals;
tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
tileBackend.getTile(mapConfigProvider, req.params, (err, tile, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
@@ -253,7 +235,7 @@ function getTile ({ tileBackend, label }) {
function getStaticImageOptions ({ tablesExtentApi }) {
return function getStaticImageOptionsMiddleware(req, res, next) {
const { user, namedMapProvider, template } = res.locals;
const { user, mapConfigProvider, template } = res.locals;
const imageOpts = getImageOptions(res.locals, template);
@@ -264,18 +246,18 @@ function getStaticImageOptions ({ tablesExtentApi }) {
res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
return next();
}
var affectedTables = affectedTablesAndLastUpdate.tables || [];
var tables = affectedTables.tables || [];
if (affectedTables.length === 0) {
if (tables.length === 0) {
return next();
}
tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
tablesExtentApi.getBounds(user, tables, (err, bounds) => {
if (err) {
return next();
}
@@ -355,7 +337,7 @@ function getImageOptionsFromBoundingBox (bbox = '') {
function getImage({ previewBackend, label }) {
return function getImageMiddleware (req, res, next) {
const { imageOpts, namedMapProvider } = res.locals;
const { imageOpts, mapConfigProvider } = res.locals;
const { zoom, center, bounds } = imageOpts;
let { width, height } = req.params;
@@ -366,7 +348,7 @@ function getImage({ previewBackend, label }) {
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
if (zoom !== undefined && center) {
return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center,
(err, image, headers, stats) => {
req.profiler.add(stats);
@@ -385,7 +367,7 @@ function getImage({ previewBackend, label }) {
});
}
previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
@@ -411,9 +393,9 @@ function incrementMapViewsError (ctx) {
function incrementMapViews ({ metadataBackend }) {
return function incrementMapViewsMiddleware(req, res, next) {
const { user, namedMapProvider } = res.locals;
const { user, mapConfigProvider } = res.locals;
namedMapProvider.getMapConfig((err, mapConfig) => {
mapConfigProvider.getMapConfig((err, mapConfig) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
return next();
@@ -462,59 +444,13 @@ function templateBounds(view) {
return false;
}
function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
if (affectedTablesAndLastUpdate.tables.length > 0) {
surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
}
}
next();
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
}
next();
};
}
function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
var lastModifiedDate;
if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
}
next();
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
const { affectedTables } = res.locals;
res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
if (!affectedTables || !!affectedTables.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
}
lib/cartodb/middleware/cache-channel-header.js
+24
View File
@@ -0,0 +1,24 @@
module.exports = function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
if (req.method !== 'GET') {
return next();
}
const { mapConfigProvider } = res.locals;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Cache Channel Header:', err);
return next();
}
if (!affectedTables) {
return next();
}
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
next();
});
};
};
lib/cartodb/middleware/last-modified-header.js
+40
View File
@@ -0,0 +1,40 @@
module.exports = function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware(req, res, next) {
if (req.method !== 'GET') {
return next();
}
const { mapConfigProvider, cache_buster } = res.locals;
if (cache_buster) {
const cacheBuster = parseInt(cache_buster, 10);
if (Number.isFinite(cacheBuster)) {
res.set('Last-Modified', new Date(cacheBuster).toUTCString());
}
return next();
}
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Last Modified Header:', err);
return next();
}
if (!affectedTables) {
return next();
}
const lastUpdatedAt = affectedTables.getLastUpdatedAt();
const lastModifiedDate = Number.isFinite(lastUpdatedAt) ?
new Date(lastUpdatedAt) :
new Date();
res.set('Last-Modified', lastModifiedDate.toUTCString());
next();
});
};
};
lib/cartodb/middleware/surrogate-key-header.js
+31
View File
@@ -0,0 +1,31 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { user, mapConfigProvider } = res.locals;
if (mapConfigProvider instanceof NamedMapMapConfigProvider) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName()));
}
if (req.method !== 'GET') {
return next();
}
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Surrogate Key Header:', err);
return next();
}
if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) {
return next();
}
surrogateKeysCache.tag(res, affectedTables);
next();
});
};
};
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
+63 -1
View File
@@ -2,6 +2,7 @@ var assert = require('assert');
var step = require('step');
var MapStoreMapConfigProvider = require('./map-store-provider');
const QueryTables = require('cartodb-query-tables');
/**
* @param {MapConfig} mapConfig
@@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider');
* @constructor
* @type {CreateLayergroupMapConfigProvider}
*/
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) {
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
this.mapConfig = mapConfig;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.params = params;
this.cacheBuster = params.cache_buster || 0;
}
@@ -46,3 +50,61 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr
CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter;
CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey;
CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) {
var self = this;
const { dbname } = self.params;
const token = self.mapConfig.id();
if (self.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = self.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
step(
function getSql() {
const queries = [];
self.mapConfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
return sql;
},
function getAffectedTables(err, sql) {
assert.ifError(err);
step(
function getConnection() {
self.pgConnection.getConnection(self.user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, affectedTables) {
if (err) {
return callback(err);
}
self.affectedTablesCache.set(dbname, token, affectedTables);
return callback(null, affectedTables);
}
);
};
lib/cartodb/models/mapconfig/provider/map-store-provider.js
+79 -3
View File
@@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var dot = require('dot');
var step = require('step');
const QueryTables = require('cartodb-query-tables');
/**
* @param {MapStore} mapStore
@@ -11,20 +12,30 @@ var step = require('step');
* @constructor
* @type {MapStoreMapConfigProvider}
*/
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) {
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
this.mapStore = mapStore;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.params = params;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.token = params.token;
this.cacheBuster = params.cache_buster || 0;
this.mapConfig = null;
this.params = params;
this.context = null;
}
module.exports = MapStoreMapConfigProvider;
MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
var self = this;
if (this.mapConfig !== null) {
return callback(null, this.mapConfig, this.params, this.context);
}
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
@@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
self.mapStore.load(self.token, this);
},
function finish(err, mapConfig) {
self.mapConfig = mapConfig;
self.context = context;
return callback(err, mapConfig, self.params, context);
}
);
@@ -74,4 +87,67 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) {
scale_factor: 1
});
return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues);
};
};
MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) {
var self = this;
const { dbname, token } = self.params;
if (self.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = self.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
step(
function getMapConfig() {
self.getMapConfig(this);
},
function getSql(err, mapConfig) {
assert.ifError(err);
const queries = [];
mapConfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
return sql;
},
function getAffectedTables(err, sql) {
assert.ifError(err);
step(
function getConnection() {
self.pgConnection.getConnection(self.user, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, affectedTables) {
if (err) {
return callback(err);
}
self.affectedTablesCache.set(dbname, token, affectedTables);
return callback(err, affectedTables);
}
);
};
lib/cartodb/models/mapconfig/provider/named-map-provider.js
+53 -14
View File
@@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables');
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter,
owner, templateId, config, authToken, params) {
function NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
owner,
templateId,
config,
authToken,
params
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
@@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend,
// use template after call to mapConfig
this.template = null;
this.affectedTablesAndLastUpdate = null;
this.affectedTablesCache = affectedTablesCache;
// providing
this.err = null;
@@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() {
NamedMapMapConfigProvider.prototype.reset = function() {
this.template = null;
this.affectedTablesAndLastUpdate = null;
this.affectedTables = null;
this.err = null;
this.mapConfig = null;
@@ -251,12 +262,11 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() {
return this.templateName;
};
NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) {
NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) {
var self = this;
if (this.affectedTablesAndLastUpdate !== null) {
return callback(null, this.affectedTablesAndLastUpdate);
}
let dbname = null;
let token = null;
step(
function getMapConfig() {
@@ -264,9 +274,33 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
},
function getSql(err, mapConfig) {
assert.ifError(err);
return mapConfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
dbname = self.rendererParams;
token = mapConfig.id();
if (self.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = self.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
return sql;
},
function getAffectedTables(err, sql) {
assert.ifError(err);
@@ -281,9 +315,14 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
this
);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;
return callback(err, result);
function finish(err, affectedTables) {
if (err) {
return callback(err);
}
self.affectedTablesCache.set(dbname, token, affectedTables);
return callback(err, affectedTables);
}
);
};
lib/cartodb/server.js
+2 -1
View File
@@ -200,7 +200,8 @@ module.exports = function(serverOptions) {
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter
mapConfigAdapter,
layergroupAffectedTablesCache
);
['update', 'delete'].forEach(function(eventType) {