diff --git a/.gitignore b/.gitignore index 1000315b..ddeef2ef 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,3 @@ redis.pid *.log coverage/ .DS_Store -libredis_cell.so diff --git a/NEWS.md b/NEWS.md index b61c816f..588166b9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,8 @@ # Changelog +## 6.0.1 +Released 2018-mm-dd + ## 6.0.0 Released 2018-03-19 Backward incompatible changes: @@ -9,6 +12,7 @@ New features: - Upgrades camshaft to 0.61.8 - Upgrades cartodb-redis to 1.0.0 - Rate limit feature (disabled by default) + - Fixes for tests with PG11 ## 5.4.0 Released 2018-03-15 diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js index e9f10262..d27ea3e1 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/api/auth_api.js @@ -25,7 +25,7 @@ module.exports = AuthApi; // null if the request is not signed by anyone // or will be a string cartodb username otherwise. // -AuthApi.prototype.authorizedBySigner = function(res, callback) { +AuthApi.prototype.authorizedBySigner = function(req, res, callback) { if ( ! res.locals.token || ! res.locals.signer ) { return callback(null, false); // no signer requested } @@ -33,7 +33,7 @@ AuthApi.prototype.authorizedBySigner = function(res, callback) { var self = this; var layergroup_id = res.locals.token; - var auth_token = res.locals.auth_token; + var auth_token = req.query.auth_token; this.mapStore.load(layergroup_id, function(err, mapConfig) { if (err) { @@ -180,7 +180,7 @@ AuthApi.prototype.authorize = function(req, res, callback) { }); } - this.authorizedBySigner(res, (err, isAuthorizedBySigner) => { + this.authorizedBySigner(req, res, (err, isAuthorizedBySigner) => { if (err) { return callback(err); } diff --git a/lib/cartodb/backends/analysis-status.js b/lib/cartodb/backends/analysis-status.js index 97f851d2..0060f36b 100644 --- a/lib/cartodb/backends/analysis-status.js +++ b/lib/cartodb/backends/analysis-status.js @@ -5,16 +5,14 @@ function AnalysisStatusBackend() { module.exports = AnalysisStatusBackend; - -AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { - var nodeId = params.nodeId; - +AnalysisStatusBackend.prototype.getNodeStatus = function (nodeId, dbParams, callback) { var statusQuery = [ 'SELECT node_id, status, updated_at, last_error_message as error_message', 'FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\'' ].join(' '); - var pg = new PSQL(dbParamsFromReqParams(params)); + var pg = new PSQL(dbParams); + pg.query(statusQuery, function(err, result) { if (err) { return callback(err, result); @@ -36,23 +34,3 @@ AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { return callback(null, statusResponse); }, true); // use read-only transaction }; - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index c110f328..4eebb2a2 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -1,13 +1,11 @@ var assert = require('assert'); - var _ = require('underscore'); var PSQL = require('cartodb-psql'); var step = require('step'); - var BBoxFilter = require('../models/filter/bbox'); - var DataviewFactory = require('../models/dataview/factory'); var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); +const dbParamsFromReqParams = require('../utils/database-params'); var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' @@ -48,7 +46,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param } var pg = new PSQL(dbParamsFromReqParams(params)); - + var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); @@ -170,23 +168,3 @@ function getDataviewDefinition(mapConfig, dataviewName) { var dataviews = mapConfig.dataviews || {}; return dataviews[dataviewName]; } - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index ebafbbac..e0850822 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -6,12 +6,20 @@ var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.mapConfigAdapter = mapConfigAdapter; + this.affectedTablesCache = affectedTablesCache; this.providerCache = new LruCache({ max: 2000 }); } @@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.metadataBackend, this.userLimitsApi, this.mapConfigAdapter, + this.affectedTablesCache, user, templateId, config, diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 3ed0d5a4..2d89a091 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -1,28 +1,41 @@ -var PSQL = require('cartodb-psql'); -var cors = require('../middleware/cors'); -var userMiddleware = require('../middleware/user'); +const PSQL = require('cartodb-psql'); +const cors = require('../middleware/cors'); +const user = require('../middleware/user'); +const cleanUpQueryParams = require('../middleware/clean-up-query-params'); +const credentials = require('../middleware/credentials'); +const authorize = require('../middleware/authorize'); +const dbConnSetup = require('../middleware/db-conn-setup'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middleware/cache-control-header'); +const sendResponse = require('../middleware/send-response'); +const dbParamsFromResLocals = require('../utils/database-params'); -function AnalysesController(prepareContext, userLimitsApi) { - this.prepareContext = prepareContext; +function AnalysesController(pgConnection, authApi, userLimitsApi) { + this.pgConnection = pgConnection; + this.authApi = authApi; this.userLimitsApi = userLimitsApi; } module.exports = AnalysesController; AnalysesController.prototype.register = function (app) { + const { base_url_mapconfig: mapconfigBasePath } = app; + app.get( - `${app.base_url_mapconfig}/analyses/catalog`, + `${mapconfigBasePath}/analyses/catalog`, cors(), - userMiddleware(), + user(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), - this.prepareContext, + cleanUpQueryParams(), createPGClient(), getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), prepareResponse(), - setCacheControlHeader(), + cacheControlHeader({ ttl: 10, revalidate: true }), sendResponse(), unauthorizedError() ); @@ -30,7 +43,10 @@ AnalysesController.prototype.register = function (app) { function createPGClient () { return function createPGClientMiddleware (req, res, next) { - res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); + const dbParams = dbParamsFromResLocals(res.locals); + + res.locals.pg = new PSQL(dbParams); + next(); }; } @@ -97,25 +113,6 @@ function prepareResponse () { }; } -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - }; -} - function unauthorizedError () { return function unathorizedErrorMiddleware(err, req, res, next) { if (err.message.match(/permission\sdenied/)) { @@ -149,23 +146,3 @@ var tablesQueryTpl = ctx => ` FROM analysis_tables ORDER BY size DESC `; - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js deleted file mode 100644 index 977e4bad..00000000 --- a/lib/cartodb/controllers/layergroup.js +++ /dev/null @@ -1,663 +0,0 @@ -const cors = require('../middleware/cors'); -const userMiddleware = require('../middleware/user'); -const allowQueryParams = require('../middleware/allow-query-params'); -const vectorError = require('../middleware/vector-error'); -const rateLimit = require('../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const DataviewBackend = require('../backends/dataview'); -const AnalysisStatusBackend = require('../backends/analysis-status'); -const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); -const QueryTables = require('cartodb-query-tables'); -const SUPPORTED_FORMATS = { - grid_json: true, - json_torque: true, - torque_json: true, - png: true, - png32: true, - mvt: true -}; - -/** - * @param {prepareContext} prepareContext - * @param {PgConnection} pgConnection - * @param {MapStore} mapStore - * @param {TileBackend} tileBackend - * @param {PreviewBackend} previewBackend - * @param {AttributesBackend} attributesBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {AnalysisBackend} analysisBackend - * @constructor - */ -function LayergroupController( - prepareContext, - pgConnection, - mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTables, - analysisBackend -) { - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.attributesBackend = attributesBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; - - this.dataviewBackend = new DataviewBackend(analysisBackend); - this.analysisStatusBackend = new AnalysisStatusBackend(); - - this.prepareContext = prepareContext; -} - -module.exports = LayergroupController; - -LayergroupController.prototype.register = function(app) { - const { base_url_mapconfig: basePath } = app; - - app.get( - `${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - incrementSuccessMetrics(global.statsClient), - sendResponse(), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - app.get( - `${basePath}/:token/:z/:x/:y.:format`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - incrementSuccessMetrics(global.statsClient), - sendResponse(), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - app.get( - `${basePath}/:token/:layer/:z/:x/:y.(:format)`, - distinguishLayergroupFromStaticRoute(), - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getTile(this.tileBackend, 'maplayer_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - incrementSuccessMetrics(global.statsClient), - sendResponse(), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - app.get( - `${basePath}/:token/:layer/attributes/:fid`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getFeatureAttributes(this.attributesBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - const forcedFormat = 'png'; - - app.get( - `${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), - allowQueryParams(['layer']), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), - getPreviewImageByCenter(this.previewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), - allowQueryParams(['layer']), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), - getPreviewImageByBoundingBox(this.previewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - // Undocumented/non-supported API endpoint methods. - // Use at your own peril. - - const allowedDataviewQueryParams = [ - 'filters', // json - 'own_filter', // 0, 1 - 'no_filters', // 0, 1 - 'bbox', // w,s,e,n - 'start', // number - 'end', // number - 'column_type', // string - 'bins', // number - 'aggregation', //string - 'offset', // number - 'q', // widgets search - 'categories', // number - ]; - - app.get( - `${basePath}/:token/dataview/:dataviewName`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getDataview(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${basePath}/:token/:layer/widget/:dataviewName`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getDataview(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${basePath}/:token/dataview/:dataviewName/search`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${basePath}/:token/:layer/widget/:dataviewName/search`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - allowQueryParams(allowedDataviewQueryParams), - this.prepareContext, - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${basePath}/:token/analysis/node/:nodeId`, - cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), - this.prepareContext, - analysisNodeStatus(this.analysisStatusBackend), - sendResponse() - ); -}; - -function distinguishLayergroupFromStaticRoute () { - return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { - if (req.params.token === 'static') { - return next('route'); - } - - next(); - }; -} - -function analysisNodeStatus (analysisStatusBackend) { - return function analysisNodeStatusMiddleware(req, res, next) { - analysisStatusBackend.getNodeStatus(res.locals, (err, nodeStatus, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET NODE STATUS'; - return next(err); - } - - res.set({ - 'Cache-Control': 'public,max-age=5', - 'Last-Modified': new Date().toUTCString() - }); - - res.body = nodeStatus; - - next(); - }); - }; -} - -function getRequestParams(locals) { - const params = Object.assign({}, locals); - - delete params.mapConfigProvider; - delete params.allowedQueryParams; - - return params; -} - -function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) { - return function createMapStoreMapConfigProviderMiddleware (req, res, next) { - const { user } = res.locals; - - const params = getRequestParams(res.locals); - - if (forcedFormat) { - params.format = forcedFormat; - params.layer = params.layer || 'all'; - } - - res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params); - - next(); - }; -} - -function getDataview (dataviewBackend) { - return function getDataviewMiddleware (req, res, next) { - const { user, mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET DATAVIEW'; - return next(err); - } - - res.body = dataview; - - next(); - }); - }; -} - -function dataviewSearch (dataviewBackend) { - return function dataviewSearchMiddleware (req, res, next) { - const { user, dataviewName, mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET DATAVIEW SEARCH'; - return next(err); - } - - res.body = searchResult; - - next(); - }); - }; -} - -function getFeatureAttributes (attributesBackend) { - return function getFeatureAttributesMiddleware (req, res, next) { - req.profiler.start('windshaft.maplayer_attribute'); - - const { mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET ATTRIBUTES'; - return next(err); - } - - res.body = tile; - - next(); - }); - }; -} - -function getStatusCode(tile, format){ - return tile.length === 0 && format === 'mvt'? 204 : 200; -} - -function parseFormat (format = '') { - const prettyFormat = format.replace('.', '_'); - return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; -} - -function getTile (tileBackend, profileLabel = 'tile') { - return function getTileMiddleware (req, res, next) { - req.profiler.start(`windshaft.${profileLabel}`); - - const { mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { - req.profiler.add(stats); - - if (err) { - return next(err); - } - - if (headers) { - res.set(headers); - } - - const formatStat = parseFormat(req.params.format); - - res.statusCode = getStatusCode(tile, formatStat); - res.body = tile; - - next(); - }); - }; -} - -function getPreviewImageByCenter (previewBackend) { - return function getPreviewImageByCenterMiddleware (req, res, next) { - const width = +req.params.width; - const height = +req.params.height; - const zoom = +req.params.z; - const center = { - lng: +req.params.lng, - lat: +req.params.lat - }; - - const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - const { mapConfigProvider: provider } = res.locals; - - previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => { - req.profiler.done(`render-${format}`); - req.profiler.add(stats); - - if (err) { - err.label = 'STATIC_MAP'; - return next(err); - } - - if (headers) { - res.set(headers); - } - - res.set('Content-Type', headers['Content-Type'] || `image/${format}`); - - res.body = image; - - next(); - }); - }; -} - -function getPreviewImageByBoundingBox (previewBackend) { - return function getPreviewImageByBoundingBoxMiddleware (req, res, next) { - const width = +req.params.width; - const height = +req.params.height; - const bounds = { - west: +req.params.west, - north: +req.params.north, - east: +req.params.east, - south: +req.params.south - }; - const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - const { mapConfigProvider: provider } = res.locals; - - previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => { - req.profiler.done(`render-${format}`); - req.profiler.add(stats); - - if (err) { - err.label = 'STATIC_MAP'; - return next(err); - } - - if (headers) { - res.set(headers); - } - - res.set('Content-Type', headers['Content-Type'] || `image/${format}`); - - res.body = image; - - next(); - }); - }; -} - -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware (req, res, next) { - let { cache_buster: cacheBuster } = res.locals; - - cacheBuster = parseInt(cacheBuster, 10); - - const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date(); - - res.set('Last-Modified', lastUpdated.toUTCString()); - - next(); - }; -} - -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=31536000'); - - next(); - }; -} - -function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) { - return function getAffectedTablesMiddleware (req, res, next) { - const { user, dbname, token } = res.locals; - - if (layergroupAffectedTables.hasAffectedTables(dbname, token)) { - res.locals.affectedTables = layergroupAffectedTables.get(dbname, token); - return next(); - } - - mapStore.load(token, (err, mapconfig) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - const queries = []; - mapconfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); - - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - global.logger.warn('ERROR generating cache channel:' + - ' this request doesn\'t need an X-Cache-Channel generated'); - return next(); - } - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - global.logger.warn('ERROR generating cache channel: ', err); - return next(); - } - - // feed affected tables cache so it can be reused from, for instance, map controller - layergroupAffectedTables.set(dbname, token, affectedTables); - - res.locals.affectedTables = affectedTables; - - next(); - }); - }); - }); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - surrogateKeysCache.tag(res, affectedTables); - } - - next(); - }; -} - -function incrementSuccessMetrics (statsClient) { - return function incrementSuccessMetricsMiddleware (req, res, next) { - const formatStat = parseFormat(req.params.format); - - statsClient.increment('windshaft.tiles.success'); - statsClient.increment(`windshaft.tiles.${formatStat}.success`); - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - - res.status(res.statusCode || 200); - - if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - } else { - res.send(res.body); - } - }; -} - -function incrementErrorMetrics (statsClient) { - return function incrementErrorMetricsMiddleware (err, req, res, next) { - const formatStat = parseFormat(req.params.format); - - statsClient.increment('windshaft.tiles.error'); - statsClient.increment(`windshaft.tiles.${formatStat}.error`); - - next(err); - }; -} - -function tileError () { - return function tileErrorMiddleware (err, req, res, next) { - // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 - let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); - - // Rewrite mapnik parsing errors to start with layer number - const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); - - if (matches) { - errMsg = `style${matches[2]}: ${matches[1]}`; - } - - err.message = errMsg; - err.label = 'TILE RENDER'; - - next(err); - }; -} diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js new file mode 100644 index 00000000..3022bce3 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/analysis.js @@ -0,0 +1,75 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const sendResponse = require('../../middleware/send-response'); +const dbParamsFromResLocals = require('../../utils/database-params'); + +module.exports = class AnalysisController { + constructor ( + analysisStatusBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.analysisStatusBackend = analysisStatusBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + app.get( + `${mapConfigBasePath}/:token/analysis/node/:nodeId`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), + cleanUpQueryParams(), + analysisNodeStatus(this.analysisStatusBackend), + sendResponse() + ); + + } +}; + +function analysisNodeStatus (analysisStatusBackend) { + return function analysisNodeStatusMiddleware(req, res, next) { + const { nodeId } = req.params; + const dbParams = dbParamsFromResLocals(res.locals); + + analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET NODE STATUS'; + return next(err); + } + + res.set({ + 'Cache-Control': 'public,max-age=5', + 'Last-Modified': new Date().toUTCString() + }); + + res.body = nodeStatus; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js new file mode 100644 index 00000000..07ae5da1 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -0,0 +1,93 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); + +module.exports = class AttribitesController { + constructor ( + attributesBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.attributesBackend = attributesBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + app.get( + `${mapConfigBasePath}/:token/:layer/attributes/:fid`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getFeatureAttributes(this.attributesBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + } +}; + +function getFeatureAttributes (attributesBackend) { + return function getFeatureAttributesMiddleware (req, res, next) { + req.profiler.start('windshaft.maplayer_attribute'); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer, fid } = req.params; + + const params = { + token, + dbuser, dbname, dbpassword, dbhost, dbport, + layer, fid + }; + + attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET ATTRIBUTES'; + return next(err); + } + + res.body = tile; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js new file mode 100644 index 00000000..bca27f1e --- /dev/null +++ b/lib/cartodb/controllers/layergroup/dataview.js @@ -0,0 +1,199 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); + +const ALLOWED_DATAVIEW_QUERY_PARAMS = [ + 'filters', // json + 'own_filter', // 0, 1 + 'no_filters', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'offset', // number + 'q', // widgets search + 'categories', // number +]; + +module.exports = class DataviewController { + constructor ( + dataviewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.dataviewBackend = dataviewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + // Undocumented/non-supported API endpoint methods. + // Use at your own peril. + + app.get( + `${mapConfigBasePath}/:token/dataview/:dataviewName`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getDataview(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getDataview(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + dataviewSearch(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + dataviewSearch(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + } +}; + +function getDataview (dataviewBackend) { + return function getDataviewMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW'; + return next(err); + } + + res.body = dataview; + + next(); + }); + }; +} + +function dataviewSearch (dataviewBackend) { + return function dataviewSearchMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW SEARCH'; + return next(err); + } + + res.body = searchResult; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/index.js b/lib/cartodb/controllers/layergroup/index.js new file mode 100644 index 00000000..e4c1770c --- /dev/null +++ b/lib/cartodb/controllers/layergroup/index.js @@ -0,0 +1,114 @@ +const DataviewBackend = require('../../backends/dataview'); +const AnalysisStatusBackend = require('../../backends/analysis-status'); + +const TileController = require('./tile'); +const AttributesController = require('./attributes'); +const StaticController = require('./static'); +const DataviewController = require('./dataview'); +const AnalysisController = require('./analysis'); + +/** + * @param {prepareContext} prepareContext + * @param {PgConnection} pgConnection + * @param {MapStore} mapStore + * @param {TileBackend} tileBackend + * @param {PreviewBackend} previewBackend + * @param {AttributesBackend} attributesBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsApi} userLimitsApi + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {AnalysisBackend} analysisBackend + * @constructor + */ +function LayergroupController( + pgConnection, + mapStore, + tileBackend, + previewBackend, + attributesBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + analysisBackend, + authApi +) { + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.tileBackend = tileBackend; + this.previewBackend = previewBackend; + this.attributesBackend = attributesBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + + this.dataviewBackend = new DataviewBackend(analysisBackend); + this.analysisStatusBackend = new AnalysisStatusBackend(); + this.authApi = authApi; +} + +module.exports = LayergroupController; + +LayergroupController.prototype.register = function(app) { + + const tileController = new TileController( + this.tileBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + tileController.register(app); + + const attributesController = new AttributesController( + this.attributesBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + attributesController.register(app); + + const staticController = new StaticController( + this.previewBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + staticController.register(app); + + const dataviewController = new DataviewController( + this.dataviewBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + dataviewController.register(app); + + const analysisController = new AnalysisController( + this.analysisStatusBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + analysisController.register(app); + + +}; diff --git a/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js b/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js new file mode 100644 index 00000000..7d64cb01 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js @@ -0,0 +1,37 @@ +const MapStoreMapConfigProvider = require('../../../models/mapconfig/provider/map-store-provider'); + +module.exports = function createMapStoreMapConfigProvider ( + mapStore, + userLimitsApi, + pgConnection, + affectedTablesCache, + forcedFormat = null +) { + return function createMapStoreMapConfigProviderMiddleware (req, res, next) { + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer, z, x, y, scale_factor, format } = req.params; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + layer, z, x, y, scale_factor, format + }; + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; + } + + res.locals.mapConfigProvider = new MapStoreMapConfigProvider( + mapStore, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + params + ); + + next(); + }; +}; diff --git a/lib/cartodb/controllers/layergroup/static.js b/lib/cartodb/controllers/layergroup/static.js new file mode 100644 index 00000000..00510780 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/static.js @@ -0,0 +1,161 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); + +module.exports = class StaticController { + constructor ( + previewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.previewBackend = previewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + const forcedFormat = 'png'; + + app.get( + `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + cleanUpQueryParams(['layer']), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), + getPreviewImageByCenter(this.previewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + cleanUpQueryParams(['layer']), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), + getPreviewImageByBoundingBox(this.previewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + } +}; + +function getPreviewImageByCenter (previewBackend) { + return function getPreviewImageByCenterMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const zoom = +req.params.z; + const center = { + lng: +req.params.lng, + lat: +req.params.lat + }; + + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.body = image; + + next(); + }); + }; +} + +function getPreviewImageByBoundingBox (previewBackend) { + return function getPreviewImageByBoundingBoxMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const bounds = { + west: +req.params.west, + north: +req.params.north, + east: +req.params.east, + south: +req.params.south + }; + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.body = image; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js new file mode 100644 index 00000000..07727236 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -0,0 +1,230 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); +const vectorError = require('../../middleware/vector-error'); + +const SUPPORTED_FORMATS = { + grid_json: true, + json_torque: true, + torque_json: true, + png: true, + png32: true, + mvt: true +}; + +module.exports = class TileController { + constructor ( + tileBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.tileBackend = tileBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + app.get( + `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend, 'map_tile'), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:z/:x/:y.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend, 'map_tile'), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, + distinguishLayergroupFromStaticRoute(), + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend, 'maplayer_tile'), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError(), + sendResponse() + ); + } +}; + +function distinguishLayergroupFromStaticRoute () { + return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { + if (req.params.token === 'static') { + return next('route'); + } + + next(); + }; +} + +function parseFormat (format = '') { + const prettyFormat = format.replace('.', '_'); + return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; +} + +function getStatusCode(tile, format){ + return tile.length === 0 && format === 'mvt' ? 204 : 200; +} + +function getTile (tileBackend, profileLabel = 'tile') { + return function getTileMiddleware (req, res, next) { + req.profiler.start(`windshaft.${profileLabel}`); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { layer, z, x, y, format } = req.params; + + const params = { token, layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { + req.profiler.add(stats); + + if (err) { + return next(err); + } + + if (headers) { + res.set(headers); + } + + const formatStat = parseFormat(req.params.format); + + res.statusCode = getStatusCode(tile, formatStat); + res.body = tile; + + next(); + }); + }; +} + +function incrementSuccessMetrics (statsClient) { + return function incrementSuccessMetricsMiddleware (req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.success'); + statsClient.increment(`windshaft.tiles.${formatStat}.success`); + + next(); + }; +} + +function incrementErrorMetrics (statsClient) { + return function incrementErrorMetricsMiddleware (err, req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.error'); + statsClient.increment(`windshaft.tiles.${formatStat}.error`); + + next(err); + }; +} + +function tileError () { + return function tileErrorMiddleware (err, req, res, next) { + if (err.message === 'Tile does not exist' && req.params.format === 'mvt') { + res.statusCode = 204; + return next(); + } + + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + + // Rewrite mapnik parsing errors to start with layer number + const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + + if (matches) { + errMsg = `style${matches[2]}: ${matches[1]}`; + } + + err.message = errMsg; + err.label = 'TILE RENDER'; + + next(err); + }; +} diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 01cd2c5e..9fd67710 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,12 +2,18 @@ const _ = require('underscore'); const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const QueryTables = require('cartodb-query-tables'); const ResourceLocator = require('../models/resource-locator'); const cors = require('../middleware/cors'); -const userMiddleware = require('../middleware/user'); -const allowQueryParams = require('../middleware/allow-query-params'); -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const user = require('../middleware/user'); +const cleanUpQueryParams = require('../middleware/clean-up-query-params'); +const credentials = require('../middleware/credentials'); +const dbConnSetup = require('../middleware/db-conn-setup'); +const authorize = require('../middleware/authorize'); +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../utils/layergroup-metadata'); @@ -27,9 +33,18 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; * @param {StatsBackend} statsBackend * @constructor */ -function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, - statsBackend) { +function MapController ( + pgConnection, + templateMaps, + mapBackend, + metadataBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTables, + mapConfigAdapter, + statsBackend, + authApi +) { this.pgConnection = pgConnection; this.templateMaps = templateMaps; this.mapBackend = mapBackend; @@ -43,35 +58,37 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m this.layergroupMetadata = new LayergroupMetadata(resourceLocator); this.statsBackend = statsBackend; - this.prepareContext = prepareContext; + this.authApi = authApi; } module.exports = MapController; MapController.prototype.register = function(app) { - const { base_url_mapconfig, base_url_templated } = app; + const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; + + app.get( + `${mapConfigBasePath}`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) + ); + + app.post( + `${mapConfigBasePath}`, + this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) + ); + const useTemplate = true; app.get( - base_url_mapconfig, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - app.post( - base_url_mapconfig, - this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) - ); - app.get( - `${base_url_templated}/:template_id/jsonp`, + `${templateBasePath}/:template_id/jsonp`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); + app.post( - `${base_url_templated}/:template_id`, + `${templateBasePath}/:template_id`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); - app.options( - app.base_url_mapconfig, - cors('Content-Type') - ); + + app.options(`${mapConfigBasePath}`, cors('Content-Type')); }; MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { @@ -83,20 +100,22 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us return [ cors(), - userMiddleware(), + user(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, endpointGroup), - allowQueryParams(['aggregation']), - this.prepareContext, + cleanUpQueryParams(['aggregation']), initProfiler(isTemplateInstantiation), checkJsonContentType(), this.getCreateMapMiddlewares(useTemplate), incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), - getAffectedTables(this.pgConnection, this.layergroupAffectedTables), - setCacheChannel(), - setLastModified(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), setLastUpdatedTimeToLayergroup(), - setCacheControl(), setLayerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), @@ -104,7 +123,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us setTurboCartoMetadataToLayergroup(this.layergroupMetadata), setAggregationMetadataToLayergroup(this.layergroupMetadata), setTilejsonMetadataToLayergroup(this.layergroupMetadata), - setSurrogateKeyHeader(this.surrogateKeysCache), sendResponse(), augmentError({ label, addContext }) ]; @@ -119,16 +137,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { this.pgConnection, this.metadataBackend, this.userLimitsApi, - this.mapConfigAdapter + this.mapConfigAdapter, + this.layergroupAffectedTables ), - instantiateLayergroup(this.mapBackend, this.userLimitsApi) + instantiateLayergroup( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; } return [ checkCreateLayergroup(), prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup (this.mapBackend, this.userLimitsApi) + createLayergroup ( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; }; @@ -181,7 +210,7 @@ function checkInstantiteLayergroup () { function checkCreateLayergroup () { return function checkCreateLayergroupMiddleware (req, res, next) { if (req.method === 'GET') { - const { config } = res.locals; + const { config } = req.query; if (!config) { return next(new Error('layergroup GET needs a "config" parameter')); @@ -199,33 +228,45 @@ function checkCreateLayergroup () { }; } -function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { return function getTemplateMiddleware (req, res, next) { const templateParams = req.body; - const { user } = res.locals; + const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id } = req.params; + const { auth_token } = req.query; - const mapconfigProvider = new NamedMapMapConfigProvider( + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + const mapConfigProvider = new NamedMapMapConfigProvider( templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, + affectedTablesCache, user, - req.params.template_id, + template_id, templateParams, - res.locals.auth_token, - res.locals + auth_token, + params ); - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { req.profiler.done('named.getMapConfig'); if (err) { return next(err); } - res.locals.mapconfig = mapconfig; + res.locals.mapConfig = mapConfig; res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); @@ -235,7 +276,10 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi function prepareAdapterMapConfig (mapConfigAdapter) { return function prepareAdapterMapConfigMiddleware(req, res, next) { const requestMapConfig = req.body; - const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; + + const { user, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); const context = { analysisConfiguration: { @@ -254,7 +298,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) { } }; - mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { + mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => { req.profiler.done('anonymous.getMapConfig'); if (err) { return next(err); @@ -268,51 +312,75 @@ function prepareAdapterMapConfig (mapConfigAdapter) { }; } -function createLayergroup (mapBackend, userLimitsApi) { +function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function createLayergroupMiddleware (req, res, next) { const requestMapConfig = req.body; - const { context, user } = res.locals; - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals); - res.locals.mapconfig = mapconfig; + const { context } = res.locals; + const { user, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = { + cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport + }; + + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapConfig = new MapConfig(requestMapConfig, datasource); + + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + params + ); + + res.locals.mapConfig = mapConfig; res.locals.analysesResults = context.analysesResults; - mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; + + mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); }; } -function instantiateLayergroup (mapBackend, userLimitsApi) { +function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams); + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + rendererParams + ); - mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; - const { mapconfigProvider } = res.locals; + const { mapConfigProvider } = res.locals; - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; next(); }); @@ -321,10 +389,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) { function incrementMapViewCount (metadataBackend) { return function incrementMapViewCountMiddleware(req, res, next) { - const { mapconfig, user } = res.locals; + const { mapConfig, user } = res.locals; // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { req.profiler.done('incMapviewCount'); if (err) { @@ -338,7 +406,7 @@ function incrementMapViewCount (metadataBackend) { function augmentLayergroupData () { return function augmentLayergroupDataMiddleware (req, res, next) { - const { layergroup } = res.locals; + const layergroup = res.body; // include in layergroup response the variables in serverMedata // those variables are useful to send to the client information @@ -349,80 +417,33 @@ function augmentLayergroupData () { }; } -function getAffectedTables (pgConnection, layergroupAffectedTables) { - return function getAffectedTablesMiddleware (req, res, next) { - const { dbname, layergroup, user, mapconfig } = res.locals; +function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; - pgConnection.getConnection(user, (err, connection) => { + mapConfigProvider.getAffectedTables((err, affectedTables) => { if (err) { return next(err); } - const sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); + if (!affectedTables) { + return next(); + } - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - return next(err); - } + var lastUpdateTime = affectedTables.getLastUpdatedAt(); - // feed affected tables cache so it can be reused from, for instance, layergroup controller - layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - res.locals.affectedTables = affectedTables; + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - next(); - }); + next(); }); }; } -function setCacheChannel () { - return function setCacheChannelMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (req.method === 'GET') { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setLastModified () { - return function setLastModifiedMiddleware (req, res, next) { - if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); - } - - next(); - }; -} - -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { affectedTables, layergroup, analysesResults } = res.locals; - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }; -} - function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; @@ -436,27 +457,17 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) { }, lastUpdateTime); } -function setCacheControl () { - return function setCacheControlMiddleware (req, res, next) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - } - - next(); - }; -} - function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { - const { user, mapconfig, layergroup } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; pgConnection.getConnection(user, (err, connection) => { if (err) { return next(err); } - statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { if (err) { return next(err); } @@ -475,7 +486,8 @@ function setLayerStats (pgConnection, statsBackend) { function setLayergroupIdHeader (templateMaps, useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { layergroup, user, template } = res.locals; + const { user, template } = res.locals; + const layergroup = res.body; if (useTemplateHash) { var templateHash = templateMaps.fingerPrint(template).substring(0, 8); @@ -490,9 +502,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) { function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); next(); }; @@ -500,7 +513,8 @@ function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, user, analysesResults = [] } = res.locals; + const { user, analysesResults = [] } = res.locals; + const layergroup = res.body; layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); @@ -510,9 +524,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { function setTurboCartoMetadataToLayergroup (layergroupMetadata) { return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; + const layergroup = res.body; - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -520,9 +535,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) { function setAggregationMetadataToLayergroup (layergroupMetadata) { return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; + const layergroup = res.body; - layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -530,54 +546,24 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) { function setTilejsonMetadataToLayergroup (layergroupMetadata) { return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); next(); }; } -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; return function augmentErrorMiddleware (err, req, res, next) { req.profiler.done('error'); - const { mapconfig } = res.locals; + const { mapConfig } = res.locals; if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; } err.label = label; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 04b80ff4..2cb4f609 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -1,7 +1,14 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const cors = require('../middleware/cors'); -const userMiddleware = require('../middleware/user'); -const allowQueryParams = require('../middleware/allow-query-params'); +const user = require('../middleware/user'); +const cleanUpQueryParams = require('../middleware/clean-up-query-params'); +const credentials = require('../middleware/credentials'); +const dbConnSetup = require('../middleware/db-conn-setup'); +const authorize = require('../middleware/authorize'); +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const vectorError = require('../middleware/vector-error'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -18,25 +25,15 @@ function numMapper(n) { return +n; } -function getRequestParams(locals) { - const params = Object.assign({}, locals); - - delete params.template; - delete params.affectedTablesAndLastUpdate; - delete params.namedMapProvider; - delete params.allowedQueryParams; - - return params; -} - -function NamedMapsController( - prepareContext, - namedMapProviderCache, - tileBackend, +function NamedMapsController ( + namedMapProviderCache, + tileBackend, previewBackend, - surrogateKeysCache, - tablesExtentApi, - metadataBackend, + surrogateKeysCache, + tablesExtentApi, + metadataBackend, + pgConnection, + authApi, userLimitsApi ) { this.namedMapProviderCache = namedMapProviderCache; @@ -45,51 +42,55 @@ function NamedMapsController( this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; this.metadataBackend = metadataBackend; + this.pgConnection = pgConnection; + this.authApi = authApi; this.userLimitsApi = userLimitsApi; - this.prepareContext = prepareContext; } module.exports = NamedMapsController; NamedMapsController.prototype.register = function(app) { - const { base_url_mapconfig, base_url_templated } = app; + const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; app.get( - `${base_url_templated}/:template_id/:layer/:z/:x/:y.(:format)`, + `${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`, cors(), - userMiddleware(), + user(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), - this.prepareContext, + cleanUpQueryParams(), getNamedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, label: 'NAMED_MAP_TILE' }), - getAffectedTables(), getTile({ tileBackend: this.tileBackend, label: 'NAMED_MAP_TILE' }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse(), vectorError() ); app.get( - `${base_url_mapconfig}/static/named/:template_id/:width/:height.:format`, + `${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`, cors(), - userMiddleware(), + user(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), - allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), - this.prepareContext, + cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), getNamedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP', forcedFormat: 'png' }), - getAffectedTables(), getTemplate({ label: 'STATIC_VIZ_MAP' }), prepareLayerFilterFromPreviewLayers({ namedMapProviderCache: this.namedMapProviderCache, @@ -97,28 +98,35 @@ NamedMapsController.prototype.register = function(app) { }), getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), - incrementMapViews({ metadataBackend: this.metadataBackend }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), + incrementMapViews({ metadataBackend: this.metadataBackend }), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); }; function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { return function getNamedMapProviderMiddleware (req, res, next) { - const { user } = res.locals; - const { config, auth_token } = req.query; - const { template_id } = req.params; + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, layer: layerFromParams, z, x, y, format } = req.params; + const { layer: layerFromQuery } = req.query; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format + }; if (forcedFormat) { - res.locals.format = forcedFormat; - res.locals.layer = res.locals.layer || 'all'; + params.format = forcedFormat; + params.layer = params.layer || 'all'; } - const params = getRequestParams(res.locals); + const { config, auth_token } = req.query; namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { if (err) { @@ -126,25 +134,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul return next(err); } - res.locals.namedMapProvider = namedMapProvider; - - next(); - }); - }; -} - -function getAffectedTables () { - return function getAffectedTables (req, res, next) { - const { namedMapProvider } = res.locals; - - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { - req.profiler.done('affectedTables'); - - if (err) { - return next(err); - } - - res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate; + res.locals.mapConfigProvider = namedMapProvider; next(); }); @@ -153,9 +143,9 @@ function getAffectedTables () { function getTemplate ({ label }) { return function getTemplateMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider } = res.locals; - namedMapProvider.getTemplate((err, template) => { + mapConfigProvider.getTemplate((err, template) => { if (err) { err.label = label; return next(err); @@ -170,8 +160,7 @@ function getTemplate ({ label }) { function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) { return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) { - const { user, template } = res.locals; - const { template_id } = req.params; + const { template } = res.locals; const { config, auth_token } = req.query; if (!template || !template.view || !template.view.preview_layers) { @@ -191,7 +180,15 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) return next(); } - const params = getRequestParams(res.locals); + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, format } = req.params; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, format + }; // overwrites 'all' default filter params.layer = layerVisibilityFilter.join(','); @@ -203,7 +200,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) return next(err); } - res.locals.namedMapProvider = provider; + res.locals.mapConfigProvider = provider; next(); }); @@ -212,19 +209,24 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) function getTile ({ tileBackend, label }) { return function getTileMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider } = res.locals; + const { layer, z, x, y, format } = req.params; + const params = { layer, z, x, y, format }; - tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => { + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; return next(err); } - res.locals.body = tile; - res.locals.headers = headers; - res.locals.stats = stats; + if (headers) { + res.set(headers); + } + + res.body = tile; next(); }); @@ -233,9 +235,11 @@ function getTile ({ tileBackend, label }) { function getStaticImageOptions ({ tablesExtentApi }) { return function getStaticImageOptionsMiddleware(req, res, next) { - const { user, namedMapProvider, template } = res.locals; + const { user, mapConfigProvider, template } = res.locals; + const { zoom, lon, lat, bbox } = req.query; + const params = { zoom, lon, lat, bbox }; - const imageOpts = getImageOptions(res.locals, template); + const imageOpts = getImageOptions(params, template); if (imageOpts) { res.locals.imageOpts = imageOpts; @@ -244,18 +248,18 @@ function getStaticImageOptions ({ tablesExtentApi }) { res.locals.imageOpts = DEFAULT_ZOOM_CENTER; - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { + mapConfigProvider.getAffectedTables((err, affectedTables) => { if (err) { return next(); } - var affectedTables = affectedTablesAndLastUpdate.tables || []; + var tables = affectedTables.tables || []; - if (affectedTables.length === 0) { + if (tables.length === 0) { return next(); } - tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => { + tablesExtentApi.getBounds(user, tables, (err, bounds) => { if (err) { return next(); } @@ -335,7 +339,7 @@ function getImageOptionsFromBoundingBox (bbox = '') { function getImage({ previewBackend, label }) { return function getImageMiddleware (req, res, next) { - const { imageOpts, namedMapProvider } = res.locals; + const { imageOpts, mapConfigProvider } = res.locals; const { zoom, center, bounds } = imageOpts; let { width, height } = req.params; @@ -346,45 +350,62 @@ function getImage({ previewBackend, label }) { const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; if (zoom !== undefined && center) { - return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center, + return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, (err, image, headers, stats) => { + req.profiler.add(stats); + if (err) { err.label = label; return next(err); } - res.locals.body = image; - res.locals.headers = headers; - res.locals.stats = stats; + if (headers) { + res.set(headers); + } + + res.body = image; next(); }); } - previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => { + previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => { + req.profiler.add(stats); + req.profiler.done('render-' + format); + if (err) { err.label = label; return next(err); } - res.locals.body = image; - res.locals.headers = headers; - res.locals.stats = stats; + if (headers) { + res.set(headers); + } + + res.body = image; next(); }); }; } +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} + function incrementMapViewsError (ctx) { return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`; } function incrementMapViews ({ metadataBackend }) { return function incrementMapViewsMiddleware(req, res, next) { - const { user, namedMapProvider } = res.locals; + const { user, mapConfigProvider } = res.locals; - namedMapProvider.getMapConfig((err, mapConfig) => { + mapConfigProvider.getMapConfig((err, mapConfig) => { if (err) { global.logger.log(incrementMapViewsError({ user, err })); return next(); @@ -432,86 +453,3 @@ function templateBounds(view) { } return false; } - -function setSurrogateKeyHeader ({ surrogateKeysCache }) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals; - - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName())); - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - if (affectedTablesAndLastUpdate.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTablesAndLastUpdate); - } - } - - next(); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel()); - } - - next(); - }; -} - -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - var lastModifiedDate; - if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) { - lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); - } - - res.set('Last-Modified', lastModifiedDate.toUTCString()); - } - - next(); - }; - } - -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - } - - next(); - }; - } - -function setContentTypeHeader () { - return function setContentTypeHeaderMiddleware(req, res, next) { - const { headers = {} } = res.locals; - - res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - const { body, stats = {}, format } = res.locals; - - req.profiler.done('render-' + format); - req.profiler.add(stats); - - res.status(200); - res.send(body); - }; -} diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index c9ba652e..afcaa1e3 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -1,10 +1,10 @@ const { templateName } = require('../backends/template_maps'); const cors = require('../middleware/cors'); -const userMiddleware = require('../middleware/user'); +const user = require('../middleware/user'); +const credentials = require('../middleware/credentials'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const localsMiddleware = require('../middleware/context/locals'); -const credentialsMiddleware = require('../middleware/context/credentials'); +const sendResponse = require('../middleware/send-response'); /** * @param {AuthApi} authApi @@ -21,72 +21,67 @@ function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) { module.exports = NamedMapsAdminController; NamedMapsAdminController.prototype.register = function (app) { - const { base_url_templated } = app; + const { base_url_templated: templateBasePath } = app; app.post( - `${base_url_templated}/`, + `${templateBasePath}/`, cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), - localsMiddleware(), - credentialsMiddleware(), + user(), + credentials(), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), createTemplate({ templateMaps: this.templateMaps }), sendResponse() ); app.put( - `${base_url_templated}/:template_id`, + `${templateBasePath}/:template_id`, cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), - localsMiddleware(), - credentialsMiddleware(), + user(), + credentials(), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), updateTemplate({ templateMaps: this.templateMaps }), sendResponse() ); app.get( - `${base_url_templated}/:template_id`, + `${templateBasePath}/:template_id`, cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), - localsMiddleware(), - credentialsMiddleware(), + user(), + credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), retrieveTemplate({ templateMaps: this.templateMaps }), sendResponse() ); app.delete( - `${base_url_templated}/:template_id`, + `${templateBasePath}/:template_id`, cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), - localsMiddleware(), - credentialsMiddleware(), + user(), + credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), destroyTemplate({ templateMaps: this.templateMaps }), sendResponse() ); app.get( - `${base_url_templated}/`, + `${templateBasePath}/`, cors(), - userMiddleware(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), - localsMiddleware(), - credentialsMiddleware(), + user(), + credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), listTemplates({ templateMaps: this.templateMaps }), sendResponse() ); app.options( - `${base_url_templated}/:template_id`, + `${templateBasePath}/:template_id`, cors('Content-Type') ); }; @@ -224,12 +219,3 @@ function listTemplates ({ templateMaps }) { }); }; } - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(res.statusCode || 200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method](res.body); - }; -} diff --git a/lib/cartodb/middleware/allow-query-params.js b/lib/cartodb/middleware/allow-query-params.js deleted file mode 100644 index cf90e69b..00000000 --- a/lib/cartodb/middleware/allow-query-params.js +++ /dev/null @@ -1,10 +0,0 @@ -module.exports = function allowQueryParams (params) { - if (!Array.isArray(params)) { - throw new Error('allowQueryParams must receive an Array of params'); - } - - return function allowQueryParamsMiddleware (req, res, next) { - res.locals.allowedQueryParams = params; - next(); - }; -}; diff --git a/lib/cartodb/middleware/context/authorize.js b/lib/cartodb/middleware/authorize.js similarity index 100% rename from lib/cartodb/middleware/context/authorize.js rename to lib/cartodb/middleware/authorize.js diff --git a/lib/cartodb/middleware/cache-channel-header.js b/lib/cartodb/middleware/cache-channel-header.js new file mode 100644 index 00000000..d6bf394b --- /dev/null +++ b/lib/cartodb/middleware/cache-channel-header.js @@ -0,0 +1,24 @@ +module.exports = function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider } = res.locals; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Cache Channel Header:', err); + return next(); + } + + if (!affectedTables) { + return next(); + } + + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/middleware/cache-control-header.js new file mode 100644 index 00000000..574ae3b5 --- /dev/null +++ b/lib/cartodb/middleware/cache-control-header.js @@ -0,0 +1,19 @@ +const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365; + +module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) { + return function setCacheControlHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const directives = [ 'public', `max-age=${ttl}` ]; + + if (revalidate) { + directives.push('must-revalidate'); + } + + res.set('Cache-Control', directives.join(',')); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/context/clean-up-query-params.js b/lib/cartodb/middleware/clean-up-query-params.js similarity index 51% rename from lib/cartodb/middleware/context/clean-up-query-params.js rename to lib/cartodb/middleware/clean-up-query-params.js index 280fe986..768b35f2 100644 --- a/lib/cartodb/middleware/context/clean-up-query-params.js +++ b/lib/cartodb/middleware/clean-up-query-params.js @@ -14,19 +14,16 @@ const REQUEST_QUERY_PARAMS_WHITELIST = [ 'filters' // json ]; -module.exports = function cleanUpQueryParamsMiddleware () { - return function cleanUpQueryParams (req, res, next) { - var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST; +module.exports = function cleanUpQueryParamsMiddleware (customQueryParams = []) { + if (!Array.isArray(customQueryParams)) { + throw new Error('customQueryParams must receive an Array of params'); + } - if (Array.isArray(res.locals.allowedQueryParams)) { - allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams); - } + return function cleanUpQueryParams (req, res, next) { + const allowedQueryParams = [...REQUEST_QUERY_PARAMS_WHITELIST, ...customQueryParams]; req.query = _.pick(req.query, allowedQueryParams); - // bring all query values onto res.locals object - _.extend(res.locals, req.query); - next(); }; }; diff --git a/lib/cartodb/middleware/context/index.js b/lib/cartodb/middleware/context/index.js deleted file mode 100644 index 70465895..00000000 --- a/lib/cartodb/middleware/context/index.js +++ /dev/null @@ -1,17 +0,0 @@ -const locals = require('./locals'); -const cleanUpQueryParams = require('./clean-up-query-params'); -const layergroupToken = require('./layergroup-token'); -const credentials = require('./credentials'); -const authorize = require('./authorize'); -const dbConnSetup = require('./db-conn-setup'); - -module.exports = function prepareContextMiddleware(authApi, pgConnection) { - return [ - locals(), - cleanUpQueryParams(), - layergroupToken(), - credentials(), - authorize(authApi), - dbConnSetup(pgConnection) - ]; -}; diff --git a/lib/cartodb/middleware/context/locals.js b/lib/cartodb/middleware/context/locals.js deleted file mode 100644 index f6f70923..00000000 --- a/lib/cartodb/middleware/context/locals.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = function locals () { - return function localsMiddleware (req, res, next) { - res.locals = Object.assign(req.params || {}, res.locals); - - next(); - }; -}; diff --git a/lib/cartodb/middleware/context/credentials.js b/lib/cartodb/middleware/credentials.js similarity index 100% rename from lib/cartodb/middleware/context/credentials.js rename to lib/cartodb/middleware/credentials.js diff --git a/lib/cartodb/middleware/context/db-conn-setup.js b/lib/cartodb/middleware/db-conn-setup.js similarity index 100% rename from lib/cartodb/middleware/context/db-conn-setup.js rename to lib/cartodb/middleware/db-conn-setup.js diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/middleware/error-middleware.js index 85f38936..c02974b4 100644 --- a/lib/cartodb/middleware/error-middleware.js +++ b/lib/cartodb/middleware/error-middleware.js @@ -15,10 +15,6 @@ module.exports = function errorMiddleware (/* options */) { var statusCode = findStatusCode(err); - if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') { - statusCode = 204; - } - setErrorHeader(allErrors, statusCode, res); debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); @@ -186,15 +182,15 @@ function setErrorHeader(errors, statusCode, res) { subtype: error.subtype }; }); - + res.set('X-Tiler-Errors', stringifyForLogs(errorsLog)); } /** - * Remove problematic nested characters + * Remove problematic nested characters * from object for logs RegEx - * - * @param {Object} object + * + * @param {Object} object */ function stringifyForLogs(object) { Object.keys(object).map(key => { diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/middleware/last-modified-header.js new file mode 100644 index 00000000..18c0d961 --- /dev/null +++ b/lib/cartodb/middleware/last-modified-header.js @@ -0,0 +1,45 @@ +module.exports = function setLastModifiedHeader ({ now = false } = {}) { + return function setLastModifiedHeaderMiddleware(req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider, cache_buster } = res.locals; + + if (cache_buster) { + const cacheBuster = parseInt(cache_buster, 10); + const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + return next(); + } + + // REVIEW: to keep 100% compatibility with maps controller + if (now) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Last Modified Header:', err); + return next(); + } + + if (!affectedTables) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + const lastUpdatedAt = affectedTables.getLastUpdatedAt(); + const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/context/layergroup-token.js b/lib/cartodb/middleware/layergroup-token.js similarity index 80% rename from lib/cartodb/middleware/context/layergroup-token.js rename to lib/cartodb/middleware/layergroup-token.js index c4aac23f..1a32e413 100644 --- a/lib/cartodb/middleware/context/layergroup-token.js +++ b/lib/cartodb/middleware/layergroup-token.js @@ -1,17 +1,12 @@ -const LayergroupToken = require('../../models/layergroup-token'); +const LayergroupToken = require('../models/layergroup-token'); const authErrorMessageTemplate = function (signer, user) { return `Cannot use map signature of user "${signer}" on db of user "${user}"`; }; module.exports = function layergroupToken () { return function layergroupTokenMiddleware (req, res, next) { - if (!res.locals.token) { - return next(); - } - const user = res.locals.user; - - const layergroupToken = LayergroupToken.parse(res.locals.token); + const layergroupToken = LayergroupToken.parse(req.params.token); res.locals.token = layergroupToken.token; res.locals.cache_buster = layergroupToken.cacheBuster; diff --git a/lib/cartodb/middleware/rate-limit.js b/lib/cartodb/middleware/rate-limit.js index 9a3daa9b..b3c6d7fe 100644 --- a/lib/cartodb/middleware/rate-limit.js +++ b/lib/cartodb/middleware/rate-limit.js @@ -39,12 +39,16 @@ function rateLimit(userLimitsApi, endpointGroup = null) { res.set({ 'Carto-Rate-Limit-Limit': limit, 'Carto-Rate-Limit-Remaining': remaining, - 'Retry-After': retry, 'Carto-Rate-Limit-Reset': reset }); if (isBlocked) { - const rateLimitError = new Error('You are over the limits.'); + // retry is floor rounded in seconds by redis-cell + res.set('Retry-After', retry + 1); + + let rateLimitError = new Error( + 'You are over platform\'s limits. Please contact us to know more details' + ); rateLimitError.http_status = 429; rateLimitError.type = 'limit'; rateLimitError.subtype = 'rate-limit'; diff --git a/lib/cartodb/middleware/send-response.js b/lib/cartodb/middleware/send-response.js new file mode 100644 index 00000000..469cf0a7 --- /dev/null +++ b/lib/cartodb/middleware/send-response.js @@ -0,0 +1,17 @@ +module.exports = function sendResponse () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + + res.status(res.statusCode || 200); + + if (Buffer.isBuffer(res.body)) { + return res.send(res.body); + } + + if (req.query.callback) { + return res.jsonp(res.body); + } + + res.json(res.body); + }; +}; diff --git a/lib/cartodb/middleware/surrogate-key-header.js b/lib/cartodb/middleware/surrogate-key-header.js new file mode 100644 index 00000000..51cec1c1 --- /dev/null +++ b/lib/cartodb/middleware/surrogate-key-header.js @@ -0,0 +1,31 @@ +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); + +module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { user, mapConfigProvider } = res.locals; + + if (mapConfigProvider instanceof NamedMapMapConfigProvider) { + surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName())); + } + + if (req.method !== 'GET') { + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Surrogate Key Header:', err); + return next(); + } + + if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) { + return next(); + } + + surrogateKeysCache.tag(res, affectedTables); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index 75e93b0a..8268ec1f 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -5,7 +5,7 @@ module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { if(req.params.format === 'mvt') { - if (isTimeoutError(err)) { + if (isTimeoutError(err) || isRateLimitError(err)) { res.set('Content-Type', 'application/x-protobuf'); return res.status(429).send(timeoutErrorVectorTile); } @@ -27,3 +27,7 @@ function isDatasourceTimeoutError (err) { function isTimeoutError (err) { return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); } + +function isRateLimitError (err) { + return err.type === 'limit' && err.subtype === 'rate-limit'; +} diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index b9487095..2bd30152 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -11,6 +11,8 @@ var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasou var MapConfig = require('windshaft').model.MapConfig; +const dbParamsFromReqParams = require('../../../utils/database-params'); + function TurboCartoAdapter() { } @@ -158,23 +160,3 @@ TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) { function shouldParseLayerCartocss(layer) { return layer && layer.options && layer.options.cartocss && layer.options.sql; } - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 340073b5..b3b3f191 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -2,6 +2,7 @@ var assert = require('assert'); var step = require('step'); var MapStoreMapConfigProvider = require('./map-store-provider'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapConfig} mapConfig @@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider'); * @constructor * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) { + +function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapConfig = mapConfig; this.user = user; this.userLimitsApi = userLimitsApi; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.params = params; this.cacheBuster = params.cache_buster || 0; } @@ -23,7 +27,13 @@ module.exports = CreateLayergroupMapConfigProvider; CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig && this.params && this.context) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -31,6 +41,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { function handleRenderLimits(err, renderLimits) { assert.ifError(err); context.limits = renderLimits; + self.context = context; return null; }, function finish(err) { @@ -46,3 +57,52 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter; CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; + +CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + const queries = []; + + this.mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(null, affectedTables); + }); + }); + }); +}; diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index 177322d4..f89a79ce 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var dot = require('dot'); var step = require('step'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapStore} mapStore @@ -11,20 +12,30 @@ var step = require('step'); * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapStore = mapStore; this.user = user; this.userLimitsApi = userLimitsApi; - this.params = params; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.token = params.token; this.cacheBuster = params.cache_buster || 0; + this.mapConfig = null; + this.params = params; + this.context = null; } module.exports = MapStoreMapConfigProvider; MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig !== null) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { self.mapStore.load(self.token, this); }, function finish(err, mapConfig) { + self.mapConfig = mapConfig; + self.context = context; return callback(err, mapConfig, self.params, context); } ); @@ -74,4 +87,54 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) { scale_factor: 1 }); return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); -}; \ No newline at end of file +}; + +MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + + return callback(null, affectedTables); + } + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); +}; diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 9b085dbc..50612065 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables'); * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, - owner, templateId, config, authToken, params) { +function NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache, + owner, + templateId, + config, + authToken, + params +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; @@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, // use template after call to mapConfig this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTablesCache = affectedTablesCache; // providing this.err = null; @@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() { NamedMapMapConfigProvider.prototype.reset = function() { this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTables = null; this.err = null; this.mapConfig = null; @@ -251,39 +262,51 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() { return this.templateName; }; -NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) { - var self = this; - - if (this.affectedTablesAndLastUpdate !== null) { - return callback(null, this.affectedTablesAndLastUpdate); - } - - step( - function getMapConfig() { - self.getMapConfig(this); - }, - function getSql(err, mapConfig) { - assert.ifError(err); - return mapConfig.getLayers().map(function(layer) { - return layer.options.sql; - }).join(';'); - }, - function getAffectedTables(err, sql) { - assert.ifError(err); - step( - function getConnection() { - self.pgConnection.getConnection(self.owner, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, result) { - self.affectedTablesAndLastUpdate = result; - return callback(err, result); +NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); } - ); + + const { dbname } = this.rendererParams; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.owner, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 27b0178f..8535783d 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -49,8 +49,6 @@ var StatsBackend = require('./backends/stats'); const lzmaMiddleware = require('./middleware/lzma'); const errorMiddleware = require('./middleware/error-middleware'); -const prepareContextMiddleware = require('./middleware/context'); - module.exports = function(serverOptions) { // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); @@ -202,7 +200,8 @@ module.exports = function(serverOptions) { pgConnection, metadataBackend, userLimitsApi, - mapConfigAdapter + mapConfigAdapter, + layergroupAffectedTablesCache ); ['update', 'delete'].forEach(function(eventType) { @@ -216,16 +215,11 @@ module.exports = function(serverOptions) { var versions = getAndValidateVersions(serverOptions); - const prepareContext = typeof serverOptions.req2params === 'function' ? - serverOptions.req2params : - prepareContextMiddleware(authApi, pgConnection); - /******************************************************************************************************************* * Routing ******************************************************************************************************************/ new controller.Layergroup( - prepareContext, pgConnection, mapStore, tileBackend, @@ -234,11 +228,11 @@ module.exports = function(serverOptions) { surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - analysisBackend + analysisBackend, + authApi ).register(app); new controller.Map( - prepareContext, pgConnection, templateMaps, mapBackend, @@ -247,23 +241,25 @@ module.exports = function(serverOptions) { userLimitsApi, layergroupAffectedTablesCache, mapConfigAdapter, - statsBackend + statsBackend, + authApi ).register(app); new controller.NamedMaps( - prepareContext, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend, + pgConnection, + authApi, userLimitsApi ).register(app); new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app); - new controller.Analyses(prepareContext, userLimitsApi).register(app); + new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app); new controller.ServerInfo(versions).register(app); diff --git a/lib/cartodb/utils/database-params.js b/lib/cartodb/utils/database-params.js new file mode 100644 index 00000000..a2176eaf --- /dev/null +++ b/lib/cartodb/utils/database-params.js @@ -0,0 +1,25 @@ +module.exports = function getDatabaseConnectionParams (params) { + const dbParams = {}; + + if (params.dbuser) { + dbParams.user = params.dbuser; + } + + if (params.dbpassword) { + dbParams.pass = params.dbpassword; + } + + if (params.dbhost) { + dbParams.host = params.dbhost; + } + + if (params.dbport) { + dbParams.port = params.dbport; + } + + if (params.dbname) { + dbParams.dbname = params.dbname; + } + + return dbParams; +}; diff --git a/package.json b/package.json index b72f9b8c..8b222d24 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "6.0.0", + "version": "6.0.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" diff --git a/run_tests.sh b/run_tests.sh index 8412a851..f26cad6d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -50,17 +50,6 @@ die() { exit 1 } -get_redis_cell() { - if test x"$OPT_REDIS_CELL" = xyes; then - echo "Downloading redis-cell" - curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1 - tar xvzf redis-cell.tar.gz > /dev/null 2>&1 - mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so - rm redis-cell.tar.gz - rm libredis_cell.d - fi -} - trap 'cleanup_and_exit' 1 2 3 5 9 13 while [ -n "$1" ]; do @@ -122,9 +111,13 @@ fi TESTS=$@ if test x"$OPT_CREATE_REDIS" = xyes; then - get_redis_cell echo "Starting redis on port ${REDIS_PORT}" - echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log & + REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.so" + if [[ "$OSTYPE" == "darwin"* ]]; then + REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.dylib" + fi + + echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${REDIS_CELL_PATH} > ${BASEDIR}/test.log & PID_REDIS=$! echo ${PID_REDIS} > ${BASEDIR}/redis.pid fi diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index a1a93e40..b4840879 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1272,6 +1272,8 @@ describe(suiteName, function() { it("cache control for layergroup default value", function(done) { global.environment.varnish.layergroupTtl = null; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate'); @@ -1287,6 +1289,8 @@ describe(suiteName, function() { var layergroupTtl = 300; global.environment.varnish.layergroupTtl = layergroupTtl; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate'); @@ -1336,7 +1340,7 @@ describe(suiteName, function() { status: 403 }, function(res) { - assert.ok(res.body.match(/permission denied for relation test_table_private_1/)); + assert.ok(res.body.match(/permission denied for .+?test_table_private_1/)); done(); } ); diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js index 5844c479..2090ffe2 100644 --- a/test/acceptance/named_layers.js +++ b/test/acceptance/named_layers.js @@ -659,7 +659,7 @@ describe('named_layers', function() { } var parsedBody = JSON.parse(response.body); - assert.ok(parsedBody.errors[0].match(/permission denied for relation test_table_private_1/)); + assert.ok(parsedBody.errors[0].match(/permission denied for .+?test_table_private_1/)); return null; }, diff --git a/test/acceptance/ported/attributes.js b/test/acceptance/ported/attributes.js index f19fb8e8..8b36c3e3 100644 --- a/test/acceptance/ported/attributes.js +++ b/test/acceptance/ported/attributes.js @@ -1,10 +1,9 @@ var testHelper = require('../../support/test_helper'); -var assert = require('../../support/assert'); -var step = require('step'); +var assert = require('../../support/assert'); +var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var PortedServerOptions = require('./support/ported_server_options'); - var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('attributes', function() { @@ -47,132 +46,147 @@ describe('attributes', function() { }); it("can only be fetched from layer having an attributes spec", function(done) { - - var expected_token; - step( - function do_post() - { - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(test_mapconfig_1) - }, {}, function(res, err) { next(err, res); }); - }, - function checkPost(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - // CORS headers should be sent with response - // from layergroup creation via POST - checkCORSHeaders(res); - var parsedBody = JSON.parse(res.body); - if ( expected_token ) { - assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2}); - } else { - expected_token = parsedBody.layergroupid; - } - return null; - }, - function do_get_attr_0(err) - { - assert.ifError(err); - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1', - method: 'GET' - }, {}, function(res, err) { next(err, res); }); - }, - function check_error_0(err, res) { - assert.ifError(err); - assert.equal( - res.statusCode, - 400, - res.statusCode + ( res.statusCode !== 200 ? (': ' + res.body) : '' ) - ); - var parsed = JSON.parse(res.body); - assert.equal(parsed.errors[0], "Layer 0 has no exposed attributes"); - return null; - }, - function do_get_attr_1(err) - { - assert.ifError(err); - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1', - method: 'GET' - }, {}, function(res, err) { next(err, res); }); - }, - function check_attr_1(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - var parsed = JSON.parse(res.body); - assert.deepEqual(parsed, {"n":6}); - return null; - }, - function do_get_attr_1_404(err) - { - assert.ifError(err); - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/-666', - method: 'GET' - }, {}, function(res, err) { next(err, res); }); - }, - function check_attr_1_404(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body); - var parsed = JSON.parse(res.body); - assert.ok(parsed.errors); - var msg = parsed.errors[0]; - assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1"); - return null; - }, - function finish(err) { - keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - - done(err); + var expected_token; + step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(test_mapconfig_1) + }, {}, function(res, err) { next(err, res); }); + }, + function checkPost(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + // CORS headers should be sent with response + // from layergroup creation via POST + checkCORSHeaders(res); + var parsedBody = JSON.parse(res.body); + if ( expected_token ) { + assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2}); + } else { + expected_token = parsedBody.layergroupid; } - ); - }); + return null; + }, + function do_get_attr_0(err) + { + assert.ifError(err); + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1', + method: 'GET', + headers: { + host: 'localhost' + }, + }, {}, function(res, err) { next(err, res); }); + }, + function check_error_0(err, res) { + assert.ifError(err); + assert.equal( + res.statusCode, + 400, + res.statusCode + ( res.statusCode !== 200 ? (': ' + res.body) : '' ) + ); + var parsed = JSON.parse(res.body); + assert.equal(parsed.errors[0], "Layer 0 has no exposed attributes"); + return null; + }, + function do_get_attr_1(err) + { + assert.ifError(err); + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1', + method: 'GET', + headers: { + host: 'localhost' + } + }, {}, function(res, err) { next(err, res); }); + }, + function check_attr_1(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var parsed = JSON.parse(res.body); + assert.deepEqual(parsed, {"n":6}); + return null; + }, + function do_get_attr_1_404(err) + { + assert.ifError(err); + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/-666', + method: 'GET', + headers: { + host: 'localhost' + } + }, {}, function(res, err) { next(err, res); }); + }, + function check_attr_1_404(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body); + var parsed = JSON.parse(res.body); + assert.ok(parsed.errors); + var msg = parsed.errors[0]; + assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1"); + return null; + }, + function finish(err) { + keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + + done(err); + } + ); + }); // See https://github.com/CartoDB/Windshaft/issues/131 it("are checked at map creation time", function(done) { - // clone the mapconfig test - var mapconfig = JSON.parse(JSON.stringify(test_mapconfig_1)); - // append unexistant attribute name - mapconfig.layers[1].options.sql = 'SELECT * FROM test_table'; - mapconfig.layers[1].options.attributes.id = 'unexistant'; - mapconfig.layers[1].options.attributes.columns = ['cartodb_id']; + // clone the mapconfig test + var mapconfig = JSON.parse(JSON.stringify(test_mapconfig_1)); + // append unexistant attribute name + mapconfig.layers[1].options.sql = 'SELECT * FROM test_table'; + mapconfig.layers[1].options.attributes.id = 'unexistant'; + mapconfig.layers[1].options.attributes.columns = ['cartodb_id']; - step( - function do_post() - { - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(mapconfig) - }, {}, function(res, err) { next(err, res); }); - }, - function checkPost(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 404, res.statusCode + ': ' + (res.statusCode===200?'...':res.body)); - var parsed = JSON.parse(res.body); - assert.ok(parsed.errors); - assert.equal(parsed.errors.length, 1); - var msg = parsed.errors[0]; - assert.equal(msg, 'column "unexistant" does not exist'); - return null; - }, - function finish(err) { - done(err); - } - ); - }); + step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/database/windshaft_test/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(mapconfig) + }, {}, function(res, err) { next(err, res); }); + }, + function checkPost(err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 404, res.statusCode + ': ' + (res.statusCode===200?'...':res.body)); + var parsed = JSON.parse(res.body); + assert.ok(parsed.errors); + assert.equal(parsed.errors.length, 1); + var msg = parsed.errors[0]; + assert.equal(msg, 'column "unexistant" does not exist'); + return null; + }, + function finish(err) { + done(err); + } + ); + }); it("can be used with jsonp", function(done) { @@ -184,7 +198,10 @@ describe('attributes', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(test_mapconfig_1) }, {}, function(res, err) { next(err, res); }); }, @@ -209,7 +226,10 @@ describe('attributes', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1?callback=test', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_error_0(err, res) { @@ -232,7 +252,10 @@ describe('attributes', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_attr_1(err, res) { @@ -270,7 +293,10 @@ describe('attributes', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, diff --git a/test/acceptance/ported/multilayer.js b/test/acceptance/ported/multilayer.js index 0ad9f5de..9da57948 100644 --- a/test/acceptance/ported/multilayer.js +++ b/test/acceptance/ported/multilayer.js @@ -44,7 +44,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json; charset=utf-8' }, + headers: { host: 'localhost', 'Content-Type': 'application/json; charset=utf-8' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -85,7 +85,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -101,7 +101,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -154,7 +155,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); @@ -177,7 +178,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -194,7 +196,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -212,7 +215,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -266,7 +270,7 @@ describe('multilayer', function() { config: JSON.stringify(layergroup) }), method: 'GET', - headers: {'Content-Type': 'application/json' } + headers: { host: 'localhost', 'Content-Type': 'application/json' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); // CORS headers should be sent with response @@ -289,7 +293,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -307,7 +312,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -325,7 +331,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -380,7 +387,7 @@ describe('multilayer', function() { callback: 'jsonp_test' }), method: 'GET', - headers: {'Content-Type': 'application/json' } + headers: { host: 'localhost', 'Content-Type': 'application/json' } }, {}, function(res, err) { next(err, res); }); }, function do_check_token(err, res) { @@ -413,7 +420,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -431,7 +439,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -449,7 +458,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -510,7 +520,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err, res); @@ -530,7 +540,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -548,7 +559,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -566,7 +578,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -583,7 +596,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/4', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); @@ -602,7 +616,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/2/0/0/0.json.torque', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); }, function do_check_torque2(err, res) { @@ -624,7 +639,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.json.torque', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); }, function do_check_torque1(err, res) { @@ -684,7 +700,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup1) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -700,7 +716,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup2) }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -717,7 +733,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -734,7 +751,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -752,7 +770,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + token2 + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -769,7 +788,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + token2 + '/0/0/0/0.grid.json?interactivity=cartodb_id', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/json; charset=utf-8"); @@ -826,7 +846,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -848,7 +868,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); @@ -893,7 +914,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); @@ -922,7 +943,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); @@ -957,7 +978,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: _.template(tpl, {font:'bogus'}) }, function(res) { next(null, res); }); }, @@ -977,7 +998,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {host: 'localhost', 'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: _.template(tpl, {font:available_system_fonts[0]}) }, function(res) { next(null, res); }); }, @@ -1040,7 +1061,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -1061,7 +1082,8 @@ describe('multilayer', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json', - method: 'GET' + method: 'GET', + headers: { host: 'localhost' } }, {}, function(res) { next(null, res); }); @@ -1093,63 +1115,8 @@ describe('multilayer', function() { ); }); - // See http://github.com/CartoDB/Windshaft/issues/157 - it("req2params is called only once for a multilayer post", function(done) { - - var layergroup = { - version: '1.0.1', - layers: [ - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2', - cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', - cartocss_version: '2.0.1', - interactivity: [ 'cartodb_id' ], - geom_column: 'the_geom' - } }, - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom, -50, 0) as the_geom from test_table limit 2 offset 2', - cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', - cartocss_version: '2.0.2', - interactivity: [ 'cartodb_id' ], - geom_column: 'the_geom' - } } - ] - }; - - var expected_token; - step( - function do_post() - { - global.req2params_calls = 0; - var next = this; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(layergroup) - }, {}, function(res, err) { next(err,res); }); - }, - function check_post(err, res) { - assert.ifError(err); - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - var parsedBody = JSON.parse(res.body); - expected_token = LayergroupToken.parse(parsedBody.layergroupid).token; - assert.equal(global.req2params_calls, 1); - return null; - }, - function finish(err) { - if (err) { - return done(err); - } - var keysToDelete = {'user:localhost:mapviews:global': 5}; - keysToDelete['map_cfg|' + expected_token] = 0; - testHelper.deleteRedisKeys(keysToDelete, done); - } - ); - }); - // See https://github.com/CartoDB/Windshaft/issues/163 - it("has different token for different database", function(done) { + it.skip("has different token for different database", function(done) { var layergroup = { version: '1.0.1', layers: [ @@ -1170,7 +1137,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err,res); }); }, @@ -1187,7 +1154,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test2/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'cartodb250user', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err,res); }); }, @@ -1233,7 +1200,7 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res, err) { next(err,res); }); }, @@ -1251,7 +1218,8 @@ describe('multilayer', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index 6e7f5964..5752a9b5 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -24,7 +24,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/x-www-form-urlencoded' } + headers: { + host: 'localhost', + 'Content-Type': 'application/x-www-form-urlencoded' + } }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); var parsedBody = JSON.parse(res.body); @@ -37,7 +40,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' } + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); var parsedBody = JSON.parse(res.body); @@ -50,7 +56,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup?callback=test', method: 'POST', - headers: {'Content-Type': 'application/json' } + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } }, {}, function(res) { assert.equal(res.statusCode, 200); assert.equal( @@ -65,27 +74,30 @@ describe('multilayer error cases', function() { }); it("layergroup with no cartocss_version", function(done) { - var layergroup = { - version: '1.0.0', - layers: [ - { options: { - sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2', - cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', - geom_column: 'the_geom' - } } - ] - }; - assert.response(server, { - url: '/database/windshaft_test/layergroup', - method: 'POST', - headers: {'Content-Type': 'application/json' }, - data: JSON.stringify(layergroup) - }, {}, function(res) { - assert.equal(res.statusCode, 400, res.body); - var parsedBody = JSON.parse(res.body); - assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]); - done(); - }); + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + geom_column: 'the_geom' + } } + ] + }; + assert.response(server, { + url: '/database/windshaft_test/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 400, res.body); + var parsedBody = JSON.parse(res.body); + assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]); + done(); + }); }); it("sql/cartocss combination errors", function(done) { @@ -98,17 +110,18 @@ describe('multilayer error cases', function() { geom_column: 'the_geom' }}] }; - ServerOptions.afterLayergroupCreateCalls = 0; assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { try { assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); // See http://github.com/CartoDB/Windshaft/issues/159 - assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); var parsed = JSON.parse(res.body); assert.ok(parsed); assert.equal(parsed.errors.length, 1); @@ -149,12 +162,9 @@ describe('multilayer error cases', function() { }} ] }; - ServerOptions.afterLayergroupCreateCalls = 0; this.client = new TestClient(layergroup); this.client.getLayergroup({ response: { status: 400 } }, function(err, parsed) { assert.ok(!err, err); - // See http://github.com/CartoDB/Windshaft/issues/159 - assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); assert.ok(parsed); assert.equal(parsed.errors.length, 1); var error = parsed.errors[0]; @@ -186,7 +196,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -222,7 +235,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { try { @@ -264,7 +280,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, data: JSON.stringify(layergroup) }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); @@ -367,7 +386,10 @@ describe('multilayer error cases', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/deadbeef/0/0/0/0.grid.json', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function checkResponse(err, res) { diff --git a/test/acceptance/ported/multilayer_interactivity.js b/test/acceptance/ported/multilayer_interactivity.js index 4542f710..ac406725 100644 --- a/test/acceptance/ported/multilayer_interactivity.js +++ b/test/acceptance/ported/multilayer_interactivity.js @@ -23,13 +23,12 @@ describe('multilayer interactivity and layers order', function() { layers: testScenario.layers }; - PortedServerOptions.afterLayergroupCreateCalls = 0; - assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) @@ -49,8 +48,6 @@ describe('multilayer interactivity and layers order', function() { '\n\tLayer types: ' + layergroup.layers.map(layerType).join(', ') ); -// assert.equal(PortedServerOptions.afterLayergroupCreateCalls, 1); - var layergroupResponse = JSON.parse(response.body); assert.ok(layergroupResponse); diff --git a/test/acceptance/ported/raster.js b/test/acceptance/ported/raster.js index 448661f2..6425e090 100644 --- a/test/acceptance/ported/raster.js +++ b/test/acceptance/ported/raster.js @@ -41,7 +41,7 @@ describe('raster', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, @@ -66,7 +66,8 @@ describe('raster', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { host: 'localhost' } }, {}, function(res, err) { next(err, res); }); }, function check_response(err, res) { @@ -124,6 +125,7 @@ describe('raster', function() { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) diff --git a/test/acceptance/ported/regressions.js b/test/acceptance/ported/regressions.js index dd765b57..71fd8388 100644 --- a/test/acceptance/ported/regressions.js +++ b/test/acceptance/ported/regressions.js @@ -8,21 +8,6 @@ describe('regressions', function() { testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); - // See https://github.com/Vizzuality/Windshaft/issues/65 - it("#65 catching non-Error exception doesn't kill the backend", function(done) { - var mapConfig = testClient.defaultTableMapConfig('test_table'); - testClient.withLayergroup(mapConfig, function(err, requestTile, finish) { - var options = { - statusCode: 400, - contentType: 'application/json; charset=utf-8' - }; - requestTile('/0/0/0.png?testUnexpectedError=1', options, function(err, res) { - assert.deepEqual(JSON.parse(res.body).errors, ["test unexpected error"]); - finish(done); - }); - }); - }); - // Test that you cannot write to the database from a tile request // // See http://github.com/CartoDB/Windshaft/issues/130 diff --git a/test/acceptance/ported/retina.js b/test/acceptance/ported/retina.js index 7469214b..137f9738 100644 --- a/test/acceptance/ported/retina.js +++ b/test/acceptance/ported/retina.js @@ -38,6 +38,7 @@ describe('retina support', function() { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(retinaSampleMapConfig) @@ -67,7 +68,10 @@ describe('retina support', function() { { url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/0/0' + scaleFactor + '.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }, responseHead, assertFn diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index f3c9ba81..f40b680b 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -59,7 +59,7 @@ describe('server_gettile', function() { assert.ok(xwc > 0); assert.ok(xwc >= lastXwc); - requestTile(tileUrl + '?cache_buster=wadus', function (err, res) { + requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) { var xwc = res.headers['x-windshaft-cache']; assert.ok(!xwc); @@ -99,18 +99,25 @@ describe('server_gettile', function() { } testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) { - requestTile(tileUrl, function(err, res) { - assert.ok(res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time"); - assert.ok(res.headers['x-windshaft-cache'] >= 0); + var xwc = res.headers['x-windshaft-cache']; + assert.ok(!xwc); - assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, - function(err) { - finish(function(finishErr) { - done(err || finishErr); - }); - } - ); + requestTile(tileUrl, function (err, res) { + assert.ok( + res.headers.hasOwnProperty('x-windshaft-cache'), + "Did not hit renderer cache on second time" + ); + assert.ok(res.headers['x-windshaft-cache'] >= 0); + + assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, + function(err) { + finish(function(finishErr) { + done(err || finishErr); + }); + } + ); + }); }); }); }); diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 18151330..ee565498 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -62,6 +62,7 @@ describe('server_png8_format', function() { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) @@ -81,7 +82,10 @@ describe('server_png8_format', function() { var requestPayload = { url: '/database/windshaft_test/layergroup/' + layergroupId + tilePartialUrl, method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }; var requestHeaders = { @@ -179,4 +183,3 @@ describe('server_png8_format', function() { }); }); }); - diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 8b75ec02..8b8ec339 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -1,7 +1,6 @@ var _ = require('underscore'); var serverOptions = require('../../../../lib/cartodb/server_options'); var mapnik = require('windshaft').mapnik; -var LayergroupToken = require('../../../../lib/cartodb/models/layergroup-token'); var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' @@ -47,48 +46,5 @@ module.exports = _.extend({}, serverOptions, { enable_cors: global.environment.enable_cors, unbuffered_logging: true, // for smoother teardown from tests log_format: null, // do not log anything - afterLayergroupCreateCalls: 0, - useProfiler: true, - req2params: function(req, res, callback){ - - if ( req.query.testUnexpectedError ) { - return callback('test unexpected error'); - } - - // this is in case you want to test sql parameters eg ...png?sql=select * from my_table limit 10 - req.params = _.extend({}, req.params); - - if (req.params.token) { - req.params.token = LayergroupToken.parse(req.params.token).token; - } - - _.extend(req.params, req.query); - req.params.user = 'localhost'; - res.locals.user = 'localhost'; - - req.params.dbhost = global.environment.postgres.host; - req.params.dbport = req.params.dbport || global.environment.postgres.port; - - req.params.dbuser = 'test_windshaft_publicuser'; - if (req.params.dbname !== 'windshaft_test2') { - req.params.dbuser = 'test_windshaft_cartodb_user_1'; - } - req.params.dbname = 'test_windshaft_cartodb_user_1_db'; - - // add all params to res.locals - res.locals = _.extend({}, req.params); - - - // increment number of calls counter - global.req2params_calls = global.req2params_calls ? global.req2params_calls + 1 : 1; - - // send the finished req object on - callback(null,req); - }, - afterLayergroupCreate: function(req, cfg, res, callback) { - res.layercount = cfg.layers.length; -// config.afterLayergroupCreateCalls++; - callback(null); - } - + useProfiler: true }); diff --git a/test/acceptance/ported/support/test_client.js b/test/acceptance/ported/support/test_client.js index 640f1b64..ab4576b6 100644 --- a/test/acceptance/ported/support/test_client.js +++ b/test/acceptance/ported/support/test_client.js @@ -72,7 +72,7 @@ function createLayergroup(layergroupConfig, options, callback) { }); }, function validateLayergroup(err, res) { - assert.ok(!err, 'Failed to request layergroup'); + assert.ifError(err); var parsedBody; var layergroupid; @@ -129,6 +129,7 @@ function layergroupRequest(layergroupConfig, method, callbackName, extraParams) var request = { url: '/database/windshaft_test/layergroup', headers: { + host: 'localhost', 'Content-Type': 'application/json' } }; @@ -337,6 +338,7 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) { url: '/database/windshaft_test/layergroup', method: 'POST', headers: { + host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroupConfig) @@ -372,7 +374,10 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) { var request = { url: finalUrl, - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }; if (contentType === pngContentType) { @@ -449,12 +454,28 @@ function withLayergroup(layergroupConfig, options, callback) { }; } - var baseUrlTpl = '/database/windshaft_test/layergroup/<%= layergroupid %>'; - var finalUrl = _.template(baseUrlTpl, { layergroupid: layergroupid }) + layergroupUrl; + const signerTpl = function ({ signer }) { + return `${signer ? `:${signer}@` : ''}`; + }; + + const cacheTpl = function ({ cache_buster, cacheBuster }) { + return `${cache_buster ? `:${cache_buster}` : `:${cacheBuster}`}`; + }; + + const urlTpl = function ({layergroupid, cache_buster = null, tile }) { + const { signer, token , cacheBuster } = LayergroupToken.parse(layergroupid); + const base = '/database/windshaft_test/layergroup/'; + return `${base}${signerTpl({signer})}${token}${cacheTpl({cache_buster, cacheBuster})}${tile}`; + }; + + const finalUrl = urlTpl({ layergroupid, cache_buster: options.cache_buster, tile: layergroupUrl }); var request = { url: finalUrl, - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }; if (options.contentType === pngContentType) { diff --git a/test/acceptance/ported/torque.js b/test/acceptance/ported/torque.js index 47ae2eee..6d761b3d 100644 --- a/test/acceptance/ported/torque.js +++ b/test/acceptance/ported/torque.js @@ -48,7 +48,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -71,7 +71,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -94,7 +94,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -136,7 +136,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { next(null, res); }); }, @@ -179,7 +179,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, @@ -218,7 +218,10 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png', method: 'GET', - encoding: 'binary' + encoding: 'binary', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_mapnik_error_1(err, res) { @@ -235,7 +238,10 @@ describe('torque', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_mapnik_error_2(err, res) { @@ -252,7 +258,10 @@ describe('torque', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_torque0_response(err, res) { @@ -270,7 +279,10 @@ describe('torque', function() { var next = this; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.torque.json', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function(res, err) { next(err, res); }); }, function check_torque0_response_1(err, res) { @@ -315,7 +327,7 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, @@ -354,19 +366,25 @@ describe('torque', function() { ] }; + let defautlPort = global.environment.postgres.port; + step( function do_post() { var next = this; + global.environment.postgres.port = 54777; assert.response(server, { - url: '/database/windshaft_test/layergroup?dbport=54777', + url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(mapconfig) }, {}, function(res, err) { next(err, res); }); }, function checkPost(err, res) { assert.ifError(err); + + global.environment.postgres.port = defautlPort; + assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body); var parsed = JSON.parse(res.body); assert.ok(parsed.errors, parsed); @@ -407,9 +425,9 @@ describe('torque', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) - }, {}, function(res) { next(null, res); }); + }, {}, function (res) { next(null, res); }); }, function checkResponse(err, res) { assert.ifError(err); diff --git a/test/acceptance/ported/torque_boundaries.js b/test/acceptance/ported/torque_boundaries.js index 9b4619b1..4dde46a0 100644 --- a/test/acceptance/ported/torque_boundaries.js +++ b/test/acceptance/ported/torque_boundaries.js @@ -237,7 +237,7 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(boundaryPointsMapConfig) }, {}, function (res, err) { @@ -250,7 +250,10 @@ describe('torque boundary points', function() { var partialUrl = tileRequest.z + '/' + tileRequest.x + '/' + tileRequest.y; assert.response(server, { url: '/database/windshaft_test/layergroup/' + expected_token + '/0/' + partialUrl + '.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function (res, err) { assert.ok(!err, 'Failed to get json'); @@ -351,7 +354,7 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(londonPointMapConfig) }, {}, function (res, err) { assert.ok(!err, 'Failed to create layergroup'); @@ -363,7 +366,10 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/2/1/1.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function (res, err) { assert.ok(!err, 'Failed to request torque.json'); @@ -414,7 +420,7 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup', method: 'POST', - headers: {'Content-Type': 'application/json' }, + headers: { host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(londonPointMapConfig) }, {}, function (res, err) { assert.ok(!err, 'Failed to create layergroup'); @@ -426,7 +432,10 @@ describe('torque boundary points', function() { assert.response(server, { url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/13/4255/2765.json.torque', - method: 'GET' + method: 'GET', + headers: { + host: 'localhost' + } }, {}, function (res, err) { assert.ok(!err, 'Failed to request torque.json'); diff --git a/test/acceptance/rate-limit.test.js b/test/acceptance/rate-limit.test.js index 339e187e..eb9f8954 100644 --- a/test/acceptance/rate-limit.test.js +++ b/test/acceptance/rate-limit.test.js @@ -15,6 +15,7 @@ let redisClient; let testClient; let keysToDelete = ['user:localhost:mapviews:global']; const user = 'localhost'; +let layergroupid; const query = ` SELECT @@ -68,13 +69,13 @@ const createMapConfig = ({ }); -function setLimit(count, period, burst) { +function setLimit(count, period, burst, endpoint = RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) { redisClient.SELECT(8, err => { if (err) { return; } - const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`; + const key = `limits:rate:store:${user}:maps:${endpoint}`; redisClient.rpush(key, burst); redisClient.rpush(key, count); redisClient.rpush(key, period); @@ -87,8 +88,12 @@ function getReqAndRes() { req: {}, res: { headers: {}, - set(headers) { - this.headers = headers; + set(headers, value) { + if(typeof headers === 'object') { + this.headers = headers; + } else { + this.headers[headers] = value; + } }, locals: { user: 'localhost' @@ -97,18 +102,21 @@ function getReqAndRes() { }; } -function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) { - const response = { +function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done) { + let response = { status, headers: { 'Content-Type': 'application/json; charset=utf-8', 'Carto-Rate-Limit-Limit': limit, 'Carto-Rate-Limit-Remaining': remaining, - 'Carto-Rate-Limit-Reset': reset, - 'Retry-After': retry + 'Carto-Rate-Limit-Reset': reset } }; + if(retry) { + response.headers['Retry-After'] = retry; + } + testClient.getLayergroup({ response }, err => { assert.ifError(err); if (done) { @@ -117,21 +125,26 @@ function assertGetLayergroupRequest (status, limit, remaining, reset, retry, don }); } -function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) { +function assertRateLimitRequest (status, limit, remaining, reset, retry, done) { const { req, res } = getReqAndRes(); rateLimit(req, res, function (err) { - assert.deepEqual(res.headers, { + let expectedHeaders = { "Carto-Rate-Limit-Limit": limit, "Carto-Rate-Limit-Remaining": remaining, - "Carto-Rate-Limit-Reset": reset, - "Retry-After": retry - }); + "Carto-Rate-Limit-Reset": reset + }; + + if(retry) { + expectedHeaders['Retry-After'] = retry; + } + + assert.deepEqual(res.headers, expectedHeaders); if(status === 200) { assert.ifError(err); } else { assert.ok(err); - assert.equal(err.message, 'You are over the limits.'); + assert.equal(err.message, 'You are over platform\'s limits. Please contact us to know more details'); assert.equal(err.http_status, 429); assert.equal(err.type, 'limit'); assert.equal(err.subtype, 'rate-limit'); @@ -178,7 +191,7 @@ describe('rate limit', function() { const burst = 1; setLimit(count, period, burst); - assertGetLayergroupRequest(200, '2', '1', '1', '-1', done); + assertGetLayergroupRequest(200, '2', '1', '1', null, done); }); it("1 req/sec: 2 req/seg should be limited", function(done) { @@ -187,12 +200,12 @@ describe('rate limit', function() { const burst = 1; setLimit(count, period, burst); - assertGetLayergroupRequest(200, '2', '1', '1', '-1'); - setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250); - setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500); - setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750); - setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950); - setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050); + assertGetLayergroupRequest(200, '2', '1', '1'); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1'), 250); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 500); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 750); + setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 950); + setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', null, done), 1050); }); }); @@ -234,12 +247,12 @@ describe('rate limit middleware', function () { }); it("1 req/sec: 2 req/seg should be limited", function (done) { - assertRateLimitRequest(200, 1, 0, 1, -1); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); - setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); + assertRateLimitRequest(200, 1, 0, 1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 250); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050); }); it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) { @@ -248,13 +261,100 @@ describe('rate limit middleware', function () { 'SCRIPT', ['FLUSH'], function () { - assertRateLimitRequest(200, 1, 0, 1, -1); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750); - setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950); - setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050); + assertRateLimitRequest(200, 1, 0, 1); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750); + setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950); + setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050); } ); }); }); + +describe('rate limit and vector tiles', function () { + + before(function(done) { + global.environment.enabledFeatures.rateLimitsEnabled = true; + global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true; + + redisClient = redis.createClient(global.environment.redis.port); + const count = 1; + const period = 1; + const burst = 0; + setLimit(count, period, burst, RATE_LIMIT_ENDPOINTS_GROUPS.TILE); + + testClient = new TestClient(createMapConfig(), 1234); + testClient.getLayergroup({status: 200}, (err, res) => { + assert.ifError(err); + + layergroupid = res.layergroupid; + + done(); + }); + }); + + after(function() { + global.environment.enabledFeatures.rateLimitsEnabled = false; + global.environment.enabledFeatures.rateLimitsByEndpoint.tile = false; + }); + + afterEach(function(done) { + keysToDelete.forEach( key => { + redisClient.del(key); + }); + + redisClient.SELECT(0, () => { + redisClient.del('user:localhost:mapviews:global'); + + redisClient.SELECT(5, () => { + redisClient.del('user:localhost:mapviews:global'); + done(); + }); + }); + }); + + it('mvt rate limited', function (done) { + const tileParams = (status, limit, remaining, reset, retry, contentType) => { + let headers = { + "Content-Type": contentType, + "Carto-Rate-Limit-Limit": limit, + "Carto-Rate-Limit-Remaining": remaining, + "Carto-Rate-Limit-Reset": reset + }; + + if (retry) { + headers['Retry-After'] = retry; + } + + return { + layergroupid: layergroupid, + format: 'mvt', + response: {status, headers} + }; + }; + + testClient.getTile(0, 0, 0, tileParams(204, '1', '0', '1'), (err) => { + assert.ifError(err); + + testClient.getTile( + 0, + 0, + 0, + tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'), + (err, res, tile) => { + assert.ifError(err); + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + + done(); + } + ); + }); + + }); +}); diff --git a/test/acceptance/turbo-carto/regressions.js b/test/acceptance/turbo-carto/regressions.js index 9e2d377f..3f3ebc57 100644 --- a/test/acceptance/turbo-carto/regressions.js +++ b/test/acceptance/turbo-carto/regressions.js @@ -88,7 +88,7 @@ describe('turbo-carto regressions', function() { var turboCartoError = layergroup.errors_with_context[0]; assert.ok(turboCartoError); assert.equal(turboCartoError.type, 'layer'); - assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\srelation\stest_table_private_1/)); + assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\s.+?test_table_private_1/)); done(); }); diff --git a/test/support/libredis_cell.dylib b/test/support/libredis_cell.dylib new file mode 100755 index 00000000..7c7d6c88 Binary files /dev/null and b/test/support/libredis_cell.dylib differ diff --git a/test/support/libredis_cell.so b/test/support/libredis_cell.so new file mode 100755 index 00000000..1da8aa66 Binary files /dev/null and b/test/support/libredis_cell.so differ diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index 283c4fdc..ff311cc6 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -7,11 +7,10 @@ var PgConnection = require('../../../lib/cartodb/backends/pg_connection'); var AuthApi = require('../../../lib/cartodb/api/auth_api'); var TemplateMaps = require('../../../lib/cartodb/backends/template_maps'); -const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/context/clean-up-query-params'); -const authorizeMiddleware = require('../../../lib/cartodb/middleware/context/authorize'); -const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/context/db-conn-setup'); -const credentialsMiddleware = require('../../../lib/cartodb/middleware/context/credentials'); -const localsMiddleware = require('../../../lib/cartodb/middleware/context/locals'); +const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/clean-up-query-params'); +const authorizeMiddleware = require('../../../lib/cartodb/middleware/authorize'); +const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/db-conn-setup'); +const credentialsMiddleware = require('../../../lib/cartodb/middleware/credentials'); var windshaft = require('windshaft'); @@ -66,18 +65,6 @@ describe('prepare-context', function() { return res; } - it('res.locals are created', function(done) { - const locals = localsMiddleware(); - let req = {}; - let res = {}; - - locals(prepareRequest(req), prepareResponse(res), function(err) { - if ( err ) { done(err); return; } - assert.ok(res.hasOwnProperty('locals'), 'response has locals'); - done(); - }); - }); - it('cleans up request', function(done){ var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}; var res = {}; @@ -178,10 +165,9 @@ describe('prepare-context', function() { return done(err); } - var query = res.locals; - assert.deepEqual(config, query.config); - assert.equal('test', query.api_key); - assert.equal(undefined, query.non_included); + assert.deepEqual(config, req.query.config); + assert.equal('test', req.query.api_key); + assert.equal(undefined, req.query.non_included); done(); }); });