diff --git a/NEWS.md b/NEWS.md index 6312c5bb..409b5ebc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,11 @@ # Changelog -## 4.0.2 +## 4.1.0 Released 2017-mm-dd Announcements: - Upgrades windshaft to [4.0.1](https://github.com/CartoDB/windshaft/releases/tag/4.0.1). + - Add `categories` query param to define the number of categories to be ranked for aggregation dataviews. ## 4.0.1 diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index b6037ae6..9df22868 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -24,7 +24,7 @@ module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { - var dataviewName = params.dataviewName; + var dataviewName = params.dataviewName; step( function getMapConfig() { mapConfigProvider.getMapConfig(this); @@ -94,7 +94,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { } function getOverrideParams(params, ownFilter) { - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset', 'categories'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 7aabc11b..ffc0bea1 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -3,21 +3,13 @@ var assert = require('assert'); var dot = require('dot'); dot.templateSettings.strip = false; var PSQL = require('cartodb-psql'); - -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); - function AnalysesController(prepareContext) { - BaseController.call(this); this.prepareContext = prepareContext; } -util.inherits(AnalysesController, BaseController); - module.exports = AnalysesController; AnalysesController.prototype.register = function(app) { @@ -30,9 +22,16 @@ AnalysesController.prototype.register = function(app) { ); }; -AnalysesController.prototype.sendResponse = function(req, res, resource) { +AnalysesController.prototype.sendResponse = function(req, res, body) { res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - this.send(req, res, resource, 200); + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } }; AnalysesController.prototype.catalog = function (req, res, next) { diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js deleted file mode 100644 index b9c6ac1e..00000000 --- a/lib/cartodb/controllers/base.js +++ /dev/null @@ -1,22 +0,0 @@ -function BaseController() { -} - -module.exports = BaseController; - -BaseController.prototype.send = function(req, res, body, status, headers) { - if (headers) { - res.set(headers); - } - - res.status(status); - - if (!Buffer.isBuffer(body) && typeof body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(body); - } else { - res.json(body); - } - } else { - res.send(body); - } -}; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 573c53ee..43dd9c3a 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -1,9 +1,6 @@ var assert = require('assert'); var step = require('step'); -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); @@ -31,8 +28,6 @@ var QueryTables = require('cartodb-query-tables'); */ function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) { - BaseController.call(this); - this.pgConnection = pgConnection; this.mapStore = mapStore; this.tileBackend = tileBackend; @@ -48,8 +43,6 @@ function LayergroupController(prepareContext, pgConnection, mapStore, tileBacken this.prepareContext = prepareContext; } -util.inherits(LayergroupController, BaseController); - module.exports = LayergroupController; LayergroupController.prototype.register = function(app) { @@ -120,7 +113,8 @@ LayergroupController.prototype.register = function(app) { 'bins', // number 'aggregation', //string 'offset', // number - 'q' // widgets search + 'q', // widgets search + 'categories', // number ]; app.get( @@ -426,10 +420,24 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h res.set('X-Cache-Channel', affectedTables.getCacheChannel()); self.surrogateKeysCache.tag(res, affectedTables); } - self.send(req, res, body, status, headers); + + if (headers) { + res.set(headers); + } + + res.status(status); + + if (!Buffer.isBuffer(body) && typeof body === 'object') { + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } + } else { + res.send(body); + } } ); - }; LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f2e86d13..6b50fc47 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -1,14 +1,9 @@ var _ = require('underscore'); -var assert = require('assert'); -var step = require('step'); var windshaft = require('windshaft'); var QueryTables = require('cartodb-query-tables'); var ResourceLocator = require('../models/resource-locator'); -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); @@ -36,9 +31,6 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, statsBackend) { - - BaseController.call(this); - this.pgConnection = pgConnection; this.templateMaps = templateMaps; this.mapBackend = mapBackend; @@ -54,354 +46,342 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m this.prepareContext = prepareContext; } -util.inherits(MapController, BaseController); - module.exports = MapController; - MapController.prototype.register = function(app) { - app.get( - app.base_url_mapconfig, - cors(), - userMiddleware, - this.prepareContext, - this.createGet.bind(this) - ); - app.post( - app.base_url_mapconfig, - cors(), - userMiddleware, - this.prepareContext, - this.createPost.bind(this) - ); - app.get( - app.base_url_templated + '/:template_id/jsonp', - cors(), - userMiddleware, - this.prepareContext, - this.jsonp.bind(this) - ); - app.post( - app.base_url_templated + '/:template_id', - cors(), - userMiddleware, - this.prepareContext, - this.instantiate.bind(this) - ); + const { base_url_mapconfig, base_url_templated } = app; + const useTemplate = true; + + app.get(base_url_mapconfig, this.composeCreateMapMiddleware()); + app.post(base_url_mapconfig, this.composeCreateMapMiddleware()); + app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate)); + app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate)); app.options(app.base_url_mapconfig, cors('Content-Type')); }; -MapController.prototype.createGet = function(req, res, next){ - req.profiler.start('windshaft.createmap_get'); +MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) { + const isTemplateInstantiation = useTemplate; + const useTemplateHash = useTemplate; + const includeQuery = !useTemplate; + const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; + const addContext = !useTemplate; - this.create(req, res, function createGet$prepareConfig(req, config) { - if ( ! config ) { - throw new Error('layergroup GET needs a "config" parameter'); - } - return JSON.parse(config); - }, next); + return [ + cors(), + userMiddleware, + this.prepareContext, + this.initProfiler(isTemplateInstantiation), + this.checkJsonContentType(), + useTemplate ? this.checkInstantiteLayergroup() : this.checkCreateLayergroup(), + useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(), + useTemplate ? this.instantiateLayergroup() : this.createLayergroup(), + this.incrementMapViewCount(), + this.augmentLayergroupData(), + this.getAffectedTables(), + this.setCacheChannel(), + this.setLastModified(), + this.setLastUpdatedTimeToLayergroup(), + this.setCacheControl(), + this.setLayerStats(), + this.setLayergroupIdHeader(useTemplateHash), + this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(), + this.setAnalysesMetadataToLayergroup(includeQuery), + this.setTurboCartoMetadataToLayergroup(), + this.setSurrogateKeyHeader(), + this.sendResponse(), + this.augmentError({ label, addContext }) + ]; }; -MapController.prototype.createPost = function(req, res, next) { - req.profiler.start('windshaft.createmap_post'); +MapController.prototype.initProfiler = function (isTemplateInstantiation) { + const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; - this.create(req, res, function createPost$prepareConfig(req) { - if (!req.is('application/json')) { - throw new Error('layergroup POST data must be of type application/json'); - } - return req.body; - }, next); + return function initProfilerMiddleware (req, res, next) { + req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + req.profiler.done(`${operation}.initProfilerMiddleware`); + next(); + }; }; -MapController.prototype.instantiate = function(req, res, next) { - req.profiler.start('windshaft-cartodb.instance_template_post'); - - this.instantiateTemplate(req, res, function prepareTemplateParams(callback) { - if (!req.is('application/json')) { - return callback(new Error('Template POST data must be of type application/json')); +MapController.prototype.checkJsonContentType = function () { + return function checkJsonContentTypeMiddleware(req, res, next) { + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('POST data must be of type application/json')); } - return callback(null, req.body); - }, next); + + req.profiler.done('checkJsonContentTypeMiddleware'); + + next(); + }; }; -MapController.prototype.jsonp = function(req, res, next) { - req.profiler.start('windshaft-cartodb.instance_template_get'); +MapController.prototype.checkInstantiteLayergroup = function () { + return function checkInstantiteLayergroupMiddleware(req, res, next) { + if (req.method === 'GET') { + const { callback, config } = req.query; - this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) { - var err = null; - if ( req.query.callback === undefined || req.query.callback.length === 0) { - err = new Error('callback parameter should be present and be a function name'); - } + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); + } - var templateParams = {}; - if (req.query.config) { - try { - templateParams = JSON.parse(req.query.config); - } catch(e) { - err = new Error('Invalid config parameter, should be a valid JSON'); + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } } } - return callback(err, templateParams); - }, next); + req.profiler.done('checkInstantiteLayergroup'); + + return next(); + }; }; -MapController.prototype.create = function(req, res, prepareConfigFn, next) { - var self = this; +MapController.prototype.checkCreateLayergroup = function () { + return function checkCreateLayergroupMiddleware (req, res, next) { + if (req.method === 'GET') { + const { config } = res.locals; - var mapConfig; + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } - var context = {}; + try { + req.body = JSON.parse(config); + } catch (err) { + return next(err); + } + } - step( - function prepareConfig () { - const requestMapConfig = prepareConfigFn(req, res.locals.config); - return requestMapConfig; - }, - function prepareAdapterMapConfig(err, requestMapConfig) { - assert.ifError(err); - context.analysisConfiguration = { - user: res.locals.user, + req.profiler.done('checkCreateLayergroup'); + return next(); + }; +}; + +MapController.prototype.getTemplate = function () { + return function getTemplateMiddleware (req, res, next) { + const templateParams = req.body; + const { user } = res.locals; + + const mapconfigProvider = new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsApi, + this.mapConfigAdapter, + user, + req.params.template_id, + templateParams, + res.locals.auth_token, + res.locals + ); + + mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + req.profiler.done('named.getMapConfig'); + if (err) { + return next(err); + } + + res.locals.mapconfig = mapconfig; + res.locals.rendererParams = rendererParams; + res.locals.mapconfigProvider = mapconfigProvider; + + next(); + }); + }.bind(this); +}; + +MapController.prototype.prepareAdapterMapConfig = function () { + return function prepareAdapterMapConfigMiddleware(req, res, next) { + const requestMapConfig = req.body; + const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; + + const context = { + analysisConfiguration: { + user, db: { - host: res.locals.dbhost, - port: res.locals.dbport, - dbname: res.locals.dbname, - user: res.locals.dbuser, - pass: res.locals.dbpassword + host: dbhost, + port: dbport, + dbname: dbname, + user: dbuser, + pass: dbpassword }, batch: { - username: res.locals.user, - apiKey: res.locals.api_key + username: user, + apiKey: api_key } - }; - self.mapConfigAdapter.getMapConfig(res.locals.user, requestMapConfig, res.locals, context, this); - }, - function createLayergroup(err, requestMapConfig) { - assert.ifError(err); - var datasource = context.datasource || Datasource.EmptyDatasource(); - mapConfig = new MapConfig(requestMapConfig, datasource); - self.mapBackend.createLayergroup( - mapConfig, - res.locals, - new CreateLayergroupMapConfigProvider(mapConfig, res.locals.user, self.userLimitsApi, res.locals), - this - ); - }, - function afterLayergroupCreate(err, layergroup) { - assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this); - }, - function finish(err, layergroup) { + } + }; + + this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { + req.profiler.done('anonymous.getMapConfig'); if (err) { - if (Number.isFinite(err.layerIndex)) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - err = error; - } - err.label = 'ANONYMOUS LAYERGROUP'; - next(err); - } else { - var analysesResults = context.analysesResults || []; - self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj()); - self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); - addContextMetadata(layergroup, mapConfig.obj(), context); - res.set('X-Layergroup-Id', layergroup.layergroupid); - self.send(req, res, layergroup, 200); + return next(err); } - } - ); -}; -function addContextMetadata(layergroup, mapConfig, context) { - if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { - layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { - if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { - layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; - } - return layer; + req.body = requestMapConfig; + res.locals.context = context; + + next(); }); - } -} - -MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn, next) { - var self = this; - - var cdbuser = res.locals.user; - - var mapConfigProvider; - var mapConfig; - step( - function getTemplateParams() { - prepareParamsFn(this); - }, - function getTemplate(err, templateParams) { - assert.ifError(err); - mapConfigProvider = new NamedMapMapConfigProvider( - self.templateMaps, - self.pgConnection, - self.metadataBackend, - self.userLimitsApi, - self.mapConfigAdapter, - cdbuser, - req.params.template_id, - templateParams, - res.locals.auth_token, - res.locals - ); - mapConfigProvider.getMapConfig(this); - }, - function createLayergroup(err, mapConfig_, rendererParams) { - assert.ifError(err); - mapConfig = mapConfig_; - self.mapBackend.createLayergroup( - mapConfig, rendererParams, - new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams), - this - ); - }, - function afterLayergroupCreate(err, layergroup) { - assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, - mapConfigProvider.analysesResults, - this); - }, - function finishTemplateInstantiation(err, layergroup) { - if (err) { - err.label = 'NAMED MAP LAYERGROUP'; - next(err); - } else { - var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); - layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; - - var _mapConfig = mapConfig.obj(); - self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig); - self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); - addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context); - - res.set('X-Layergroup-Id', layergroup.layergroupid); - self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); - - self.send(req, res, layergroup, 200); - } - } - ); + }.bind(this); }; -MapController.prototype.afterLayergroupCreate = -function(req, res, mapconfig, layergroup, analysesResults, callback) { - var self = this; +MapController.prototype.createLayergroup = function () { + return function createLayergroupMiddleware (req, res, next) { + const requestMapConfig = req.body; + const { context, user } = res.locals; + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapconfig = new MapConfig(requestMapConfig, datasource); + const mapconfigProvider = + new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); - var username = res.locals.user; + res.locals.mapconfig = mapconfig; + res.locals.analysesResults = context.analysesResults; - var tasksleft = 2; // redis key and affectedTables - var errors = []; - - var done = function(err) { - if ( err ) { - errors.push('' + err); - } - if ( ! --tasksleft ) { - err = errors.length ? new Error(errors.join('\n')) : null; - callback(err, layergroup); - } - }; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - // Don't wait for the mapview count increment to - // take place before proceeding. Error will be logged - // asynchronously - this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) { - req.profiler.done('incMapviewCount'); - if ( err ) { - global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); - } - done(); - }); - - var sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); - - var dbName = res.locals.dbname; - var layergroupId = layergroup.layergroupid; - var dbConnection; - - step( - function getPgConnection() { - self.pgConnection.getConnection(username, this); - }, - function getAffectedTablesAndLastUpdatedTime(err, connection) { - assert.ifError(err); - dbConnection = connection; - QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this); - }, - function handleAffectedTablesAndLastUpdatedTime(err, result) { - req.profiler.done('queryTablesAndLastUpdated'); - assert.ifError(err); - // feed affected tables cache so it can be reused from, for instance, layergroup controller - self.layergroupAffectedTables.set(dbName, layergroupId, result); - - var lastUpdateTime = result.getLastUpdatedAt(); - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - res.set('Last-Modified', (new Date()).toUTCString()); - res.set('X-Cache-Channel', result.getCacheChannel()); - if (result.tables && result.tables.length > 0) { - self.surrogateKeysCache.tag(res, result); - } + this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); + if (err) { + return next(err); } - return null; - }, - function fetchLayersStats(err) { - assert.ifError(err); - var next = this; - self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) { + res.locals.layergroup = layergroup; + + next(); + }); + }.bind(this); +}; + +MapController.prototype.instantiateLayergroup = function () { + return function instantiateLayergroupMiddleware (req, res, next) { + const { user, mapconfig, rendererParams } = res.locals; + const mapconfigProvider = + new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); + + this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); + if (err) { + return next(err); + } + + res.locals.layergroup = layergroup; + + const { mapconfigProvider } = res.locals; + + res.locals.analysesResults = mapconfigProvider.analysesResults; + res.locals.template = mapconfigProvider.template; + res.locals.templateName = mapconfigProvider.getTemplateName(); + res.locals.context = mapconfigProvider.context; + + next(); + }); + }.bind(this); +}; + +MapController.prototype.incrementMapViewCount = function () { + return function incrementMapViewCountMiddleware(req, res, next) { + const { mapconfig, user } = res.locals; + + // Error won't blow up, just be logged. + this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { + req.profiler.done('incMapviewCount'); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + } + + next(); + }); + }.bind(this); +}; + +MapController.prototype.augmentLayergroupData = function () { + return function augmentLayergroupDataMiddleware (req, res, next) { + const { layergroup } = res.locals; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + next(); + }; +}; + +MapController.prototype.getAffectedTables = function () { + return function getAffectedTablesMiddleware (req, res, next) { + const { dbname, layergroup, user, mapconfig } = res.locals; + + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + const sql = []; + mapconfig.getLayers().forEach(function(layer) { + sql.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(function(table) { + sql.push('SELECT * FROM ' + table + ' LIMIT 0'); + }); + } + }); + + QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { + req.profiler.done('getAffectedTablesFromQuery'); if (err) { return next(err); } - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - return next(); + + // feed affected tables cache so it can be reused from, for instance, layergroup controller + this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + + res.locals.affectedTables = affectedTables; + + next(); }); - }, - function finish(err) { - done(err); + }); + }.bind(this); +}; + +MapController.prototype.setCacheChannel = function () { + return function setCacheChannelMiddleware (req, res, next) { + const { affectedTables } = res.locals; + + if (req.method === 'GET') { + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); } - ); + + next(); + }; +}; + +MapController.prototype.setLastModified = function () { + return function setLastModifiedMiddleware (req, res, next) { + if (req.method === 'GET') { + res.set('Last-Modified', (new Date()).toUTCString()); + } + + next(); + }; +}; + +MapController.prototype.setLastUpdatedTimeToLayergroup = function () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { affectedTables, layergroup, analysesResults } = res.locals; + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }; }; function getLastUpdatedTime(analysesResults, lastUpdateTime) { @@ -417,34 +397,66 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) { }, lastUpdateTime); } -MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) { - includeQuery = includeQuery || false; - analysesResults = analysesResults || []; - layergroup.metadata.analyses = []; +MapController.prototype.setCacheControl = function () { + return function setCacheControlMiddleware (req, res, next) { + if (req.method === 'GET') { + var ttl = global.environment.varnish.layergroupTtl || 86400; + res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); + } - analysesResults.forEach(function(analysis) { - var nodes = analysis.getNodes(); - layergroup.metadata.analyses.push({ - nodes: nodes.reduce(function(nodesIdMap, node) { - if (node.params.id) { - var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); - var nodeRepr = { - status: node.getStatus(), - url: this.resourceLocator.getUrls(username, nodeResource) - }; - if (includeQuery) { - nodeRepr.query = node.getQuery(); - } - if (node.getStatus() === 'failed') { - nodeRepr.error_message = node.getErrorMessage(); - } - nodesIdMap[node.params.id] = nodeRepr; + next(); + }; +}; + +MapController.prototype.setLayerStats = function () { + return function setLayerStatsMiddleware(req, res, next) { + const { user, mapconfig, layergroup } = res.locals; + + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + if (err) { + return next(err); } - return nodesIdMap; - }.bind(this), {}) + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + next(); + }); }); - }.bind(this)); + }.bind(this); +}; + +MapController.prototype.setLayergroupIdHeader = function (useTemplateHash) { + return function setLayergroupIdHeaderMiddleware (req, res, next) { + const { layergroup, user, template } = res.locals; + + if (useTemplateHash) { + var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + } + + res.set('X-Layergroup-Id', layergroup.layergroupid); + + next(); + }.bind(this); +}; + +MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () { + return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { + const { layergroup, user, mapconfig } = res.locals; + + this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + + next(); + }.bind(this); }; // TODO this should take into account several URL patterns @@ -483,3 +495,131 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig }.bind(this)); } }; + +MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) { + return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { + const { layergroup, user, analysesResults = [] } = res.locals; + + this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + + next(); + }.bind(this); +}; + +MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) { + includeQuery = includeQuery || false; + analysesResults = analysesResults || []; + layergroup.metadata.analyses = []; + + analysesResults.forEach(function(analysis) { + var nodes = analysis.getNodes(); + layergroup.metadata.analyses.push({ + nodes: nodes.reduce(function(nodesIdMap, node) { + if (node.params.id) { + var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); + var nodeRepr = { + status: node.getStatus(), + url: this.resourceLocator.getUrls(username, nodeResource) + }; + if (includeQuery) { + nodeRepr.query = node.getQuery(); + } + if (node.getStatus() === 'failed') { + nodeRepr.error_message = node.getErrorMessage(); + } + nodesIdMap[node.params.id] = nodeRepr; + } + + return nodesIdMap; + }.bind(this), {}) + }); + }.bind(this)); +}; + +MapController.prototype.setTurboCartoMetadataToLayergroup = function () { + return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { + const { layergroup, mapconfig, context } = res.locals; + + addContextMetadata(layergroup, mapconfig.obj(), context); + + next(); + }; +}; + +function addContextMetadata(layergroup, mapConfig, context) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { + layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; + } + return layer; + }); + } +} + +MapController.prototype.setSurrogateKeyHeader = function () { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { affectedTables, user, templateName } = res.locals; + + if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { + this.surrogateKeysCache.tag(res, affectedTables); + } + + if (templateName) { + this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); + } + + next(); + }.bind(this); +}; + +MapController.prototype.sendResponse = function () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + const { layergroup } = res.locals; + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); + } else { + res.json(layergroup); + } + }; +}; + +MapController.prototype.augmentError = function (options) { + const { addContext = false, label = 'MAPS CONTROLLER' } = options; + + return function augmentErrorMiddleware (err, req, res, next) { + req.profiler.done('error'); + const { mapconfig } = res.locals; + + if (addContext) { + err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; + } + + err.label = label; + + next(err); + }; +}; + +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 442db927..a88c8592 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -3,9 +3,6 @@ var assert = require('assert'); var _ = require('underscore'); var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); @@ -13,8 +10,6 @@ var vectorError = require('../middleware/vector-error'); function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { - BaseController.call(this); - this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.previewBackend = previewBackend; @@ -24,8 +19,6 @@ function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, this.prepareContext = prepareContext; } -util.inherits(NamedMapsController, BaseController); - module.exports = NamedMapsController; NamedMapsController.prototype.register = function(app) { @@ -48,7 +41,7 @@ NamedMapsController.prototype.register = function(app) { ); }; -NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) { +NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) { this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName())); res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); @@ -81,7 +74,8 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header self.surrogateKeysCache.tag(res, result); } } - self.send(req, res, resource, 200); + res.status(200); + res.send(body); } ); }; @@ -146,7 +140,7 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { assert.ifError(err); namedMapProvider = _namedMapProvider; - + self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this); }, function prepareImageOptions(err) { @@ -195,10 +189,10 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { }; NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function ( - user, - req, - params, - namedMapProvider, + user, + req, + params, + namedMapProvider, callback ) { var self = this; diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index ae57ada3..0f54ddf3 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -2,9 +2,6 @@ var step = require('step'); var assert = require('assert'); var templateName = require('../backends/template_maps').templateName; -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); @@ -16,14 +13,10 @@ var userMiddleware = require('../middleware/user'); * @constructor */ function NamedMapsAdminController(authApi, templateMaps) { - BaseController.call(this); - this.authApi = authApi; this.templateMaps = templateMaps; } -util.inherits(NamedMapsAdminController, BaseController); - module.exports = NamedMapsAdminController; NamedMapsAdminController.prototype.register = function (app) { @@ -206,12 +199,18 @@ NamedMapsAdminController.prototype.list = function(req, res, next) { }; function finishFn(controller, req, res, description, status, next) { - return function finish(err, response){ + return function finish(err, body){ if (err) { err.label = description; next(err); } else { - controller.send(req, res, response, status || 200); + res.status(status || 200); + + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } } }; } diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index b4c59af2..40774353 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -245,6 +245,10 @@ module.exports = class Aggregation extends BaseDataview { return null; } + const limit = Number.isFinite(override.categories) && override.categories > 0 ? + override.categories : + CATEGORIES_LIMIT; + const aggregationSql = aggregationDataviewQueryTpl({ override: override, query: this.query, @@ -256,7 +260,7 @@ module.exports = class Aggregation extends BaseDataview { aggregationColumn: this.aggregationColumn || 1 }), isFloatColumn: this._isFloatColumn, - limit: CATEGORIES_LIMIT + limit }); debug(aggregationSql); diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index d8d03177..e29d9ff4 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -324,3 +324,104 @@ describe('aggregation-dataview: special float values', function() { }); }); }); + +describe('aggregation dataview tuned by categories query param', function () { + const mapConfig = { + version: '1.5.0', + layers: [ + { + type: "cartodb", + options: { + source: { + "id": "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + dataviews: { + categories: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + } + }, + analyses: [ + { + id: "a0", + type: "source", + params: { + query: ` + SELECT + null::geometry the_geom_webmercator, + CASE + WHEN x % 4 = 0 THEN 1 + WHEN x % 4 = 1 THEN 2 + WHEN x % 4 = 2 THEN 3 + ELSE 4 + END AS val, + CASE + WHEN x % 4 = 0 THEN 'category_1' + WHEN x % 4 = 1 THEN 'category_2' + WHEN x % 4 = 2 THEN 'category_3' + ELSE 'category_4' + END AS cat + FROM generate_series(1, 1000) x + ` + } + } + ] + }; + + beforeEach(function () { + this.testClient = new TestClient(mapConfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + var scenarios = [ + { + params: { own_filter: 0, categories: -1 }, + categoriesExpected: 4 + }, + { + params: { own_filter: 0, categories: 0 }, + categoriesExpected: 4 + }, + { + params: { own_filter: 0, categories: 1 }, + categoriesExpected: 1 + }, + { + params: { own_filter: 0, categories: 2 }, + categoriesExpected: 2 + }, + { + params: { own_filter: 0, categories: 4 }, + categoriesExpected: 4 + }, + { + params: { own_filter: 0, categories: 5 }, + categoriesExpected: 4 + } + ]; + + scenarios.forEach(function (scenario) { + it(`should handle cartegories to customize aggregations: ${JSON.stringify(scenario.params)}`, function (done) { + this.testClient.getDataview('categories', scenario.params, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categories.length, scenario.categoriesExpected); + done(); + }); + }); + }); +}); diff --git a/test/acceptance/overviews_queries.js b/test/acceptance/overviews_queries.js index 0434ebfc..ad7599a6 100644 --- a/test/acceptance/overviews_queries.js +++ b/test/acceptance/overviews_queries.js @@ -4,7 +4,6 @@ var assert = require('../support/assert'); var cartodbServer = require('../../lib/cartodb/server'); var ServerOptions = require('./ported/support/ported_server_options'); var testClient = require('./ported/support/test_client'); -var BaseController = require('../../lib/cartodb/controllers/base'); describe('overviews_queries', function() { @@ -13,15 +12,7 @@ describe('overviews_queries', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - after(function() { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/attributes.js b/test/acceptance/ported/attributes.js index 39f3f461..abd4e5ab 100644 --- a/test/acceptance/ported/attributes.js +++ b/test/acceptance/ported/attributes.js @@ -4,13 +4,10 @@ var assert = require('../../support/assert'); var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); - describe('attributes', function() { - var server = cartodbServer(PortedServerOptions); server.setMaxListeners(0); @@ -49,16 +46,6 @@ describe('attributes', function() { testHelper.deleteRedisKeys(keysToDelete, done); }); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - it("can only be fetched from layer having an attributes spec", function(done) { var expected_token; diff --git a/test/acceptance/ported/blend.js b/test/acceptance/ported/blend.js index 1e844d80..52273568 100644 --- a/test/acceptance/ported/blend.js +++ b/test/acceptance/ported/blend.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('blend png renderer', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var IMAGE_TOLERANCE_PER_MIL = 20; function plainTorqueMapConfig(plainColor) { diff --git a/test/acceptance/ported/blend_filtering.js b/test/acceptance/ported/blend_filtering.js index dc8cd470..2ada716e 100644 --- a/test/acceptance/ported/blend_filtering.js +++ b/test/acceptance/ported/blend_filtering.js @@ -5,20 +5,13 @@ var testClient = require('./support/test_client'); var fs = require('fs'); var http = require('http'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('blend layer filtering', function() { var IMG_TOLERANCE_PER_MIL = 20; var httpRendererResourcesServer; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - // Start a server to test external resources httpRendererResourcesServer = http.createServer( function(request, response) { var filename = __dirname + '/../../fixtures/http/light_nolabels-1-0-0.png'; @@ -32,7 +25,6 @@ describe('blend layer filtering', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; httpRendererResourcesServer.close(done); }); diff --git a/test/acceptance/ported/blend_http_fallback.js b/test/acceptance/ported/blend_http_fallback.js index 480dc5c0..f86d3dfc 100644 --- a/test/acceptance/ported/blend_http_fallback.js +++ b/test/acceptance/ported/blend_http_fallback.js @@ -5,19 +5,13 @@ var testClient = require('./support/test_client'); var fs = require('fs'); var http = require('http'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('blend http fallback', function() { var IMG_TOLERANCE_PER_MIL = 20; var httpRendererResourcesServer; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; // Start a server to test external resources httpRendererResourcesServer = http.createServer( function(request, response) { if (request.url.match(/^\/error404\//)) { @@ -39,7 +33,6 @@ describe('blend http fallback', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; httpRendererResourcesServer.close(done); }); diff --git a/test/acceptance/ported/blend_http_timeout.js b/test/acceptance/ported/blend_http_timeout.js index 341f72b3..54481fd0 100644 --- a/test/acceptance/ported/blend_http_timeout.js +++ b/test/acceptance/ported/blend_http_timeout.js @@ -6,21 +6,7 @@ var serverOptions = require('./support/ported_server_options'); var fs = require('fs'); var http = require('http'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe.skip('blend http client timeout', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var mapConfig = { version: '1.3.0', layers: [ diff --git a/test/acceptance/ported/external_resources.js b/test/acceptance/ported/external_resources.js index 11ad5176..0aae14c3 100644 --- a/test/acceptance/ported/external_resources.js +++ b/test/acceptance/ported/external_resources.js @@ -1,16 +1,12 @@ var testHelper = require('../../support/test_helper'); - var assert = require('../../support/assert'); var fs = require('fs'); var PortedServerOptions = require('./support/ported_server_options'); var http = require('http'); var testClient = require('./support/test_client'); - var nock = require('nock'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('external resources', function() { var res_serv; // resources server @@ -19,12 +15,8 @@ describe('external resources', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; - var req2paramsFn; before(function(done) { nock.enableNetConnect('127.0.0.1'); - - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; // Start a server to test external resources res_serv = http.createServer( function(request, response) { ++res_serv_status.numrequests; @@ -44,8 +36,6 @@ describe('external resources', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); // Close the resources server diff --git a/test/acceptance/ported/limits.js b/test/acceptance/ported/limits.js index c111c87c..0a8ee378 100644 --- a/test/acceptance/ported/limits.js +++ b/test/acceptance/ported/limits.js @@ -6,19 +6,14 @@ var assert = require('../../support/assert'); var testClient = require('./support/test_client'); var serverOptions = require('./support/ported_server_options'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe.skip('render limits', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; var limitsConfig; var onTileErrorStrategy; - var req2paramsFn; + before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; limitsConfig = serverOptions.renderer.mapnik.limits; serverOptions.renderer.mapnik.limits = { render: 50, @@ -31,7 +26,6 @@ describe.skip('render limits', function() { }); after(function() { - BaseController.prototype.req2params = req2paramsFn; serverOptions.renderer.mapnik.limits = limitsConfig; serverOptions.renderer.onTileErrorStrategy = onTileErrorStrategy; }); diff --git a/test/acceptance/ported/multilayer.js b/test/acceptance/ported/multilayer.js index 0592c147..0ad9f5de 100644 --- a/test/acceptance/ported/multilayer.js +++ b/test/acceptance/ported/multilayer.js @@ -8,7 +8,6 @@ var mapnik = require('windshaft').mapnik; var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); -var BaseController = require('../../../lib/cartodb/controllers/base'); describe('multilayer', function() { @@ -24,16 +23,6 @@ describe('multilayer', function() { assert.equal(res.headers['access-control-allow-origin'], '*'); } - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - // See https://github.com/Vizzuality/Windshaft/issues/70 it("post layergroup with encoding in content-type", function(done) { var layergroup = { diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index 3408baa2..9efd7c7b 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -7,23 +7,11 @@ var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); var TestClient = require('../../support/test-client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('multilayer error cases', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - // var client = null; afterEach(function(done) { if (this.client) { @@ -40,7 +28,7 @@ describe('multilayer error cases', function() { }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); var parsedBody = JSON.parse(res.body); - assert.deepEqual(parsedBody.errors, ["layergroup POST data must be of type application/json"]); + assert.deepEqual(parsedBody.errors, ["POST data must be of type application/json"]); done(); }); }); diff --git a/test/acceptance/ported/multilayer_interactivity.js b/test/acceptance/ported/multilayer_interactivity.js index 7d512670..4542f710 100644 --- a/test/acceptance/ported/multilayer_interactivity.js +++ b/test/acceptance/ported/multilayer_interactivity.js @@ -7,23 +7,11 @@ var getLayerTypeFn = require('windshaft').model.MapConfig.prototype.getType; var PortedServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('multilayer interactivity and layers order', function() { var server = cartodbServer(PortedServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - function layerType(layer) { return layer.type || 'undefined'; } diff --git a/test/acceptance/ported/raster.js b/test/acceptance/ported/raster.js index b16dd56d..448661f2 100644 --- a/test/acceptance/ported/raster.js +++ b/test/acceptance/ported/raster.js @@ -4,8 +4,6 @@ var assert = require('../../support/assert'); var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); - -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('raster', function() { @@ -20,18 +18,7 @@ describe('raster', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - it("can render raster for valid mapconfig", function(done) { - var mapconfig = { version: '1.2.0', layers: [ diff --git a/test/acceptance/ported/regressions.js b/test/acceptance/ported/regressions.js index 2a2188c1..dd765b57 100644 --- a/test/acceptance/ported/regressions.js +++ b/test/acceptance/ported/regressions.js @@ -1,22 +1,10 @@ var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); -var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('regressions', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/retina.js b/test/acceptance/ported/retina.js index 80c3fcc5..7469214b 100644 --- a/test/acceptance/ported/retina.js +++ b/test/acceptance/ported/retina.js @@ -5,7 +5,6 @@ var mapnik = require('windshaft').mapnik; var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('retina support', function() { @@ -15,15 +14,6 @@ describe('retina support', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); var keysToDelete; beforeEach(function(done) { diff --git a/test/acceptance/ported/server.js b/test/acceptance/ported/server.js index 650cf2c5..b18bae07 100644 --- a/test/acceptance/ported/server.js +++ b/test/acceptance/ported/server.js @@ -5,22 +5,12 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('server', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - after(function() { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index d9a92661..f3c9ba81 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -7,8 +7,6 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('server_gettile', function() { var server = cartodbServer(ServerOptions); @@ -16,16 +14,7 @@ describe('server_gettile', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 092e9ad7..18151330 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -6,13 +6,11 @@ var fs = require('fs'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 85; describe('server_png8_format', function() { - var serverOptionsPng32 = ServerOptions; serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore); serverOptionsPng32.grainstore.mapnik_tile_format = 'png32'; @@ -25,13 +23,9 @@ describe('server_png8_format', function() { var serverPng8 = cartodbServer(serverOptionsPng8); serverPng8.setMaxListeners(0); - var layergroupId; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; var testPngFilesDir = __dirname + '/../../results/png'; fs.readdirSync(testPngFilesDir) .filter(function(fileName) { @@ -45,10 +39,6 @@ describe('server_png8_format', function() { done(); }); - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var keysToDelete; beforeEach(function() { keysToDelete = { diff --git a/test/acceptance/ported/static_maps.js b/test/acceptance/ported/static_maps.js index 2a04ca29..1002cdd5 100644 --- a/test/acceptance/ported/static_maps.js +++ b/test/acceptance/ported/static_maps.js @@ -5,9 +5,6 @@ var testClient = require('./support/test_client'); var http = require('http'); var fs = require('fs'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('static_maps', function() { var validUrlTemplate = 'http://127.0.0.1:8033/{s}/{z}/{x}/{y}.png'; @@ -15,11 +12,7 @@ describe('static_maps', function() { var httpRendererResourcesServer; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - // Start a server to test external resources httpRendererResourcesServer = http.createServer( function(request, response) { var filename = __dirname + '/../../fixtures/http/basemap.png'; @@ -33,7 +26,6 @@ describe('static_maps', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; httpRendererResourcesServer.close(done); }); diff --git a/test/acceptance/ported/torque.js b/test/acceptance/ported/torque.js index c148a521..47ae2eee 100644 --- a/test/acceptance/ported/torque.js +++ b/test/acceptance/ported/torque.js @@ -6,7 +6,6 @@ var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('torque', function() { @@ -14,16 +13,6 @@ describe('torque', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var keysToDelete; beforeEach(function() { keysToDelete = {}; diff --git a/test/acceptance/ported/torque_boundaries.js b/test/acceptance/ported/torque_boundaries.js index 1456c055..a984389d 100644 --- a/test/acceptance/ported/torque_boundaries.js +++ b/test/acceptance/ported/torque_boundaries.js @@ -4,21 +4,9 @@ var assert = require('../../support/assert'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('torque boundary points', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var layergroupIdToDelete = null; beforeEach(function() { diff --git a/test/acceptance/ported/torque_png.js b/test/acceptance/ported/torque_png.js index 69eaadab..57b3ffa1 100644 --- a/test/acceptance/ported/torque_png.js +++ b/test/acceptance/ported/torque_png.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('torque png renderer', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var IMAGE_TOLERANCE_PER_MIL = 20; var torquePngPointsMapConfig = { diff --git a/test/acceptance/ported/torque_zero_zero.js b/test/acceptance/ported/torque_zero_zero.js index b0b00081..fe6755ae 100644 --- a/test/acceptance/ported/torque_zero_zero.js +++ b/test/acceptance/ported/torque_zero_zero.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('torque tiles at 0,0 point', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - /* Tiles are represented as in: diff --git a/test/acceptance/ported/wrap.js b/test/acceptance/ported/wrap.js index 64a8e2f4..d79e2b56 100644 --- a/test/acceptance/ported/wrap.js +++ b/test/acceptance/ported/wrap.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('wrap x coordinate', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - describe('renders correct tile', function() { var IMG_TOLERANCE_PER_MIL = 20; diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index dcf1cba7..41f0fb8c 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -13,6 +13,7 @@ PREPARE_REDIS=yes PREPARE_PGSQL=yes DOWNLOAD_SQL_FILES=yes +PG_PARALLEL=$(pg_config --version | (awk '{$2*=1000; if ($2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) while [ -n "$1" ]; do if test "$1" = "--skip-pg"; then @@ -92,6 +93,11 @@ if test x"$PREPARE_PGSQL" = xyes; then ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" for i in ${ALL_SQL_SCRIPTS} do + # Strip PARALLEL labels for PostgreSQL releases before 9.6 + if [ $PG_PARALLEL -eq 0 ]; then + sed -e 's/PARALLEL \= [A-Z]*,/''/g' \ + -e 's/PARALLEL [A-Z]*/''/g' -i sql/$i.sql + fi cat sql/${i}.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | sed "s/:PUBLICUSER/${PUBLICUSER}/" | diff --git a/test/support/test-client.js b/test/support/test-client.js index cd7b4f60..f31f9db4 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -415,7 +415,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset', 'categories'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; }