diff --git a/Makefile b/Makefile index 641c291a..2aa59a6a 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,23 @@ test: config/environments/test.js test/acceptance/*.js \ test/acceptance/cache/*.js +test-unit: config/environments/test.js + @echo "***tests***" + @$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \ + test/unit/cartodb/*.js \ + test/unit/cartodb/cache/model/*.js + +test-integration: config/environments/test.js + @echo "***tests***" + @$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \ + test/integration/*.js + +test-acceptance: config/environments/test.js + @echo "***tests***" + @$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \ + test/acceptance/*.js \ + test/acceptance/cache/*.js + jshint: @echo "***jshint***" @./node_modules/.bin/jshint lib/ test/ app.js diff --git a/app.js b/app.js index 19361566..cad62adc 100755 --- a/app.js +++ b/app.js @@ -8,6 +8,7 @@ */ var path = require('path'); +var os = require('os'); var fs = require('fs'); var RedisPool = require('redis-mpool'); var _ = require('underscore'); @@ -68,10 +69,19 @@ global.logger = global.log4js.getLogger(); var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }), redisPool = new RedisPool(redisOpts); +// Perform keyword substitution in statsd +// See https://github.com/CartoDB/Windshaft-cartodb/issues/153 +if ( global.environment.statsd ) { + if ( global.environment.statsd.prefix ) { + var host_token = os.hostname().split('.').reverse().join('.'); + global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token); + } +} + // Include cartodb_windshaft only _after_ the "global" variable is set // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 -var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'), - serverOptions = require('./lib/cartodb/server_options')(redisPool); +var cartodbWindshaft = require('./lib/cartodb/server'), + serverOptions = require('./lib/cartodb/server_options'); var ws = cartodbWindshaft(serverOptions); diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js deleted file mode 100644 index f10bcd01..00000000 --- a/lib/cartodb/cartodb_windshaft.js +++ /dev/null @@ -1,220 +0,0 @@ -var _ = require('underscore'); -var step = require('step'); -var Windshaft = require('windshaft'); -var os = require('os'); -var HealthCheck = require('./monitoring/health_check'); - -var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); -var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); -var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); -var FastlyCacheBackend = require('./cache/backend/fastly'); - -if ( ! process.env.PGAPPNAME ) - process.env.PGAPPNAME='cartodb_tiler'; - -var CartodbWindshaft = function(serverOptions) { - // Perform keyword substitution in statsd - // See https://github.com/CartoDB/Windshaft-cartodb/issues/153 - if ( global.environment.statsd ) { - if ( global.environment.statsd.prefix ) { - var host_token = os.hostname().split('.').reverse().join('.'); - global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token); - } - } - - var redisPool = serverOptions.redis.pool || - require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'})); - - var cartoData = require('cartodb-redis')({pool: redisPool}); - - var templateMaps = serverOptions.templateMaps; - - // This is for Templated maps - // - // "named" is the official, "template" is for backward compatibility up to 1.6.x - // - var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)'; - - var surrogateKeysCacheBackends = []; - - if (serverOptions.varnish_purge_enabled) { - surrogateKeysCacheBackends.push( - new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) - ); - } - - if (!!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { - surrogateKeysCacheBackends.push( - new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) - ); - } - - var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends); - - function invalidateNamedMap (owner, templateName) { - var startTime = Date.now(); - surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { - var logMessage = JSON.stringify({ - username: owner, - type: 'named_map_invalidation', - elapsed: Date.now() - startTime, - error: !!err ? JSON.stringify(err.message) : undefined - }); - if (err) { - console.warn(logMessage); - } else { - console.info(logMessage); - } - }); - } - ['update', 'delete'].forEach(function(eventType) { - templateMaps.on(eventType, invalidateNamedMap); - }); - - // boot - var ws = new Windshaft.Server(serverOptions); - - // Override getVersion to include cartodb-specific versions - var wsversion = ws.getVersion; - ws.getVersion = function() { - var version = wsversion(); - version.windshaft_cartodb = require('../../package.json').version; - return version; - }; - - var ws_sendResponse = ws.sendResponse; - // GET routes for which we don't want to request any caching. - // POST/PUT/DELETE requests are never cached anyway. - var noCacheGETRoutes = [ - '/', - '/version', - // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 - serverOptions.base_url_mapconfig, - serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', - template_baseurl, - template_baseurl + '/:template_id', - template_baseurl + '/:template_id/jsonp' - ]; - ws.sendResponse = function(res, args) { - var that = this; - var thatArgs = arguments; - var statusCode; - if ( res._windshaftStatusCode ) { - // Added by our override of sendError - statusCode = res._windshaftStatusCode; - } else { - if ( args.length > 2 ) statusCode = args[2]; - else { - statusCode = args[1] || 200; - } - } - var req = res.req; - step ( - function addCacheChannel() { - if ( ! req ) { - // having no associated request can happen when - // using fake response objects for testing layergroup - // creation - return false; - } - if ( ! req.params ) { - // service requests (/version, /) - // have no need for an X-Cache-Channel - return false; - } - if ( statusCode != 200 ) { - // We do not want to cache - // unsuccessful responses - return false; - } - if ( _.contains(noCacheGETRoutes, req.route.path) ) { -//console.log("Skipping cache channel in route:\n" + req.route.path); - return false; - } -//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + -// mapCreateRoutes.join("\n")); - serverOptions.addCacheChannel(that, req, this); - }, - function sendResponse(err/*, added*/) { - if ( err ) console.log(err + err.stack); - ws_sendResponse.apply(that, thatArgs); - return null; - }, - function finish(err) { - if ( err ) console.log(err + err.stack); - } - ); - }; - - var ws_sendError = ws.sendError; - ws.sendError = function() { - var res = arguments[0]; - var statusCode = arguments[2]; - res._windshaftStatusCode = statusCode; - ws_sendError.apply(this, arguments); - }; - - /******************************************************************************************************************* - * Routing - ******************************************************************************************************************/ - - var NamedMapsController = require('./controllers/named_maps'), - namedMapsController = new NamedMapsController( - ws, - serverOptions, - templateMaps, - cartoData, - ws.mapBackend, - template_baseurl, - surrogateKeysCache - ); - namedMapsController.register(ws); - - var TablesExtentApi = require('./api/tables_extent_api'); - var tablesExtentApi = new TablesExtentApi(serverOptions.pgQueryRunner); - - var NamedStaticMapsController = require('./controllers/named_static_maps'); - var namedStaticMapsController = new NamedStaticMapsController( - ws, - serverOptions, - templateMaps, - ws.mapBackend, - ws.staticMapBackend, - surrogateKeysCache, - tablesExtentApi - ); - namedStaticMapsController.register(ws); - - /******************************************************************************************************************* - * END Routing - ******************************************************************************************************************/ - - var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive); - ws.get('/health', function(req, res) { - var healthConfig = global.environment.health || {}; - - if (!!healthConfig.enabled) { - var startTime = Date.now(); - healthCheck.check(healthConfig, function(err, result) { - var ok = !err; - var response = { - enabled: true, - ok: ok, - elapsed: Date.now() - startTime, - result: result - }; - if (err) { - response.err = err.message; - } - res.send(response, ok ? 200 : 503); - - }); - } else { - res.send({enabled: false, ok: true}, 200); - } - }); - - return ws; -}; - -module.exports = CartodbWindshaft; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js new file mode 100644 index 00000000..0d38bcf0 --- /dev/null +++ b/lib/cartodb/controllers/map.js @@ -0,0 +1,211 @@ +var assert = require('assert'); + +var step = require('step'); + +/** + * @param app + * @param {MapBackend} mapBackend + * @param options + * @constructor + */ +function MapController(app, mapBackend, layergroupRequestDecorator) { + this._app = app; + this._mapBackend = mapBackend; + this._layergroupRequestDecorator = layergroupRequestDecorator; +} + +module.exports = MapController; + + +MapController.prototype.register = function(app) { + app.get(app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format', this.tile.bind(this)); + app.get(app.base_url_mapconfig + '/:token/:z/:x/:y.:format', this.tile.bind(this)); + app.get(app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)', this.layer.bind(this)); + app.options(app.base_url_mapconfig, this.cors.bind(this)); + app.get(app.base_url_mapconfig, this.createGet.bind(this)); + app.post(app.base_url_mapconfig, this.createPost.bind(this)); + app.get(app.base_url_mapconfig + '/:token/:layer/attributes/:fid', this.attributes.bind(this)); +}; + +// send CORS headers when client send options. +MapController.prototype.cors = function(req, res, next) { + this._app.doCORS(res, "Content-Type"); + return next(); +}; + +// Gets attributes for a given layer feature +MapController.prototype.attributes = function(req, res) { + var self = this; + + req.profiler.start('windshaft.maplayer_attribute'); + + this._app.doCORS(res); + + step( + function setupParams() { + self._app.req2params(req, this); + }, + function retrieveFeatureAttributes(err) { + req.profiler.done('req2params'); + + assert.ifError(err); + + self._mapBackend.getFeatureAttributes(req.params, false, this); + }, + function finish(err, tile, stats) { + req.profiler.add(stats || {}); + + if (err) { + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + var errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + var statusCode = self._app.findStatusCode(err); + self._app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err); + } else { + self._app.sendResponse(res, [tile, 200]); + } + } + ); + +}; + +MapController.prototype.create = function(req, res, prepareConfigFn) { + var self = this; + + this._app.doCORS(res); + + var layergroupDecorator = { + beforeLayergroupCreate: function(requestMapConfig, callback) { + self._layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback); + }, + afterLayergroupCreate: function(layergroup, response, callback) { + self._layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback); + } + }; + + step( + function setupParams(){ + self._app.req2params(req, this); + }, + prepareConfigFn, + function initLayergroup(err, requestMapConfig) { + assert.ifError(err); + self._mapBackend.createLayergroup(requestMapConfig, req.params, layergroupDecorator, this); + }, + function finish(err, response){ + if (err) { + response = { errors: [ err.message ] }; + var statusCode = self._app.findStatusCode(err); + self._app.sendError(res, response, statusCode, 'GET LAYERGROUP', err); + } else { + self._app.sendResponse(res, [response, 200]); + } + } + ); +}; + +MapController.prototype.createGet = function(req, res){ + req.profiler.start('windshaft.createmap_get'); + + this.create(req, res, function createGet$prepareConfig(err, req) { + assert.ifError(err); + if ( ! req.params.config ) { + throw new Error('layergroup GET needs a "config" parameter'); + } + return JSON.parse(req.params.config); + }); +}; + +// TODO rewrite this so it is possible to share code with `MapController::create` method +MapController.prototype.createPost = function(req, res) { + req.profiler.start('windshaft.createmap_post'); + + this.create(req, res, function createPost$prepareConfig(err, req) { + assert.ifError(err); + if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] !== 'application/json' ) { + throw new Error('layergroup POST data must be of type application/json'); + } + return req.body; + }); +}; + +// Gets a tile for a given token and set of tile ZXY coords. (OSM style) +MapController.prototype.tile = function(req, res) { + req.profiler.start('windshaft.map_tile'); + this.tileOrLayer(req, res); +}; + +// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style) +MapController.prototype.layer = function(req, res, next) { + if (req.params.token === 'static') { + return next(); + } + req.profiler.start('windshaft.maplayer_tile'); + this.tileOrLayer(req, res); +}; + +MapController.prototype.tileOrLayer = function (req, res) { + var self = this; + + this._app.doCORS(res); + step( + function mapController$prepareParams() { + self._app.req2params(req, this); + }, + function mapController$getTileOrGrid(err) { + req.profiler.done('req2params'); + if ( err ) { + throw err; + } + self._mapBackend.getTileOrGrid(req.params, this); + }, + function mapController$finalize(err, tile, headers, stats) { + req.profiler.add(stats); + self.finalizeGetTileOrGrid(err, req, res, tile, headers); + return null; + }, + function finish(err) { + if ( err ) { + console.error("windshaft.tiles: " + err); + } + } + ); +}; + +// This function is meant for being called as the very last +// step by all endpoints serving tiles or grids +MapController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) { + var supportedFormats = { + grid_json: true, + json_torque: true, + torque_json: true, + png: true + }; + + var formatStat = 'invalid'; + if (req.params.format) { + var format = req.params.format.replace('.', '_'); + if (supportedFormats[format]) { + formatStat = format; + } + } + + if (err){ + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + var errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + var statusCode = this._app.findStatusCode(err); + + // Rewrite mapnik parsing errors to start with layer number + var matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + if (matches) { + errMsg = 'style'+matches[2]+': ' + matches[1]; + } + + this._app.sendError(res, { errors: ['' + errMsg] }, statusCode, 'TILE RENDER', err); + global.statsClient.increment('windshaft.tiles.error'); + global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); + } else { + this._app.sendWithHeaders(res, tile, 200, headers); + global.statsClient.increment('windshaft.tiles.success'); + global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); + } +}; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 1d419db7..16a1c0c6 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -6,7 +6,7 @@ var CdbRequest = require('../models/cdb_request'); var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); function NamedMapsController(app, serverOptions, templateMaps, metadataBackend, mapBackend, templateBaseUrl, - surrogateKeysCache) { + surrogateKeysCache, layergroupRequestDecorator) { this.app = app; this.serverOptions = serverOptions; this.templateMaps = templateMaps; @@ -14,6 +14,7 @@ function NamedMapsController(app, serverOptions, templateMaps, metadataBackend, this.mapBackend = mapBackend; this.templateBaseUrl = templateBaseUrl; this.surrogateKeysCache = surrogateKeysCache; + this.layergroupRequestDecorator = layergroupRequestDecorator } module.exports = NamedMapsController; @@ -46,10 +47,10 @@ NamedMapsController.prototype.tile = function(req, res) { var layergroupDecorator = { beforeLayergroupCreate: function(requestMapConfig, callback) { - self.serverOptions.beforeLayergroupCreate(req, requestMapConfig, callback); + self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback); }, afterLayergroupCreate: function(layergroup, response, callback) { - self.serverOptions.afterLayergroupCreate(req, layergroup, response, callback); + self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback); } }; @@ -118,7 +119,7 @@ NamedMapsController.prototype.tile = function(req, res) { params = _.extend({}, req.params, { user: req.params.user }); - self.serverOptions.setDBParams(cdbUser, params, this); + self.app.setDBParams(cdbUser, params, this); }, function setApiKey(err){ assert.ifError(err); @@ -170,7 +171,7 @@ NamedMapsController.prototype.create = function(req, res) { step( function checkPerms(){ - self.serverOptions.authorizedByAPIKey(req, this); + self.app.authorizedByAPIKey(req, this); }, function addTemplate(err, authenticated) { assert.ifError(err); @@ -198,7 +199,7 @@ NamedMapsController.prototype.update = function(req, res) { var tpl_id; step( function checkPerms(){ - self.serverOptions.authorizedByAPIKey(req, this); + self.app.authorizedByAPIKey(req, this); }, function updateTemplate(err, authenticated) { assert.ifError(err); @@ -232,7 +233,7 @@ NamedMapsController.prototype.retrieve = function(req, res) { var tpl_id; step( function checkPerms(){ - self.serverOptions.authorizedByAPIKey(req, this); + self.app.authorizedByAPIKey(req, this); }, function getTemplate(err, authenticated) { assert.ifError(err); @@ -270,7 +271,7 @@ NamedMapsController.prototype.destroy = function(req, res) { var tpl_id; step( function checkPerms(){ - self.serverOptions.authorizedByAPIKey(req, this); + self.app.authorizedByAPIKey(req, this); }, function deleteTemplate(err, authenticated) { assert.ifError(err); @@ -299,7 +300,7 @@ NamedMapsController.prototype.list = function(req, res) { step( function checkPerms(){ - self.serverOptions.authorizedByAPIKey(req, this); + self.app.authorizedByAPIKey(req, this); }, function listTemplates(err, authenticated) { assert.ifError(err); @@ -376,10 +377,10 @@ NamedMapsController.prototype.instantiateTemplate = function(req, res, template_ var layergroupDecorator = { beforeLayergroupCreate: function(requestMapConfig, callback) { - self.serverOptions.beforeLayergroupCreate(req, requestMapConfig, callback); + self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback); }, afterLayergroupCreate: function(layergroup, response, callback) { - self.serverOptions.afterLayergroupCreate(req, layergroup, response, callback); + self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback); } }; @@ -431,7 +432,7 @@ NamedMapsController.prototype.instantiateTemplate = function(req, res, template_ if ( req.profiler ) req.profiler.done('TemplateMaps_instance'); if ( err ) throw err; layergroup = instance; - self.serverOptions.setDBParams(cdbuser, params, this); + self.app.setDBParams(cdbuser, params, this); }, function setApiKey(err){ if ( req.profiler ) req.profiler.done('setDBParams'); diff --git a/lib/cartodb/controllers/named_static_maps.js b/lib/cartodb/controllers/named_static_maps.js index d160912a..89428262 100644 --- a/lib/cartodb/controllers/named_static_maps.js +++ b/lib/cartodb/controllers/named_static_maps.js @@ -6,7 +6,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var _ = require('underscore'); function NamedStaticMapsController(app, serverOptions, templateMaps, mapBackend, staticMapBackend, surrogateKeysCache, - tablesExtentApi) { + tablesExtentApi, layergroupRequestDecorator) { this.app = app; this.serverOptions = serverOptions; this.templateMaps = templateMaps; @@ -14,6 +14,7 @@ function NamedStaticMapsController(app, serverOptions, templateMaps, mapBackend, this.staticMapBackend = staticMapBackend; this.surrogateKeysCache = surrogateKeysCache; this.tablesExtentApi = tablesExtentApi; + this.layergroupRequestDecorator = layergroupRequestDecorator; } module.exports = NamedStaticMapsController; @@ -40,10 +41,10 @@ NamedStaticMapsController.prototype.named = function(req, res) { var layergroupDecorator = { beforeLayergroupCreate: function(requestMapConfig, callback) { - self.serverOptions.beforeLayergroupCreate(req, requestMapConfig, callback); + self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback); }, afterLayergroupCreate: function(layergroup, response, callback) { - self.serverOptions.afterLayergroupCreate(req, layergroup, response, callback); + self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback); } }; @@ -112,7 +113,7 @@ NamedStaticMapsController.prototype.named = function(req, res) { params = { user: req.params.user }; - self.serverOptions.setDBParams(cdbUser, params, this); + self.app.setDBParams(cdbUser, params, this); }, function setApiKey(err){ assert.ifError(err); diff --git a/lib/cartodb/controllers/static_maps.js b/lib/cartodb/controllers/static_maps.js new file mode 100644 index 00000000..6511dc76 --- /dev/null +++ b/lib/cartodb/controllers/static_maps.js @@ -0,0 +1,71 @@ +var step = require('step'); +var assert = require('assert'); + +function StaticMapsController(app, staticMapBackend) { + this._app = app; + this._staticMapBackend = staticMapBackend; +} + +module.exports = StaticMapsController; + + +StaticMapsController.prototype.register = function(app) { + app.get(app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format', + this.center.bind(this)); + + app.get(app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', + this.bbox.bind(this)); +}; + +StaticMapsController.prototype.bbox = function(req, res) { + this.staticMap(req, res, +req.params.width, +req.params.height, { + west: +req.params.west, + north: +req.params.north, + east: +req.params.east, + south: +req.params.south + }); +}; + +StaticMapsController.prototype.center = function(req, res) { + this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, { + lng: +req.params.lng, + lat: +req.params.lat + }); +}; + +StaticMapsController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) { + this._app.doCORS(res); + + var format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + + var self = this; + + step( + function() { + self._app.req2params(req, this); + }, + function(err) { + req.profiler.done('req2params'); + assert.ifError(err); + if (center) { + self._staticMapBackend.getImage(req.params, width, height, zoom, center, this); + } else { + self._staticMapBackend.getImage(req.params, width, height, zoom /* bounds */, this); + } + }, + function handleImage(err, image, headers, stats) { + req.profiler.done('render-' + format); + req.profiler.add(stats || {}); + + if (err) { + if (!err.error) { + err.error = err.message; + } + self._app.sendError(res, {errors: ['' + err] }, self._app.findStatusCode(err), 'STATIC_MAP', err); + } else { + res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format); + self._app.sendResponse(res, [image, 200]); + } + } + ); +}; diff --git a/lib/cartodb/monitoring/health_check.js b/lib/cartodb/monitoring/health_check.js index 1fca02b8..fc1765cd 100644 --- a/lib/cartodb/monitoring/health_check.js +++ b/lib/cartodb/monitoring/health_check.js @@ -1,9 +1,7 @@ var fs = require('fs'); var step = require('step'); -function HealthCheck(metadataBackend, tilelive) { - this.metadataBackend = metadataBackend; - this.tilelive = tilelive; +function HealthCheck() { } module.exports = HealthCheck; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js new file mode 100644 index 00000000..6ff59abf --- /dev/null +++ b/lib/cartodb/server.js @@ -0,0 +1,1064 @@ +var assert = require('assert'); + +var express = require('express'); + +var RedisPool = require('redis-mpool'); +var _ = require('underscore'); + + +var step = require('step'); +var HealthCheck = require('./monitoring/health_check'); + + +var StaticMapsController = require('./controllers/static_maps'); +var MapController = require('./controllers/map'); + +var SurrogateKeysCache = require('./cache/surrogate_keys_cache'); +var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); +var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); +var FastlyCacheBackend = require('./cache/backend/fastly'); + +var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter'); + + +var windshaft = require('windshaft'); +var grainstore = windshaft.grainstore; +var mapnik = windshaft.mapnik; + +var TemplateMaps = require('./backends/template_maps.js'); +var QueryTablesApi = require('./api/query_tables_api'); +var PgQueryRunner = require('./backends/pg_query_runner'); +var PgConnection = require('./backends/pg_connection'); + +var CdbRequest = require('./models/cdb_request'); +var cdbRequest = new CdbRequest(); + +var LZMA = require('lzma').LZMA; +// Whitelist query parameters and attach format +var REQUEST_QUERY_PARAMS_WHITELIST = [ + 'config', + 'map_key', + 'api_key', + 'auth_token', + 'callback' +]; + +var lzmaWorker = new LZMA(); + + +var WELCOME_MSG = "This is the CartoDB Maps API, " + + "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html"; + + +var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; +var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); + + + +module.exports = function(serverOptions) { + + var redisPool = makeRedisPool(serverOptions.redis); + + var cartoData = require('cartodb-redis')({pool: redisPool}); + + var pgConnection = new PgConnection(cartoData); + var pgQueryRunner = new PgQueryRunner(pgConnection); + var queryTablesApi = new QueryTablesApi(pgQueryRunner); + + var templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + + // This is for Templated maps + // + // "named" is the official, "template" is for backward compatibility up to 1.6.x + // + var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)'; + + var surrogateKeysCacheBackends = []; + + if (serverOptions.varnish_purge_enabled) { + surrogateKeysCacheBackends.push( + new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port) + ); + } + + if (serverOptions.fastly && + !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) { + surrogateKeysCacheBackends.push( + new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId) + ); + } + + var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends); + + function invalidateNamedMap (owner, templateName) { + var startTime = Date.now(); + surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) { + var logMessage = JSON.stringify({ + username: owner, + type: 'named_map_invalidation', + elapsed: Date.now() - startTime, + error: !!err ? JSON.stringify(err.message) : undefined + }); + if (err) { + console.warn(logMessage); + } else { + console.info(logMessage); + } + }); + } + ['update', 'delete'].forEach(function(eventType) { + templateMaps.on(eventType, invalidateNamedMap); + }); + + serverOptions = serverOptions || {}; + + serverOptions.grainstore = serverOptions.grainstore || {}; + serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions); + + validateOptions(serverOptions); + + bootstrapFonts(serverOptions); + + // Make stats client globally accessible + global.statsClient = windshaft.stats.Client.getInstance(serverOptions.statsd); + + // initialize express server + var app = bootstrap(serverOptions); + addFilters(app, serverOptions); + + var map_store = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: serverOptions.grainstore.default_layergroup_ttl + }); + + serverOptions.renderer = serverOptions.renderer || {}; + + var onTileErrorStrategy; + if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { + onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) { + if (err && err.message === 'Render timed out' && format === 'png') { + return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {}); + } else { + return callback(err, tile, headers, stats); + } + }; + } + + var rendererFactory = new windshaft.renderer.Factory({ + onTileErrorStrategy: onTileErrorStrategy, + mapnik: { + redisPool: redisPool, + grainstore: serverOptions.grainstore, + mapnik: serverOptions.renderer.mapnik || serverOptions.mapnik + }, + http: serverOptions.renderer.http + }); + + // initialize render cache + var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, { + ttl: 60000, // 60 seconds TTL by default + statsInterval: 60000, // reports stats every milliseconds defined here + beforeRendererCreate: function(req, callback) { + var user = cdbRequest.userByReq(req); + + var rendererOptions = {}; + + step( + function getLimits(err) { + assert.ifError(err); + cartoData.getTilerRenderLimit(user, this); + }, + function handleTilerLimits(err, renderLimit) { + assert.ifError(err); + rendererOptions.limits = { + cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, + render: renderLimit || serverOptions.renderer.mapnik.limits.render || 0 + }; + return null; + }, + function finish(err) { + if (err) { + return callback(err); + } + + return callback(null, rendererOptions); + } + ); + } + }); + var rendererCache = new windshaft.cache.RendererCache(rendererCacheOpts, map_store, rendererFactory); + + // Make map store accessible as part of the app + app.mapStore = map_store; + var mapBackend = new windshaft.backend.Map(rendererCache, map_store); + var staticMapBackend = new windshaft.backend.StaticMap(rendererCache); + + app.findStatusCode = function(err) { + var statusCode; + if ( err.http_status ) { + statusCode = err.http_status; + } else { + statusCode = statusFromErrorMessage('' + err); + } + return statusCode; + }; + + /******************************************************************************************************************* + * Routing + ******************************************************************************************************************/ + + var namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + var layergroupRequestDecorator = { + beforeLayergroupCreate: function(req, requestMapConfig, callback) { + namedLayersAdapter.getLayers(cdbRequest.userByReq(req), requestMapConfig.layers, pgConnection, + function(err, layers, datasource) { + if (err) { + return callback(err); + } + + requestMapConfig.layers = layers; + return callback(null, requestMapConfig, datasource); + } + ); + }, + afterLayergroupCreate: function(req, mapconfig, response, callback) { + var token = response.layergroupid; + + var username = cdbRequest.userByReq(req); + + var tasksleft = 2; // redis key and affectedTables + var errors = []; + + var done = function(err) { + if ( err ) { + errors.push('' + err); + } + if ( ! --tasksleft ) { + err = errors.length ? new Error(errors.join('\n')) : null; + callback(err); + } + }; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + var serverMetadata = global.environment.serverMetadata; + if (serverMetadata) { + _.extend(response, serverMetadata); + } + + // Don't wait for the mapview count increment to + // take place before proceeding. Error will be logged + // asyncronously + cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) { + if (req.profiler) { + req.profiler.done('incMapviewCount'); + } + if ( err ) { + console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); + } + done(); + }); + + var sql = mapconfig.layers.map(function(layer) { + return layer.options.sql; + }).join(';'); + + var dbName = req.params.dbname; + var cacheKey = dbName + ':' + token; + + step( + function getAffectedTablesAndLastUpdatedTime() { + queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); + }, + function handleAffectedTablesAndLastUpdatedTime(err, result) { + if (req.profiler) { + req.profiler.done('queryTablesAndLastUpdated'); + } + assert.ifError(err); + var cacheChannel = app.buildCacheChannel(dbName, result.affectedTables); + app.channelCache[cacheKey] = cacheChannel; + + // last update for layergroup cache buster + response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime; + response.last_updated = new Date(result.lastUpdatedTime).toISOString(); + + var res = req.res; + if (res) { + if (req.method === 'GET') { + var ttl = global.environment.varnish.layergroupTtl || 86400; + res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); + res.header('Last-Modified', (new Date()).toUTCString()); + res.header('X-Cache-Channel', cacheChannel); + } + + res.header('X-Layergroup-Id', response.layergroupid); + } + + return null; + }, + function finish(err) { + done(err); + } + ); + } + }; + + var mapController = new MapController(app, mapBackend, layergroupRequestDecorator); + mapController.register(app); + + var staticMapsController = new StaticMapsController(app, staticMapBackend); + staticMapsController.register(app); + + var NamedMapsController = require('./controllers/named_maps'), + namedMapsController = new NamedMapsController( + app, + serverOptions, + templateMaps, + cartoData, + mapBackend, + template_baseurl, + surrogateKeysCache, + layergroupRequestDecorator + ); + namedMapsController.register(app); + + var TablesExtentApi = require('./api/tables_extent_api'); + var tablesExtentApi = new TablesExtentApi(pgQueryRunner); + + var NamedStaticMapsController = require('./controllers/named_static_maps'); + var namedStaticMapsController = new NamedStaticMapsController( + app, + serverOptions, + templateMaps, + mapBackend, + staticMapBackend, + surrogateKeysCache, + tablesExtentApi, + layergroupRequestDecorator + ); + namedStaticMapsController.register(app); + + + var healthCheck = new HealthCheck(); + app.get('/health', function(req, res) { + var healthConfig = global.environment.health || {}; + + if (!!healthConfig.enabled) { + var startTime = Date.now(); + healthCheck.check(healthConfig, function(err, result) { + var ok = !err; + var response = { + enabled: true, + ok: ok, + elapsed: Date.now() - startTime, + result: result + }; + if (err) { + response.err = err.message; + } + res.send(response, ok ? 200 : 503); + + }); + } else { + res.send({enabled: false, ok: true}, 200); + } + }); + + // simple testable route + app.get('/', function(req, res) { + app.sendResponse(res, [WELCOME_MSG]); + }); + + // version + app.get('/version', function(req, res) { + app.sendResponse(res, [app.getVersion(), 200]); + }); + + /******************************************************************************************************************* + * END Routing + ******************************************************************************************************************/ + + // temporary measure until we upgrade to newer version expressjs so we can check err.status + app.use(function(err, req, res, next) { + if (err) { + if (err.name === 'SyntaxError') { + app.sendError(res, { errors: [err.name + ': ' + err.message] }, 400, 'JSON', err); + } else { + next(err); + } + } else { + next(); + } + }); + + app.getVersion = function() { + return { + windshaft: require('../../package.json').version, + grainstore: grainstore.version(), + node_mapnik: mapnik.version, + mapnik: mapnik.versions.mapnik, + windshaft_cartodb: require('../../package.json').version + }; + }; + + + // GET routes for which we don't want to request any caching. + // POST/PUT/DELETE requests are never cached anyway. + var noCacheGETRoutes = [ + '/', + '/version', + // See https://github.com/CartoDB/Windshaft-cartodb/issues/176 + serverOptions.base_url_mapconfig, + serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format', + template_baseurl, + template_baseurl + '/:template_id', + template_baseurl + '/:template_id/jsonp' + ]; + + app.sendResponse = function(res, args) { + var that = this; + + var statusCode; + if ( res._windshaftStatusCode ) { + // Added by our override of sendError + statusCode = res._windshaftStatusCode; + } else { + if ( args.length > 2 ) statusCode = args[2]; + else { + statusCode = args[1] || 200; + } + } + var req = res.req; + step ( + function addCacheChannel() { + if ( ! req ) { + // having no associated request can happen when + // using fake response objects for testing layergroup + // creation + return false; + } + if ( ! req.params ) { + // service requests (/version, /) + // have no need for an X-Cache-Channel + return false; + } + if ( statusCode != 200 ) { + // We do not want to cache + // unsuccessful responses + return false; + } + if ( _.contains(noCacheGETRoutes, req.route.path) ) { +//console.log("Skipping cache channel in route:\n" + req.route.path); + return false; + } +//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + +// mapCreateRoutes.join("\n")); + app.addCacheChannel(that, req, this); + }, + function sendResponse(err/*, added*/) { + if ( err ) console.log(err + err.stack); + // When using custom results from tryFetch* methods, + // there is no "req" link in the result object. + // In those cases we don't want to send stats now + // as they will be sent at the real end of request + var req = res.req; + + if (global.environment && global.environment.api_hostname) { + res.header('X-Served-By-Host', global.environment.api_hostname); + } + + if (req && req.params && req.params.dbhost) { + res.header('X-Served-By-DB-Host', req.params.dbhost); + } + + if ( req && req.profiler ) { + res.header('X-Tiler-Profiler', req.profiler.toJSONString()); + } + + res.send.apply(res, args); + + if ( req && req.profiler ) { + try { + // May throw due to dns, see + // See http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + console.error("error sending profiling stats: " + err); + } + } + return null; + }, + function finish(err) { + if ( err ) console.log(err + err.stack); + } + ); + }; + + app.sendWithHeaders = function(res, what, status, headers) { + app.sendResponse(res, [what, headers, status]); + }; + + app.sendError = function(res, err, statusCode, label, tolog) { + res._windshaftStatusCode = statusCode; + + var olabel = '['; + if ( label ) { + olabel += label + ' '; + } + olabel += 'ERROR]'; + if ( ! tolog ) { + tolog = err; + } + var log_msg = olabel + " -- " + statusCode + ": " + tolog; + //if ( tolog.stack ) log_msg += "\n" + tolog.stack; + console.error(log_msg); // use console.log for statusCode != 500 ? + // If a callback was requested, force status to 200 + if ( res.req ) { + // NOTE: res.req can be undefined when we fake a call to + // ourself from POST to /layergroup + if ( res.req.query.callback ) { + statusCode = 200; + } + } + // Strip connection info, if any + // See https://github.com/CartoDB/Windshaft/issues/173 + err = JSON.stringify(err); + err = err.replace(/Connection string: '[^']*'\\n/, ''); + // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644 + err = err.replace(/is the server.*encountered/im, 'encountered'); + err = JSON.parse(err); + + app.sendResponse(res, [err, statusCode]); + }; + + // jshint maxcomplexity:10 + /** + * Whitelist input and get database name & default geometry type from + * subdomain/user metadata held in CartoDB Redis + * @param req - standard express request obj. Should have host & table + * @param callback + */ + app.req2params = function(req, callback){ + + if ( req.query.lzma ) { + + // Decode (from base64) + var lzma = new Buffer(req.query.lzma, 'base64') + .toString('binary') + .split('') + .map(function(c) { + return c.charCodeAt(0) - 128; + }); + + // Decompress + lzmaWorker.decompress( + lzma, + function(result) { + if (req.profiler) { + req.profiler.done('lzma'); + } + try { + delete req.query.lzma; + _.extend(req.query, JSON.parse(result)); + app.req2params(req, callback); + } catch (err) { + callback(new Error('Error parsing lzma as JSON: ' + err)); + } + } + ); + return; + } + + req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST); + req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object + + var user = cdbRequest.userByReq(req); + + if ( req.params.token ) { + //console.log("Request parameters include token " + req.params.token); + var tksplit = req.params.token.split(':'); + req.params.token = tksplit[0]; + if ( tksplit.length > 1 ) { + req.params.cache_buster= tksplit[1]; + } + tksplit = req.params.token.split('@'); + if ( tksplit.length > 1 ) { + req.params.signer = tksplit.shift(); + if ( ! req.params.signer ) { + req.params.signer = user; + } + else if ( req.params.signer !== user ) { + var err = new Error( + 'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"' + ); + err.http_status = 403; + callback(err); + return; + } + if ( tksplit.length > 1 ) { + /*var template_hash = */tksplit.shift(); // unused + } + req.params.token = tksplit.shift(); + //console.log("Request for token " + req.params.token + " with signature from " + req.params.signer); + } + } + + // bring all query values onto req.params object + _.extend(req.params, req.query); + + if (req.profiler) { + req.profiler.done('req2params.setup'); + } + + step( + function getPrivacy(){ + app.authorize(req, this); + }, + function gatekeep(err, authorized){ + if (req.profiler) { + req.profiler.done('authorize'); + } + assert.ifError(err); + if(!authorized) { + err = new Error("Sorry, you are unauthorized (permission denied)"); + err.http_status = 403; + throw err; + } + return null; + }, + function getDatabase(err){ + assert.ifError(err); + pgConnection.setDBConn(user, req.params, this); + }, + function finishSetup(err) { + if ( err ) { callback(err, req); return; } + + // Add default database connection parameters + // if none given + _.defaults(req.params, { + dbuser: global.environment.postgres.user, + dbpassword: global.environment.postgres.password, + dbhost: global.environment.postgres.host, + dbport: global.environment.postgres.port + }); + + callback(null, req); + } + ); + }; + + // TODO: review lifetime of elements of this cache + // NOTE: by-token indices should only be dropped when + // the corresponding layegroup is dropped, because + // we have no SQL after layer creation. + app.channelCache = {}; + + app.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames.join(','); + }; + + app.generateCacheChannel = function(app, req, callback){ + // Build channelCache key + var dbName = req.params.dbname; + var cacheKey = [ dbName, req.params.token ].join(':'); + + // no token means no tables associated + if (!req.params.token) { + return callback(null, this.buildCacheChannel(dbName, [])); + } + + step( + function checkCached() { + if ( app.channelCache.hasOwnProperty(cacheKey) ) { + return callback(null, app.channelCache[cacheKey]); + } + return null; + }, + function extractSQL(err) { + assert.ifError(err); + + // TODO: cached cache channel for token-based access should + // be constructed at renderer cache creation time + // See http://github.com/CartoDB/Windshaft-cartodb/issues/152 + if ( ! app.mapStore ) { + throw new Error('missing channel cache for token ' + req.params.token); + } + var mapStore = app.mapStore; + step( + function loadFromStore() { + mapStore.load(req.params.token, this); + }, + function getSQL(err, mapConfig) { + if (req.profiler) { + req.profiler.done('mapStore_load'); + } + assert.ifError(err); + + var queries = mapConfig.getLayers() + .map(function(lyr) { + return lyr.options.sql; + }) + .filter(function(sql) { + return !!sql; + }); + + return queries.length ? queries.join(';') : null; + }, + this + ); + }, + function findAffectedTables(err, sql) { + assert.ifError(err); + + if ( ! sql ) { + throw new Error("this request doesn't need an X-Cache-Channel generated"); + } + + queryTablesApi.getAffectedTablesInQuery(cdbRequest.userByReq(req), sql, this); // in addCacheChannel + }, + function buildCacheChannel(err, tableNames) { + assert.ifError(err); + + if (req.profiler) { + req.profiler.done('affectedTables'); + } + + var cacheChannel = app.buildCacheChannel(dbName,tableNames); + app.channelCache[cacheKey] = cacheChannel; + + return cacheChannel; + }, + function finish(err, cacheChannel) { + callback(err, cacheChannel); + } + ); + }; + + // Set the cache chanel info to invalidate the cache on the frontend server + // + // @param req The request object. + // The function will have no effect unless req.res exists. + // It is expected that req.params contains 'table' and 'dbname' + // + // @param cb function(err, channel) will be called when ready. + // the channel parameter will be null if nothing was added + // + app.addCacheChannel = function(app, req, cb) { + // skip non-GET requests, or requests for which there's no response + if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } + if (req.profiler) { + req.profiler.start('addCacheChannel'); + } + var res = req.res; + if ( req.params.token ) { + res.header('Cache-Control', 'public,max-age=31536000'); // 1 year + } else { + var ttl = global.environment.varnish.ttl || 86400; + res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + } + + // Set Last-Modified header + var lastUpdated; + if ( req.params.cache_buster ) { + // Assuming cache_buster is a timestamp + // FIXME: store lastModified in the cache channel instead + lastUpdated = new Date(parseInt(req.params.cache_buster)); + } else { + lastUpdated = new Date(); + } + res.header('Last-Modified', lastUpdated.toUTCString()); + + app.generateCacheChannel(app, req, function(err, channel){ + if (req.profiler) { + req.profiler.done('generateCacheChannel'); + req.profiler.end(); + } + if ( ! err ) { + res.header('X-Cache-Channel', channel); + cb(null, channel); + } else { + console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err )); + // TODO: evaluate if we should bubble up the error instead + cb(null, 'ERROR'); + } + }); + }; + + // Check if a request is authorized by a signer + // + // @param req express request object + // @param callback function(err, signed_by) signed_by will be + // null if the request is not signed by anyone + // or will be a string cartodb username otherwise. + // + app.authorizedBySigner = function(req, callback) { + if ( ! req.params.token || ! req.params.signer ) { + return callback(null, null); // no signer requested + } + + var signer = req.params.signer; + var layergroup_id = req.params.token; + var auth_token = req.params.auth_token; + + var mapStore = req.app.mapStore; + if (!mapStore) { + throw new Error('Unable to retrieve map configuration token'); + } + + mapStore.load(layergroup_id, function(err, mapConfig) { + if (err) { + return callback(err); + } + + var authorized = templateMaps.isAuthorized(mapConfig.obj().template, auth_token); + + return callback(null, authorized ? signer : null); + }); + }; + + // Check if a request is authorized by api_key + // + // @param req express request object + // @param callback function(err, authorized) + // NOTE: authorized is expected to be 0 or 1 (integer) + // + app.authorizedByAPIKey = function(req, callback) { + var givenKey = req.query.api_key || req.query.map_key; + if ( ! givenKey && req.body ) { + // check also in request body + givenKey = req.body.api_key || req.body.map_key; + } + if ( ! givenKey ) { + callback(null, 0); // no api key, no authorization... + return; + } + //console.log("given ApiKey: " + givenKey); + var user = cdbRequest.userByReq(req); + step( + function (){ + cartoData.getUserMapKey(user, this); + }, + function checkApiKey(err, val){ + assert.ifError(err); + return ( val && givenKey == val ) ? 1 : 0; + }, + function finish(err, authorized) { + callback(err, authorized); + } + ); + }; + + /** + * Check access authorization + * + * @param req - standard req object. Importantly contains table and host information + * @param callback function(err, allowed) is access allowed not? + */ + app.authorize = function(req, callback) { + var self = this; + var user = cdbRequest.userByReq(req); + + step( + function () { + self.authorizedByAPIKey(req, this); + }, + function checkApiKey(err, authorized){ + if (req.profiler) { + req.profiler.done('authorizedByAPIKey'); + } + assert.ifError(err); + + // if not authorized by api_key, continue + if (authorized !== 1) { + // not authorized by api_key, + // check if authorized by signer + self.authorizedBySigner(req, this); + return; + } + + // authorized by api key, login as the given username and stop + pgConnection.setDBAuth(user, req.params, function(err) { + callback(err, true); // authorized (or error) + }); + }, + function checkSignAuthorized(err, signed_by){ + if (err) { + return callback(err); + } + + if ( ! signed_by ) { + // request not authorized by signer. + + // if no signer name was given, let dbparams and + // PostgreSQL do the rest. + // + if ( ! req.params.signer ) { + callback(null, true); // authorized so far + return; + } + + // if signer name was given, return no authorization + callback(null, false); + return; + } + + pgConnection.setDBAuth(signed_by, req.params, function(err) { + if (req.profiler) { + req.profiler.done('setDBAuth'); + } + callback(err, true); // authorized (or error) + }); + } + ); + }; + + app.setDBParams = function(cdbuser, params, callback) { + step( + function setAuth() { + pgConnection.setDBAuth(cdbuser, params, this); + }, + function setConn(err) { + if ( err ) throw err; + pgConnection.setDBConn(cdbuser, params, this); + }, + function finish(err) { + callback(err); + } + ); + }; + + + + return app; +}; + +function validateOptions(opts) { + if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig)) { + throw new Error("Must initialise Windshaft with: 'base_url'/'base_url_mapconfig' URLs"); + } + + // Be nice and warn if configured mapnik version is != instaled mapnik version + if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) { + console.warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' + + ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')'); + } +} + +function makeRedisPool(redisOpts) { + redisOpts = redisOpts || {}; + return redisOpts.pool || new RedisPool(_.extend(redisOpts, {name: 'windshaft:server'})); +} + +function bootstrapFonts(opts) { + // Set carto renderer configuration for MMLStore + opts.grainstore.carto_env = opts.grainstore.carto_env || {}; + var cenv = opts.grainstore.carto_env; + cenv.validation_data = cenv.validation_data || {}; + if ( ! cenv.validation_data.fonts ) { + mapnik.register_system_fonts(); + mapnik.register_default_fonts(); + cenv.validation_data.fonts = _.keys(mapnik.fontFiles()); + } +} + +function bootstrap(opts) { + var app; + if (_.isObject(opts.https)) { + // use https if possible + app = express.createServer(opts.https); + } else { + // fall back to http by default + app = express.createServer(); + } + app.enable('jsonp callback'); + app.use(express.bodyParser()); + + app.use(function createRequestContext(req, res, next) { + req.context = req.context || {}; + next(); + }); + + // Use our step-profiler + app.use(function(req, res, next) { + req.profiler = new windshaft.stats.Profiler({ + statsd_client: global.statsClient, + profile: opts.useProfiler + }); + next(); + }); + + setupLogger(app, opts); + + return app; +} + +function setupLogger(app, opts) { + if (opts.log_format) { + var loggerOpts = { + // Allowing for unbuffered logging is mainly + // used to avoid hanging during unit testing. + // TODO: provide an explicit teardown function instead, + // releasing any event handler or timer set by + // this component. + buffer: !opts.unbuffered_logging, + // optional log format + format: opts.log_format + }; + if (global.log4js) { + app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'}))); + } else { + app.use(express.logger(loggerOpts)); + } + } +} + +// set default before/after filters if not set in opts object +function addFilters(app, opts) { + + // Extend windshaft with all the elements of the options object + _.extend(app, opts); + + // filters can be used for custom authentication, caching, logging etc + _.defaults(app, { + // Enable CORS access by web browsers if set + doCORS: function(res, extraHeaders) { + if (opts.enable_cors) { + var baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token"; + if(extraHeaders) { + baseHeaders += ", " + extraHeaders; + } + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", baseHeaders); + } + } + }); +} + +function statusFromErrorMessage(errMsg) { + // Find an appropriate statusCode based on message + var statusCode = 400; + if ( -1 !== errMsg.indexOf('permission denied') ) { + statusCode = 403; + } + else if ( -1 !== errMsg.indexOf('authentication failed') ) { + statusCode = 403; + } + else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) { + statusCode = 400; + } + else if ( -1 !== errMsg.indexOf('does not exist') ) { + if ( -1 !== errMsg.indexOf(' role ') ) { + statusCode = 403; // role 'xxx' does not exist + } else { + statusCode = 404; + } + } + return statusCode; +} + +function mapnikVersion(opts) { + return opts.grainstore.mapnik_version || mapnik.versions.mapnik; +} diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index bf9210c3..452b6ffa 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,53 +1,20 @@ var _ = require('underscore'); -var step = require('step'); -var LZMA = require('lzma').LZMA; -var assert = require('assert'); -var RedisPool = require('redis-mpool'); -var QueryTablesApi = require('./api/query_tables_api'); -var PgQueryRunner = require('./backends/pg_query_runner'); -var PgConnection = require('./backends/pg_connection'); -var TemplateMaps = require('./backends/template_maps.js'); -var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter'); -var CdbRequest = require('./models/cdb_request'); +var rendererConfig = _.defaults(global.environment.renderer || {}, { + cache_ttl: 60000, // milliseconds + statsInterval: 60000, + mapnik: { + poolSize: 8, + metatile: 2, + bufferSize: 64, + snapToGrid: false, + clipByBox2d: false, + limits: {} + }, + http: {} +}); -var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; -var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); - -// Whitelist query parameters and attach format -var REQUEST_QUERY_PARAMS_WHITELIST = [ - 'config', - 'map_key', - 'api_key', - 'auth_token', - 'callback' -]; - -module.exports = function(redisPool) { - redisPool = redisPool || new RedisPool(_.extend(global.environment.redis, {name: 'windshaft:server_options'})); - - var cartoData = require('cartodb-redis')({ pool: redisPool }); - var lzmaWorker = new LZMA(); - var pgConnection = new PgConnection(cartoData); - var pgQueryRunner = new PgQueryRunner(pgConnection); - var queryTablesApi = new QueryTablesApi(pgQueryRunner); - var cdbRequest = new CdbRequest(); - - var rendererConfig = _.defaults(global.environment.renderer || {}, { - cache_ttl: 60000, // milliseconds - statsInterval: 60000, - mapnik: { - poolSize: 8, - metatile: 2, - bufferSize: 64, - snapToGrid: false, - clipByBox2d: false, - limits: {} - }, - http: {} - }); - - var me = { +module.exports = { // This is for inline maps and table maps base_url: global.environment.base_url_legacy || '/tiles/:table', @@ -81,7 +48,8 @@ module.exports = function(redisPool) { mapnik: rendererConfig.mapnik, http: rendererConfig.http }, - redis: global.environment.redis, + // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 + redis: _.extend(global.environment.redis, {unwatchOnRelease: false}), enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, varnish_port: global.environment.varnish.port, @@ -92,560 +60,4 @@ module.exports = function(redisPool) { cache_enabled: global.environment.cache_enabled, log_format: global.environment.log_format, useProfiler: global.environment.useProfiler - }; - - // Do not send unwatch on release - // See http://github.com/CartoDB/Windshaft-cartodb/issues/161 - me.redis.unwatchOnRelease = false; - - // Re-use redisPool - me.redis.pool = redisPool; - - // Re-use pgConnection - me.pgConnection = pgConnection; - // Re-use pgQueryRunner - me.pgQueryRunner = pgQueryRunner; - - var templateMaps = new TemplateMaps(redisPool, { - max_user_templates: global.environment.maxUserTemplates - }); - me.templateMaps = templateMaps; - - var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); - -/* This whole block is about generating X-Cache-Channel { */ - - // TODO: review lifetime of elements of this cache - // NOTE: by-token indices should only be dropped when - // the corresponding layegroup is dropped, because - // we have no SQL after layer creation. - me.channelCache = {}; - - me.buildCacheChannel = function (dbName, tableNames){ - return dbName + ':' + tableNames.join(','); - }; - - me.generateCacheChannel = function(app, req, callback){ - // Build channelCache key - var dbName = req.params.dbname; - var cacheKey = [ dbName, req.params.token ].join(':'); - - // no token means no tables associated - if (!req.params.token) { - return callback(null, this.buildCacheChannel(dbName, [])); - } - - step( - function checkCached() { - if ( me.channelCache.hasOwnProperty(cacheKey) ) { - return callback(null, me.channelCache[cacheKey]); - } - return null; - }, - function extractSQL(err) { - assert.ifError(err); - - // TODO: cached cache channel for token-based access should - // be constructed at renderer cache creation time - // See http://github.com/CartoDB/Windshaft-cartodb/issues/152 - if ( ! app.mapStore ) { - throw new Error('missing channel cache for token ' + req.params.token); - } - var mapStore = app.mapStore; - step( - function loadFromStore() { - mapStore.load(req.params.token, this); - }, - function getSQL(err, mapConfig) { - if (req.profiler) { - req.profiler.done('mapStore_load'); - } - assert.ifError(err); - - var queries = mapConfig.getLayers() - .map(function(lyr) { - return lyr.options.sql; - }) - .filter(function(sql) { - return !!sql; - }); - - return queries.length ? queries.join(';') : null; - }, - this - ); - }, - function findAffectedTables(err, sql) { - assert.ifError(err); - - if ( ! sql ) { - throw new Error("this request doesn't need an X-Cache-Channel generated"); - } - - queryTablesApi.getAffectedTablesInQuery(cdbRequest.userByReq(req), sql, this); // in addCacheChannel - }, - function buildCacheChannel(err, tableNames) { - assert.ifError(err); - - if (req.profiler) { - req.profiler.done('affectedTables'); - } - - var cacheChannel = me.buildCacheChannel(dbName,tableNames); - me.channelCache[cacheKey] = cacheChannel; - - return cacheChannel; - }, - function finish(err, cacheChannel) { - callback(err, cacheChannel); - } - ); - }; - - // Set the cache chanel info to invalidate the cache on the frontend server - // - // @param req The request object. - // The function will have no effect unless req.res exists. - // It is expected that req.params contains 'table' and 'dbname' - // - // @param cb function(err, channel) will be called when ready. - // the channel parameter will be null if nothing was added - // - me.addCacheChannel = function(app, req, cb) { - // skip non-GET requests, or requests for which there's no response - if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } - if (req.profiler) { - req.profiler.start('addCacheChannel'); - } - var res = req.res; - if ( req.params.token ) { - res.header('Cache-Control', 'public,max-age=31536000'); // 1 year - } else { - var ttl = global.environment.varnish.ttl || 86400; - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); - } - - // Set Last-Modified header - var lastUpdated; - if ( req.params.cache_buster ) { - // Assuming cache_buster is a timestamp - // FIXME: store lastModified in the cache channel instead - lastUpdated = new Date(parseInt(req.params.cache_buster)); - } else { - lastUpdated = new Date(); - } - res.header('Last-Modified', lastUpdated.toUTCString()); - - me.generateCacheChannel(app, req, function(err, channel){ - if (req.profiler) { - req.profiler.done('generateCacheChannel'); - req.profiler.end(); - } - if ( ! err ) { - res.header('X-Cache-Channel', channel); - cb(null, channel); - } else { - console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err )); - // TODO: evaluate if we should bubble up the error instead - cb(null, 'ERROR'); - } - }); - }; - - - if (global.environment.enabledFeatures.onTileErrorStrategy !== false) { - me.renderer.onTileErrorStrategy = function(err, tile, headers, stats, format, callback) { - if (err && err.message === 'Render timed out' && format === 'png') { - return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {}); - } else { - return callback(err, tile, headers, stats); - } - }; - } - - me.renderCache.beforeRendererCreate = function(req, callback) { - var user = cdbRequest.userByReq(req); - - var rendererOptions = {}; - - step( - function getLimits(err) { - assert.ifError(err); - cartoData.getTilerRenderLimit(user, this); - }, - function handleTilerLimits(err, renderLimit) { - assert.ifError(err); - rendererOptions.limits = { - cacheOnTimeout: rendererConfig.mapnik.limits.cacheOnTimeout || false, - render: renderLimit || rendererConfig.mapnik.limits.render || 0 - }; - return null; - }, - function finish(err) { - if (err) { - return callback(err); - } - - return callback(null, rendererOptions); - } - ); - }; - - me.beforeLayergroupCreate = function(req, requestMapConfig, callback) { - mapConfigNamedLayersAdapter.getLayers(cdbRequest.userByReq(req), requestMapConfig.layers, pgConnection, - function(err, layers, datasource) { - if (err) { - return callback(err); - } - - requestMapConfig.layers = layers; - return callback(null, requestMapConfig, datasource); - } - ); - }; - - me.afterLayergroupCreate = function(req, mapconfig, response, callback) { - var token = response.layergroupid; - - var username = cdbRequest.userByReq(req); - - var tasksleft = 2; // redis key and affectedTables - var errors = []; - - var done = function(err) { - if ( err ) { - errors.push('' + err); - } - if ( ! --tasksleft ) { - err = errors.length ? new Error(errors.join('\n')) : null; - callback(err); - } - }; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - var serverMetadata = global.environment.serverMetadata; - if (serverMetadata) { - _.extend(response, serverMetadata); - } - - // Don't wait for the mapview count increment to - // take place before proceeding. Error will be logged - // asyncronously - cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) { - if (req.profiler) { - req.profiler.done('incMapviewCount'); - } - if ( err ) { - console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); - } - done(); - }); - - var sql = mapconfig.layers.map(function(layer) { - return layer.options.sql; - }).join(';'); - - var dbName = req.params.dbname; - var cacheKey = dbName + ':' + token; - - step( - function getAffectedTablesAndLastUpdatedTime() { - queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); - }, - function handleAffectedTablesAndLastUpdatedTime(err, result) { - if (req.profiler) { - req.profiler.done('queryTablesAndLastUpdated'); - } - assert.ifError(err); - var cacheChannel = me.buildCacheChannel(dbName, result.affectedTables); - me.channelCache[cacheKey] = cacheChannel; - - // last update for layergroup cache buster - response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime; - response.last_updated = new Date(result.lastUpdatedTime).toISOString(); - - var res = req.res; - if (res) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - res.header('Last-Modified', (new Date()).toUTCString()); - res.header('X-Cache-Channel', cacheChannel); - } - - res.header('X-Layergroup-Id', response.layergroupid); - } - - return null; - }, - function finish(err) { - done(err); - } - ); - }; - -/* X-Cache-Channel generation } */ - - // Check if a request is authorized by a signer - // - // @param req express request object - // @param callback function(err, signed_by) signed_by will be - // null if the request is not signed by anyone - // or will be a string cartodb username otherwise. - // - me.authorizedBySigner = function(req, callback) { - if ( ! req.params.token || ! req.params.signer ) { - return callback(null, null); // no signer requested - } - - var signer = req.params.signer; - var layergroup_id = req.params.token; - var auth_token = req.params.auth_token; - - var mapStore = req.app.mapStore; - if (!mapStore) { - throw new Error('Unable to retrieve map configuration token'); - } - - mapStore.load(layergroup_id, function(err, mapConfig) { - if (err) { - return callback(err); - } - - var authorized = me.templateMaps.isAuthorized(mapConfig.obj().template, auth_token); - - return callback(null, authorized ? signer : null); - }); - }; - - // Check if a request is authorized by api_key - // - // @param req express request object - // @param callback function(err, authorized) - // NOTE: authorized is expected to be 0 or 1 (integer) - // - me.authorizedByAPIKey = function(req, callback) - { - var givenKey = req.query.api_key || req.query.map_key; - if ( ! givenKey && req.body ) { - // check also in request body - givenKey = req.body.api_key || req.body.map_key; - } - if ( ! givenKey ) { - callback(null, 0); // no api key, no authorization... - return; - } - //console.log("given ApiKey: " + givenKey); - var user = cdbRequest.userByReq(req); - step( - function (){ - cartoData.getUserMapKey(user, this); - }, - function checkApiKey(err, val){ - assert.ifError(err); - return ( val && givenKey == val ) ? 1 : 0; - }, - function finish(err, authorized) { - callback(err, authorized); - } - ); - }; - - /** - * Check access authorization - * - * @param req - standard req object. Importantly contains table and host information - * @param callback function(err, allowed) is access allowed not? - */ - me.authorize = function(req, callback) { - var that = this; - var user = cdbRequest.userByReq(req); - - step( - function (){ - that.authorizedByAPIKey(req, this); - }, - function checkApiKey(err, authorized){ - if (req.profiler) { - req.profiler.done('authorizedByAPIKey'); - } - assert.ifError(err); - - // if not authorized by api_key, continue - if (authorized !== 1) { - // not authorized by api_key, - // check if authorized by signer - that.authorizedBySigner(req, this); - return; - } - - // authorized by api key, login as the given username and stop - pgConnection.setDBAuth(user, req.params, function(err) { - callback(err, true); // authorized (or error) - }); - }, - function checkSignAuthorized(err, signed_by){ - if (err) { - return callback(err); - } - - if ( ! signed_by ) { - // request not authorized by signer. - - // if no signer name was given, let dbparams and - // PostgreSQL do the rest. - // - if ( ! req.params.signer ) { - callback(null, true); // authorized so far - return; - } - - // if signer name was given, return no authorization - callback(null, false); - return; - } - - pgConnection.setDBAuth(signed_by, req.params, function(err) { - if (req.profiler) { - req.profiler.done('setDBAuth'); - } - callback(err, true); // authorized (or error) - }); - } - ); - }; - - me.setDBParams = function(cdbuser, params, callback) { - step( - function setAuth() { - pgConnection.setDBAuth(cdbuser, params, this); - }, - function setConn(err) { - if ( err ) throw err; - pgConnection.setDBConn(cdbuser, params, this); - }, - function finish(err) { - callback(err); - } - ); - }; - - // jshint maxcomplexity:10 - /** - * Whitelist input and get database name & default geometry type from - * subdomain/user metadata held in CartoDB Redis - * @param req - standard express request obj. Should have host & table - * @param callback - */ - me.req2params = function(req, callback){ - - if ( req.query.lzma ) { - - // Decode (from base64) - var lzma = new Buffer(req.query.lzma, 'base64') - .toString('binary') - .split('') - .map(function(c) { - return c.charCodeAt(0) - 128; - }); - - // Decompress - lzmaWorker.decompress( - lzma, - function(result) { - if (req.profiler) { - req.profiler.done('lzma'); - } - try { - delete req.query.lzma; - _.extend(req.query, JSON.parse(result)); - me.req2params(req, callback); - } catch (err) { - callback(new Error('Error parsing lzma as JSON: ' + err)); - } - } - ); - return; - } - - req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST); - req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object - - var user = cdbRequest.userByReq(req); - - if ( req.params.token ) { - //console.log("Request parameters include token " + req.params.token); - var tksplit = req.params.token.split(':'); - req.params.token = tksplit[0]; - if ( tksplit.length > 1 ) { - req.params.cache_buster= tksplit[1]; - } - tksplit = req.params.token.split('@'); - if ( tksplit.length > 1 ) { - req.params.signer = tksplit.shift(); - if ( ! req.params.signer ) { - req.params.signer = user; - } - else if ( req.params.signer !== user ) { - var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + - user + '"'); - err.http_status = 403; - callback(err); - return; - } - if ( tksplit.length > 1 ) { - /*var template_hash = */tksplit.shift(); // unused - } - req.params.token = tksplit.shift(); - //console.log("Request for token " + req.params.token + " with signature from " + req.params.signer); - } - } - - // bring all query values onto req.params object - _.extend(req.params, req.query); - - if (req.profiler) { - req.profiler.done('req2params.setup'); - } - - step( - function getPrivacy(){ - me.authorize(req, this); - }, - function gatekeep(err, authorized){ - if (req.profiler) { - req.profiler.done('authorize'); - } - assert.ifError(err); - if(!authorized) { - err = new Error("Sorry, you are unauthorized (permission denied)"); - err.http_status = 403; - throw err; - } - return null; - }, - function getDatabase(err){ - assert.ifError(err); - pgConnection.setDBConn(user, req.params, this); - }, - function finishSetup(err) { - if ( err ) { callback(err, req); return; } - - // Add default database connection parameters - // if none given - _.defaults(req.params, { - dbuser: global.environment.postgres.user, - dbpassword: global.environment.postgres.password, - dbhost: global.environment.postgres.host, - dbport: global.environment.postgres.port - }); - - callback(null, req); - } - ); - }; - - return me; }; diff --git a/package.json b/package.json index 4b86e524..60dfb065 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "Sandro Santilli " ], "dependencies": { + "express": "~2.5.11", "underscore" : "~1.6.0", "dot": "~1.0.2", "windshaft": "https://github.com/CartoDB/Windshaft/tarball/backend-foundations", diff --git a/test/acceptance/cache/surrogate_keys_invalidation.js b/test/acceptance/cache/surrogate_keys_invalidation.js index c46ab8c1..d222a0ad 100644 --- a/test/acceptance/cache/surrogate_keys_invalidation.js +++ b/test/acceptance/cache/surrogate_keys_invalidation.js @@ -6,23 +6,25 @@ var step = require('step'); var FastlyPurge = require('fastly-purge'); var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry'); -var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/cartodb_windshaft'); +var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/server'); describe('templates surrogate keys', function() { var redisClient = redis.createClient(global.environment.redis.port); - // Enable Varnish purge for tests - var varnishHost = global.environment.varnish.host; - global.environment.varnish.host = '127.0.0.1'; - var varnishPurgeEnabled = global.environment.varnish.purge_enabled; - global.environment.varnish.purge_enabled = true; + var serverOptions = require('../../../lib/cartodb/server_options'); - var fastlyConfig = global.environment.fastly; + // Enable Varnish purge for tests + var varnishHost = serverOptions.varnish_host; + serverOptions.varnish_host = '127.0.0.1'; + var varnishPurgeEnabled = serverOptions.varnish_purge_enabled; + serverOptions.varnish_purge_enabled = true; + + var fastlyConfig = serverOptions.fastly; var FAKE_FASTLY_API_KEY = 'fastly-api-key'; var FAKE_FASTLY_SERVICE_ID = 'fake-service-id'; - global.environment.fastly = { + serverOptions.fastly = { enabled: true, // the fastly api key apiKey: FAKE_FASTLY_API_KEY, @@ -30,7 +32,6 @@ describe('templates surrogate keys', function() { serviceId: FAKE_FASTLY_SERVICE_ID }; - var serverOptions = require('../../../lib/cartodb/server_options')(); var server = new CartodbWindshaft(serverOptions); var templateOwner = 'localhost', @@ -58,7 +59,7 @@ describe('templates surrogate keys', function() { expectedBody = { template_id: expectedTemplateId }; var varnishHttpUrl = [ - 'http://', global.environment.varnish.host, ':', global.environment.varnish.http_port + 'http://', serverOptions.varnish_host, ':', serverOptions.varnish_http_port ].join(''); var cacheEntryKey = new NamedMapsCacheEntry(templateOwner, templateName).key(); @@ -70,10 +71,10 @@ describe('templates surrogate keys', function() { after(function(done) { serverOptions.varnish_purge_enabled = false; - global.environment.varnish.host = varnishHost; - global.environment.varnish.purge_enabled = varnishPurgeEnabled; + serverOptions.varnish_host = varnishHost; + serverOptions.varnish_purge_enabled = varnishPurgeEnabled; - global.environment.fastly = fastlyConfig; + serverOptions.fastly = fastlyConfig; nock.restore(); done(); diff --git a/test/acceptance/health_check.js b/test/acceptance/health_check.js index 274e9cfc..2a6d08a7 100644 --- a/test/acceptance/health_check.js +++ b/test/acceptance/health_check.js @@ -1,8 +1,8 @@ require(__dirname + '/../support/test_helper'); var assert = require('../support/assert'); -var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')(); +var CartodbWindshaft = require('../../lib/cartodb/server'); +var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); var metadataBackend = {}; diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index 3864ceac..707b5e55 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -4,7 +4,7 @@ var assert = require('../support/assert'); var _ = require('underscore'); var redis = require('redis'); -var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft'); +var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); describe.skip('render limits', function() { @@ -22,7 +22,7 @@ describe.skip('render limits', function() { var server; beforeEach(function() { - server = new CartodbWindshaft(serverOptions()); + server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); }); diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 28ec7419..b23521ce 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -12,9 +12,9 @@ var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixture var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20; var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25; -var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); -var server = new CartodbWindshaft(serverOptions()); +var CartodbWindshaft = require('../../lib/cartodb/server'); +var serverOptions = require('../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); ['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) { @@ -859,7 +859,7 @@ suite(suiteName, function() { function do_restart_server(err/*, res*/) { if ( err ) throw err; // hack simulating restart... - server = new CartodbWindshaft(serverOptions()); + server = new CartodbWindshaft(serverOptions); return null; }, function do_get1(err) diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index 13563abb..4845545d 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -7,8 +7,8 @@ var _ = require('underscore'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); -var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require('../../lib/cartodb/server_options')(); +var CartodbWindshaft = require('../../lib/cartodb/server'); +var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); @@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() { }; // reset internal cacheChannel cache - serverOptions.channelCache = {}; + server.channelCache = {}; assert.response(server, { diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js index d485be29..71f8ea44 100644 --- a/test/acceptance/named_layers.js +++ b/test/acceptance/named_layers.js @@ -1,8 +1,8 @@ var test_helper = require('../support/test_helper'); var assert = require('../support/assert'); -var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')(); +var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); var RedisPool = require('redis-mpool'); diff --git a/test/acceptance/named_static_maps.js b/test/acceptance/named_static_maps.js index 838eade2..3df35b22 100644 --- a/test/acceptance/named_static_maps.js +++ b/test/acceptance/named_static_maps.js @@ -3,8 +3,8 @@ var RedisPool = require('redis-mpool'); var querystring = require('querystring'); var assert = require('../support/assert'); -var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')(); +var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry'); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index ffb32fe4..a9fa2e35 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -4,8 +4,8 @@ var assert = require('../support/assert'); var querystring = require('querystring'); var step = require('step'); -var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require('../../lib/cartodb/server_options')(); +var CartodbWindshaft = require('../../lib/cartodb/server'); +var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 71c5e667..79f24ac5 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -14,13 +14,13 @@ process.env.PGHOST = 'fake'; var helper = require(__dirname + '/../support/test_helper'); -var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')(); +var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); describe('template_api', function() { - serverOptions.channelCache = {}; + server.channelCache = {}; var redis_client = redis.createClient(global.environment.redis.port); @@ -1154,7 +1154,7 @@ describe('template_api', function() { assert.ok(cc); assert.ok(cc.match, /ciao/, cc); // hack simulating restart... - serverOptions.channelCache = {}; // need to clean channel cache + server.channelCache = {}; // need to clean channel cache var get_request = { url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1', method: 'GET', diff --git a/test/acceptance/x_cache_channel.js b/test/acceptance/x_cache_channel.js index d70d1f6f..5ed26a90 100644 --- a/test/acceptance/x_cache_channel.js +++ b/test/acceptance/x_cache_channel.js @@ -3,8 +3,8 @@ require('../support/test_helper'); var assert = require('../support/assert'); var qs = require('querystring'); -var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft'); -var serverOptions = require('../../lib/cartodb/server_options')(); +var CartodbWindshaft = require('../../lib/cartodb/server'); +var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); diff --git a/test/support/assert.js b/test/support/assert.js index 421b9c06..b13cf88f 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -9,64 +9,6 @@ var exec = require('child_process').exec, var assert = module.exports = exports = require('assert'); -// @param tolerance number of tolerated grid cell differences -assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) { - fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update - var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8')); - - var err = null; - - var Celldiff = function(x, y, ev, ov) { - this.x = x; - this.y = y; - this.ev = ev; - this.ov = ov; - }; - - Celldiff.prototype.toString = function() { - return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]'; - }; - - try { - var obtained_json = JSON.parse(buffer); - - // compare grid - var obtained_grid = obtained_json.grid; - var expected_grid = expected_json.grid; - var nrows = obtained_grid.length - if (nrows != expected_grid.length) { - throw new Error( "Obtained grid rows (" + nrows + - ") != expected grid rows (" + expected_grid.length + ")" ); - } - var celldiff = []; - for (var i=0; i tolerance ) { - throw new Error( celldiff.length + " cell differences: " + celldiff ); - } - - assert.deepEqual(obtained_json.keys, expected_json.keys); - } catch (e) { err = e; } - - callback(err); -}; - /** * Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the * similarity is not within the tolerance limit it will callback with an error. @@ -78,15 +20,38 @@ assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) { * @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric */ assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) { - if (!callback) callback = function(err) { if (err) throw err; }; + callback = callback || function(err) { assert.ifError(err); }; var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath), - testImageFilePath = '/tmp/windshaft-test-image-' + (Math.random() * 1e16); // TODO: make predictable - var err = fs.writeFileSync(testImageFilePath, buffer, 'binary'); - if (err) throw err; + testImageFilePath = createImageFromBuffer(buffer, 'test'); + imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, function(err) { + fs.unlinkSync(testImageFilePath); + callback(err); + }); +}; + +assert.imageBuffersAreEqual = function(bufferA, bufferB, tolerance, callback) { + var randStr = (Math.random() * 1e16).toString().substring(0, 8); + var imageFilePathA = createImageFromBuffer(bufferA, randStr + '-a'), + imageFilePathB = createImageFromBuffer(bufferB, randStr + '-b'); + + imageFilesAreEqual(imageFilePathA, imageFilePathB, tolerance, function(err, similarity) { + callback(err, [imageFilePathA, imageFilePathB], similarity); + }); +}; + +function createImageFromBuffer(buffer, nameHint) { + var imageFilePath = path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png'); + var err = fs.writeFileSync(imageFilePath, buffer, 'binary'); + assert.ifError(err); + return imageFilePath; +} + +function imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, callback) { + var resultFilePath = path.resolve(util.format('/tmp/windshaft-result-%s-diff.png', Date.now())); var imageMagickCmd = util.format( - 'compare -metric fuzz "%s" "%s" /dev/null', - testImageFilePath, referenceImageFilePath + 'compare -metric fuzz "%s" "%s" "%s"', + testImageFilePath, referenceImageFilePath, resultFilePath ); exec(imageMagickCmd, function(err, stdout, stderr) { @@ -104,29 +69,29 @@ assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolera tolerancePerMil = (tolerance / 1000); if (similarity > tolerancePerMil) { err = new Error(util.format( - 'Images %s and %s are not equal (got %d similarity, expected %d)', - testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil) + 'Images %s and %s are not equal (got %d similarity, expected %d). Result %s', + testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil, resultFilePath) ); err.similarity = similarity; - callback(err); + callback(err, similarity); } else { - fs.unlinkSync(testImageFilePath); - callback(null); + fs.unlinkSync(resultFilePath); + callback(null, similarity); } } }); -}; - +} /** * Assert response from `server` with * the given `req` object and `res` assertions object. * - * @param {Server} server + * @param {Server|Object} server * @param {Object} req * @param {Object|Function} res - * @param {String} msg + * @param {String|Function} msg */ +// jshint maxcomplexity:12 assert.response = function(server, req, res, msg){ var port = 5555; function check(){ @@ -161,14 +126,12 @@ assert.response = function(server, req, res, msg){ } // Callback as third or fourth arg - var callback = typeof res === 'function' - ? res - : typeof msg === 'function' - ? msg - : function(){}; + var callback = typeof res === 'function' ? res : (typeof msg === 'function' ? msg : function(){}); // Default messate to test title - if (typeof msg === 'function') msg = null; + if (typeof msg === 'function') { + msg = null; + } msg = msg || assert.testTitle; msg += '. '; @@ -183,6 +146,7 @@ assert.response = function(server, req, res, msg){ issue(); } + // jshint maxcomplexity:8 function issue(){ // Issue request @@ -217,27 +181,29 @@ assert.response = function(server, req, res, msg){ }, requestTimeout); } - if (data) request.write(data); + if (data) { + request.write(data); + } request.on('response', function(response){ response.body = ''; response.setEncoding(encoding); response.on('data', function(chunk){ response.body += chunk; }); response.on('end', function(){ - if (timer) clearTimeout(timer); + if (timer) { + clearTimeout(timer); + } check(); // Assert response body if (res.body !== undefined) { - var eql = res.body instanceof RegExp - ? res.body.test(response.body) - : res.body === response.body; + var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body; assert.ok( eql, - msg + 'Invalid response body.\n' - + ' Expected: ' + res.body + '\n' - + ' Got: ' + response.body + msg + colorize('[red]{Invalid response body.}\n' + + ' Expected: [green]{' + res.body + '}\n' + + ' Got: [red]{' + response.body + '}') ); } @@ -246,10 +212,10 @@ assert.response = function(server, req, res, msg){ assert.equal( response.statusCode, status, - msg + colorize('Invalid response status code.\n' - + ' Expected: [green]{' + status + '}\n' - + ' Got: [red]{' + response.statusCode + '}\n' - + ' Response body: ' + response.body) + msg + colorize('[red]{Invalid response status code.}\n' + + ' Expected: [green]{' + status + '}\n' + + ' Got: [red]{' + response.statusCode + '}\n' + + ' Body: ' + response.body) ); } @@ -260,14 +226,12 @@ assert.response = function(server, req, res, msg){ var name = keys[i], actual = response.headers[name.toLowerCase()], expected = res.headers[name], - eql = expected instanceof RegExp - ? expected.test(actual) - : expected == actual; + headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual; assert.ok( - eql, - msg + 'Invalid response header [bold]{' + name + '}.\n' - + ' Expected: [green]{' + expected + '}\n' - + ' Got: [red]{' + actual + '}' + headerEql, + msg + colorize('Invalid response header [bold]{' + name + '}.\n' + + ' Expected: [green]{' + expected + '}\n' + + ' Got: [red]{' + actual + '}') ); } } @@ -278,9 +242,69 @@ assert.response = function(server, req, res, msg){ }); request.end(); - } + } }; +// @param tolerance number of tolerated grid cell differences +// jshint maxcomplexity:8 +assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) { + fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update + var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8')); + + var err = null; + + var Celldiff = function(x, y, ev, ov) { + this.x = x; + this.y = y; + this.ev = ev; + this.ov = ov; + }; + + Celldiff.prototype.toString = function() { + return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]'; + }; + + try { + var obtained_json = JSON.parse(buffer); + + // compare grid + var obtained_grid = obtained_json.grid; + var expected_grid = expected_json.grid; + var nrows = obtained_grid.length; + if (nrows !== expected_grid.length) { + throw new Error( "Obtained grid rows (" + nrows + + ") != expected grid rows (" + expected_grid.length + ")" ); + } + var celldiff = []; + for (var i=0; i tolerance ) { + throw new Error( celldiff.length + " cell differences: " + celldiff ); + } + + assert.deepEqual(obtained_json.keys, expected_json.keys); + } catch (e) { err = e; } + + callback(err); +}; + + /** * Colorize the given string using ansi-escape sequences. * Disabled when --boring is set. @@ -293,4 +317,4 @@ function colorize(str) { return str.replace(/\[(\w+)\]\{([^]*?)\}/g, function(_, color, str) { return '\x1B[' + colors[color] + 'm' + str + '\x1B[0m'; }); -} \ No newline at end of file +} diff --git a/test/support/test_helper.js b/test/support/test_helper.js index 9b4f0631..6bd63af3 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -5,7 +5,6 @@ * Desc: Loads test specific variables */ -var _ = require('underscore'); var assert = require('assert'); var LZMA = require('lzma').LZMA; @@ -20,11 +19,11 @@ process.env.NODE_ENV = 'test'; function lzma_compress_to_base64(payload, mode, callback) { lzmaWorker.compress(payload, mode, function(ints) { - ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('') + ints = ints.map(function(c) { return String.fromCharCode(c + 128); }).join(''); var base64 = new Buffer(ints, 'binary').toString('base64'); callback(null, base64); }, - function(percent) { + function(/*percent*/) { //console.log("Compressing: " + percent + "%"); } ); @@ -55,6 +54,21 @@ function checkSurrogateKey(res, expectedKey) { assert.equal(res.headers['surrogate-key'], expectedKey); } +//var _ = require('underscore'); +//var redis = require('redis'); +// global afterEach to capture tests that leave keys in redis +//afterEach(function(done) { +// var redisClient = redis.createClient(global.environment.redis.port); +// // Check that we start with an empty redis db +// redisClient.keys("*", function(err, keys) { +// if ( err ) { +// return done(err); +// } +// assert.equal(keys.length, 0, "test left objects in redis:\n" + keys.join("\n")); +// redisClient.flushall(done); +// }); +//}); + module.exports = { lzma_compress_to_base64: lzma_compress_to_base64, diff --git a/test/unit/cartodb/req2params.test.js b/test/unit/cartodb/req2params.test.js index 932bec05..e9659461 100644 --- a/test/unit/cartodb/req2params.test.js +++ b/test/unit/cartodb/req2params.test.js @@ -5,7 +5,9 @@ var test_helper = require('../../support/test_helper'); suite('req2params', function() { // configure redis pool instance to use in tests - var opts = require('../../../lib/cartodb/server_options')(); + var CartodbWindshaft = require('../../../lib/cartodb/server'); + var serverOptions = require('../../../lib/cartodb/server_options'); + var server = new CartodbWindshaft(serverOptions); var test_user = _.template(global.environment.postgres_auth_user, {user_id:1}); var test_pubuser = global.environment.postgres.user; @@ -13,11 +15,11 @@ suite('req2params', function() { test('can be found in server_options', function(){ - assert.ok(_.isFunction(opts.req2params)); + assert.ok(_.isFunction(server.req2params)); }); test('cleans up request', function(done){ - opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) { + server.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) { if ( err ) { done(err); return; } assert.ok(_.isObject(req.query), 'request has query'); assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query'); @@ -30,7 +32,7 @@ suite('req2params', function() { }); test('sets dbname from redis metadata', function(done){ - opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) { + server.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) { if ( err ) { done(err); return; } //console.dir(req); assert.ok(_.isObject(req.query), 'request has query'); @@ -44,7 +46,7 @@ suite('req2params', function() { }); test('sets also dbuser for authenticated requests', function(done){ - opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) { + server.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) { if ( err ) { done(err); return; } //console.dir(req); assert.ok(_.isObject(req.query), 'request has query'); @@ -54,7 +56,7 @@ suite('req2params', function() { assert.equal(req.params.dbname, test_database); assert.equal(req.params.dbuser, test_user); - opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) { + server.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) { // wrong key resets params to no user assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')'); done(); @@ -80,7 +82,7 @@ suite('req2params', function() { lzma: data } }; - opts.req2params(req, function(err, req) { + server.req2params(req, function(err, req) { if ( err ) { return done(err); }