cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Standalone server initial implementation

Browse Source 
- no dependency over Windshaft.Server
This commit is contained in:
Raul Ochoa
2015-07-04 20:41:22 +02:00
parent ed90cadd75
commit 2d3f2667ca
24 changed files with 1598 additions and 991 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+17
View File
@@ -27,6 +27,23 @@ test: config/environments/test.js
test/acceptance/*.js \
test/acceptance/cache/*.js
test-unit: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
test/unit/cartodb/*.js \
test/unit/cartodb/cache/model/*.js
test-integration: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
test/integration/*.js
test-acceptance: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} \
test/acceptance/*.js \
test/acceptance/cache/*.js
jshint:
@echo "***jshint***"
@./node_modules/.bin/jshint lib/ test/ app.js
app.js
+12 -2
View File
@@ -8,6 +8,7 @@
*/
var path = require('path');
var os = require('os');
var fs = require('fs');
var RedisPool = require('redis-mpool');
var _ = require('underscore');
@@ -68,10 +69,19 @@ global.logger = global.log4js.getLogger();
var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
redisPool = new RedisPool(redisOpts);
// Perform keyword substitution in statsd
// See https://github.com/CartoDB/Windshaft-cartodb/issues/153
if ( global.environment.statsd ) {
if ( global.environment.statsd.prefix ) {
var host_token = os.hostname().split('.').reverse().join('.');
global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
}
}
// Include cartodb_windshaft only _after_ the "global" variable is set
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
var cartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
serverOptions = require('./lib/cartodb/server_options')(redisPool);
var cartodbWindshaft = require('./lib/cartodb/server'),
serverOptions = require('./lib/cartodb/server_options');
var ws = cartodbWindshaft(serverOptions);
lib/cartodb/cartodb_windshaft.js
-220
View File
@@ -1,220 +0,0 @@
var _ = require('underscore');
var step = require('step');
var Windshaft = require('windshaft');
var os = require('os');
var HealthCheck = require('./monitoring/health_check');
var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
var FastlyCacheBackend = require('./cache/backend/fastly');
if ( ! process.env.PGAPPNAME )
process.env.PGAPPNAME='cartodb_tiler';
var CartodbWindshaft = function(serverOptions) {
// Perform keyword substitution in statsd
// See https://github.com/CartoDB/Windshaft-cartodb/issues/153
if ( global.environment.statsd ) {
if ( global.environment.statsd.prefix ) {
var host_token = os.hostname().split('.').reverse().join('.');
global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
}
}
var redisPool = serverOptions.redis.pool ||
require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}));
var cartoData = require('cartodb-redis')({pool: redisPool});
var templateMaps = serverOptions.templateMaps;
// This is for Templated maps
//
// "named" is the official, "template" is for backward compatibility up to 1.6.x
//
var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
var surrogateKeysCacheBackends = [];
if (serverOptions.varnish_purge_enabled) {
surrogateKeysCacheBackends.push(
new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
);
}
if (!!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
surrogateKeysCacheBackends.push(
new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
);
}
var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
function invalidateNamedMap (owner, templateName) {
var startTime = Date.now();
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
var logMessage = JSON.stringify({
username: owner,
type: 'named_map_invalidation',
elapsed: Date.now() - startTime,
error: !!err ? JSON.stringify(err.message) : undefined
});
if (err) {
console.warn(logMessage);
} else {
console.info(logMessage);
}
});
}
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, invalidateNamedMap);
});
// boot
var ws = new Windshaft.Server(serverOptions);
// Override getVersion to include cartodb-specific versions
var wsversion = ws.getVersion;
ws.getVersion = function() {
var version = wsversion();
version.windshaft_cartodb = require('../../package.json').version;
return version;
};
var ws_sendResponse = ws.sendResponse;
// GET routes for which we don't want to request any caching.
// POST/PUT/DELETE requests are never cached anyway.
var noCacheGETRoutes = [
'/',
'/version',
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
serverOptions.base_url_mapconfig,
serverOptions.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
template_baseurl,
template_baseurl + '/:template_id',
template_baseurl + '/:template_id/jsonp'
];
ws.sendResponse = function(res, args) {
var that = this;
var thatArgs = arguments;
var statusCode;
if ( res._windshaftStatusCode ) {
// Added by our override of sendError
statusCode = res._windshaftStatusCode;
} else {
if ( args.length > 2 ) statusCode = args[2];
else {
statusCode = args[1] || 200;
}
}
var req = res.req;
step (
function addCacheChannel() {
if ( ! req ) {
// having no associated request can happen when
// using fake response objects for testing layergroup
// creation
return false;
}
if ( ! req.params ) {
// service requests (/version, /)
// have no need for an X-Cache-Channel
return false;
}
if ( statusCode != 200 ) {
// We do not want to cache
// unsuccessful responses
return false;
}
if ( _.contains(noCacheGETRoutes, req.route.path) ) {
//console.log("Skipping cache channel in route:\n" + req.route.path);
return false;
}
//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" +
// mapCreateRoutes.join("\n"));
serverOptions.addCacheChannel(that, req, this);
},
function sendResponse(err/*, added*/) {
if ( err ) console.log(err + err.stack);
ws_sendResponse.apply(that, thatArgs);
return null;
},
function finish(err) {
if ( err ) console.log(err + err.stack);
}
);
};
var ws_sendError = ws.sendError;
ws.sendError = function() {
var res = arguments[0];
var statusCode = arguments[2];
res._windshaftStatusCode = statusCode;
ws_sendError.apply(this, arguments);
};
/*******************************************************************************************************************
* Routing
******************************************************************************************************************/
var NamedMapsController = require('./controllers/named_maps'),
namedMapsController = new NamedMapsController(
ws,
serverOptions,
templateMaps,
cartoData,
ws.mapBackend,
template_baseurl,
surrogateKeysCache
);
namedMapsController.register(ws);
var TablesExtentApi = require('./api/tables_extent_api');
var tablesExtentApi = new TablesExtentApi(serverOptions.pgQueryRunner);
var NamedStaticMapsController = require('./controllers/named_static_maps');
var namedStaticMapsController = new NamedStaticMapsController(
ws,
serverOptions,
templateMaps,
ws.mapBackend,
ws.staticMapBackend,
surrogateKeysCache,
tablesExtentApi
);
namedStaticMapsController.register(ws);
/*******************************************************************************************************************
* END Routing
******************************************************************************************************************/
var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive);
ws.get('/health', function(req, res) {
var healthConfig = global.environment.health || {};
if (!!healthConfig.enabled) {
var startTime = Date.now();
healthCheck.check(healthConfig, function(err, result) {
var ok = !err;
var response = {
enabled: true,
ok: ok,
elapsed: Date.now() - startTime,
result: result
};
if (err) {
response.err = err.message;
}
res.send(response, ok ? 200 : 503);
});
} else {
res.send({enabled: false, ok: true}, 200);
}
});
return ws;
};
module.exports = CartodbWindshaft;
lib/cartodb/controllers/map.js
+211
View File
@@ -0,0 +1,211 @@
var assert = require('assert');
var step = require('step');
/**
* @param app
* @param {MapBackend} mapBackend
* @param options
* @constructor
*/
function MapController(app, mapBackend, layergroupRequestDecorator) {
this._app = app;
this._mapBackend = mapBackend;
this._layergroupRequestDecorator = layergroupRequestDecorator;
}
module.exports = MapController;
MapController.prototype.register = function(app) {
app.get(app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format', this.tile.bind(this));
app.get(app.base_url_mapconfig + '/:token/:z/:x/:y.:format', this.tile.bind(this));
app.get(app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)', this.layer.bind(this));
app.options(app.base_url_mapconfig, this.cors.bind(this));
app.get(app.base_url_mapconfig, this.createGet.bind(this));
app.post(app.base_url_mapconfig, this.createPost.bind(this));
app.get(app.base_url_mapconfig + '/:token/:layer/attributes/:fid', this.attributes.bind(this));
};
// send CORS headers when client send options.
MapController.prototype.cors = function(req, res, next) {
this._app.doCORS(res, "Content-Type");
return next();
};
// Gets attributes for a given layer feature
MapController.prototype.attributes = function(req, res) {
var self = this;
req.profiler.start('windshaft.maplayer_attribute');
this._app.doCORS(res);
step(
function setupParams() {
self._app.req2params(req, this);
},
function retrieveFeatureAttributes(err) {
req.profiler.done('req2params');
assert.ifError(err);
self._mapBackend.getFeatureAttributes(req.params, false, this);
},
function finish(err, tile, stats) {
req.profiler.add(stats || {});
if (err) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
var statusCode = self._app.findStatusCode(err);
self._app.sendError(res, { errors: [errMsg] }, statusCode, 'GET ATTRIBUTES', err);
} else {
self._app.sendResponse(res, [tile, 200]);
}
}
);
};
MapController.prototype.create = function(req, res, prepareConfigFn) {
var self = this;
this._app.doCORS(res);
var layergroupDecorator = {
beforeLayergroupCreate: function(requestMapConfig, callback) {
self._layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback);
},
afterLayergroupCreate: function(layergroup, response, callback) {
self._layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback);
}
};
step(
function setupParams(){
self._app.req2params(req, this);
},
prepareConfigFn,
function initLayergroup(err, requestMapConfig) {
assert.ifError(err);
self._mapBackend.createLayergroup(requestMapConfig, req.params, layergroupDecorator, this);
},
function finish(err, response){
if (err) {
response = { errors: [ err.message ] };
var statusCode = self._app.findStatusCode(err);
self._app.sendError(res, response, statusCode, 'GET LAYERGROUP', err);
} else {
self._app.sendResponse(res, [response, 200]);
}
}
);
};
MapController.prototype.createGet = function(req, res){
req.profiler.start('windshaft.createmap_get');
this.create(req, res, function createGet$prepareConfig(err, req) {
assert.ifError(err);
if ( ! req.params.config ) {
throw new Error('layergroup GET needs a "config" parameter');
}
return JSON.parse(req.params.config);
});
};
// TODO rewrite this so it is possible to share code with `MapController::create` method
MapController.prototype.createPost = function(req, res) {
req.profiler.start('windshaft.createmap_post');
this.create(req, res, function createPost$prepareConfig(err, req) {
assert.ifError(err);
if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] !== 'application/json' ) {
throw new Error('layergroup POST data must be of type application/json');
}
return req.body;
});
};
// Gets a tile for a given token and set of tile ZXY coords. (OSM style)
MapController.prototype.tile = function(req, res) {
req.profiler.start('windshaft.map_tile');
this.tileOrLayer(req, res);
};
// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
MapController.prototype.layer = function(req, res, next) {
if (req.params.token === 'static') {
return next();
}
req.profiler.start('windshaft.maplayer_tile');
this.tileOrLayer(req, res);
};
MapController.prototype.tileOrLayer = function (req, res) {
var self = this;
this._app.doCORS(res);
step(
function mapController$prepareParams() {
self._app.req2params(req, this);
},
function mapController$getTileOrGrid(err) {
req.profiler.done('req2params');
if ( err ) {
throw err;
}
self._mapBackend.getTileOrGrid(req.params, this);
},
function mapController$finalize(err, tile, headers, stats) {
req.profiler.add(stats);
self.finalizeGetTileOrGrid(err, req, res, tile, headers);
return null;
},
function finish(err) {
if ( err ) {
console.error("windshaft.tiles: " + err);
}
}
);
};
// This function is meant for being called as the very last
// step by all endpoints serving tiles or grids
MapController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) {
var supportedFormats = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true
};
var formatStat = 'invalid';
if (req.params.format) {
var format = req.params.format.replace('.', '_');
if (supportedFormats[format]) {
formatStat = format;
}
}
if (err){
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
var statusCode = this._app.findStatusCode(err);
// Rewrite mapnik parsing errors to start with layer number
var matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = 'style'+matches[2]+': ' + matches[1];
}
this._app.sendError(res, { errors: ['' + errMsg] }, statusCode, 'TILE RENDER', err);
global.statsClient.increment('windshaft.tiles.error');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
} else {
this._app.sendWithHeaders(res, tile, 200, headers);
global.statsClient.increment('windshaft.tiles.success');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
}
};
lib/cartodb/controllers/named_maps.js
+13 -12
View File
@@ -6,7 +6,7 @@ var CdbRequest = require('../models/cdb_request');
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
function NamedMapsController(app, serverOptions, templateMaps, metadataBackend, mapBackend, templateBaseUrl,
surrogateKeysCache) {
surrogateKeysCache, layergroupRequestDecorator) {
this.app = app;
this.serverOptions = serverOptions;
this.templateMaps = templateMaps;
@@ -14,6 +14,7 @@ function NamedMapsController(app, serverOptions, templateMaps, metadataBackend,
this.mapBackend = mapBackend;
this.templateBaseUrl = templateBaseUrl;
this.surrogateKeysCache = surrogateKeysCache;
this.layergroupRequestDecorator = layergroupRequestDecorator
}
module.exports = NamedMapsController;
@@ -46,10 +47,10 @@ NamedMapsController.prototype.tile = function(req, res) {
var layergroupDecorator = {
beforeLayergroupCreate: function(requestMapConfig, callback) {
self.serverOptions.beforeLayergroupCreate(req, requestMapConfig, callback);
self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback);
},
afterLayergroupCreate: function(layergroup, response, callback) {
self.serverOptions.afterLayergroupCreate(req, layergroup, response, callback);
self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback);
}
};
@@ -118,7 +119,7 @@ NamedMapsController.prototype.tile = function(req, res) {
params = _.extend({}, req.params, {
user: req.params.user
});
self.serverOptions.setDBParams(cdbUser, params, this);
self.app.setDBParams(cdbUser, params, this);
},
function setApiKey(err){
assert.ifError(err);
@@ -170,7 +171,7 @@ NamedMapsController.prototype.create = function(req, res) {
step(
function checkPerms(){
self.serverOptions.authorizedByAPIKey(req, this);
self.app.authorizedByAPIKey(req, this);
},
function addTemplate(err, authenticated) {
assert.ifError(err);
@@ -198,7 +199,7 @@ NamedMapsController.prototype.update = function(req, res) {
var tpl_id;
step(
function checkPerms(){
self.serverOptions.authorizedByAPIKey(req, this);
self.app.authorizedByAPIKey(req, this);
},
function updateTemplate(err, authenticated) {
assert.ifError(err);
@@ -232,7 +233,7 @@ NamedMapsController.prototype.retrieve = function(req, res) {
var tpl_id;
step(
function checkPerms(){
self.serverOptions.authorizedByAPIKey(req, this);
self.app.authorizedByAPIKey(req, this);
},
function getTemplate(err, authenticated) {
assert.ifError(err);
@@ -270,7 +271,7 @@ NamedMapsController.prototype.destroy = function(req, res) {
var tpl_id;
step(
function checkPerms(){
self.serverOptions.authorizedByAPIKey(req, this);
self.app.authorizedByAPIKey(req, this);
},
function deleteTemplate(err, authenticated) {
assert.ifError(err);
@@ -299,7 +300,7 @@ NamedMapsController.prototype.list = function(req, res) {
step(
function checkPerms(){
self.serverOptions.authorizedByAPIKey(req, this);
self.app.authorizedByAPIKey(req, this);
},
function listTemplates(err, authenticated) {
assert.ifError(err);
@@ -376,10 +377,10 @@ NamedMapsController.prototype.instantiateTemplate = function(req, res, template_
var layergroupDecorator = {
beforeLayergroupCreate: function(requestMapConfig, callback) {
self.serverOptions.beforeLayergroupCreate(req, requestMapConfig, callback);
self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback);
},
afterLayergroupCreate: function(layergroup, response, callback) {
self.serverOptions.afterLayergroupCreate(req, layergroup, response, callback);
self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback);
}
};
@@ -431,7 +432,7 @@ NamedMapsController.prototype.instantiateTemplate = function(req, res, template_
if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
if ( err ) throw err;
layergroup = instance;
self.serverOptions.setDBParams(cdbuser, params, this);
self.app.setDBParams(cdbuser, params, this);
},
function setApiKey(err){
if ( req.profiler ) req.profiler.done('setDBParams');
lib/cartodb/controllers/named_static_maps.js
+5 -4
View File
@@ -6,7 +6,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var _ = require('underscore');
function NamedStaticMapsController(app, serverOptions, templateMaps, mapBackend, staticMapBackend, surrogateKeysCache,
tablesExtentApi) {
tablesExtentApi, layergroupRequestDecorator) {
this.app = app;
this.serverOptions = serverOptions;
this.templateMaps = templateMaps;
@@ -14,6 +14,7 @@ function NamedStaticMapsController(app, serverOptions, templateMaps, mapBackend,
this.staticMapBackend = staticMapBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.tablesExtentApi = tablesExtentApi;
this.layergroupRequestDecorator = layergroupRequestDecorator;
}
module.exports = NamedStaticMapsController;
@@ -40,10 +41,10 @@ NamedStaticMapsController.prototype.named = function(req, res) {
var layergroupDecorator = {
beforeLayergroupCreate: function(requestMapConfig, callback) {
self.serverOptions.beforeLayergroupCreate(req, requestMapConfig, callback);
self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback);
},
afterLayergroupCreate: function(layergroup, response, callback) {
self.serverOptions.afterLayergroupCreate(req, layergroup, response, callback);
self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback);
}
};
@@ -112,7 +113,7 @@ NamedStaticMapsController.prototype.named = function(req, res) {
params = {
user: req.params.user
};
self.serverOptions.setDBParams(cdbUser, params, this);
self.app.setDBParams(cdbUser, params, this);
},
function setApiKey(err){
assert.ifError(err);
lib/cartodb/controllers/static_maps.js
+71
View File
@@ -0,0 +1,71 @@
var step = require('step');
var assert = require('assert');
function StaticMapsController(app, staticMapBackend) {
this._app = app;
this._staticMapBackend = staticMapBackend;
}
module.exports = StaticMapsController;
StaticMapsController.prototype.register = function(app) {
app.get(app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
this.center.bind(this));
app.get(app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
this.bbox.bind(this));
};
StaticMapsController.prototype.bbox = function(req, res) {
this.staticMap(req, res, +req.params.width, +req.params.height, {
west: +req.params.west,
north: +req.params.north,
east: +req.params.east,
south: +req.params.south
});
};
StaticMapsController.prototype.center = function(req, res) {
this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
lng: +req.params.lng,
lat: +req.params.lat
});
};
StaticMapsController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) {
this._app.doCORS(res);
var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
var self = this;
step(
function() {
self._app.req2params(req, this);
},
function(err) {
req.profiler.done('req2params');
assert.ifError(err);
if (center) {
self._staticMapBackend.getImage(req.params, width, height, zoom, center, this);
} else {
self._staticMapBackend.getImage(req.params, width, height, zoom /* bounds */, this);
}
},
function handleImage(err, image, headers, stats) {
req.profiler.done('render-' + format);
req.profiler.add(stats || {});
if (err) {
if (!err.error) {
err.error = err.message;
}
self._app.sendError(res, {errors: ['' + err] }, self._app.findStatusCode(err), 'STATIC_MAP', err);
} else {
res.setHeader('Content-Type', headers['Content-Type'] || 'image/' + format);
self._app.sendResponse(res, [image, 200]);
}
}
);
};
lib/cartodb/monitoring/health_check.js
+1 -3
View File
@@ -1,9 +1,7 @@
var fs = require('fs');
var step = require('step');
function HealthCheck(metadataBackend, tilelive) {
this.metadataBackend = metadataBackend;
this.tilelive = tilelive;
function HealthCheck() {
}
module.exports = HealthCheck;
lib/cartodb/server.js
+1064
View File
File diff suppressed because it is too large Load Diff
lib/cartodb/server_options.js
+16 -604
View File
@@ -1,53 +1,20 @@
var _ = require('underscore');
var step = require('step');
var LZMA = require('lzma').LZMA;
var assert = require('assert');
var RedisPool = require('redis-mpool');
var QueryTablesApi = require('./api/query_tables_api');
var PgQueryRunner = require('./backends/pg_query_runner');
var PgConnection = require('./backends/pg_connection');
var TemplateMaps = require('./backends/template_maps.js');
var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter');
var CdbRequest = require('./models/cdb_request');
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
statsInterval: 60000,
mapnik: {
poolSize: 8,
metatile: 2,
bufferSize: 64,
snapToGrid: false,
clipByBox2d: false,
limits: {}
},
http: {}
});
var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
// Whitelist query parameters and attach format
var REQUEST_QUERY_PARAMS_WHITELIST = [
'config',
'map_key',
'api_key',
'auth_token',
'callback'
];
module.exports = function(redisPool) {
redisPool = redisPool || new RedisPool(_.extend(global.environment.redis, {name: 'windshaft:server_options'}));
var cartoData = require('cartodb-redis')({ pool: redisPool });
var lzmaWorker = new LZMA();
var pgConnection = new PgConnection(cartoData);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var queryTablesApi = new QueryTablesApi(pgQueryRunner);
var cdbRequest = new CdbRequest();
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
statsInterval: 60000,
mapnik: {
poolSize: 8,
metatile: 2,
bufferSize: 64,
snapToGrid: false,
clipByBox2d: false,
limits: {}
},
http: {}
});
var me = {
module.exports = {
// This is for inline maps and table maps
base_url: global.environment.base_url_legacy || '/tiles/:table',
@@ -81,7 +48,8 @@ module.exports = function(redisPool) {
mapnik: rendererConfig.mapnik,
http: rendererConfig.http
},
redis: global.environment.redis,
// Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),
enable_cors: global.environment.enable_cors,
varnish_host: global.environment.varnish.host,
varnish_port: global.environment.varnish.port,
@@ -92,560 +60,4 @@ module.exports = function(redisPool) {
cache_enabled: global.environment.cache_enabled,
log_format: global.environment.log_format,
useProfiler: global.environment.useProfiler
};
// Do not send unwatch on release
// See http://github.com/CartoDB/Windshaft-cartodb/issues/161
me.redis.unwatchOnRelease = false;
// Re-use redisPool
me.redis.pool = redisPool;
// Re-use pgConnection
me.pgConnection = pgConnection;
// Re-use pgQueryRunner
me.pgQueryRunner = pgQueryRunner;
var templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
me.templateMaps = templateMaps;
var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
/* This whole block is about generating X-Cache-Channel { */
// TODO: review lifetime of elements of this cache
// NOTE: by-token indices should only be dropped when
// the corresponding layegroup is dropped, because
// we have no SQL after layer creation.
me.channelCache = {};
me.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
me.generateCacheChannel = function(app, req, callback){
// Build channelCache key
var dbName = req.params.dbname;
var cacheKey = [ dbName, req.params.token ].join(':');
// no token means no tables associated
if (!req.params.token) {
return callback(null, this.buildCacheChannel(dbName, []));
}
step(
function checkCached() {
if ( me.channelCache.hasOwnProperty(cacheKey) ) {
return callback(null, me.channelCache[cacheKey]);
}
return null;
},
function extractSQL(err) {
assert.ifError(err);
// TODO: cached cache channel for token-based access should
// be constructed at renderer cache creation time
// See http://github.com/CartoDB/Windshaft-cartodb/issues/152
if ( ! app.mapStore ) {
throw new Error('missing channel cache for token ' + req.params.token);
}
var mapStore = app.mapStore;
step(
function loadFromStore() {
mapStore.load(req.params.token, this);
},
function getSQL(err, mapConfig) {
if (req.profiler) {
req.profiler.done('mapStore_load');
}
assert.ifError(err);
var queries = mapConfig.getLayers()
.map(function(lyr) {
return lyr.options.sql;
})
.filter(function(sql) {
return !!sql;
});
return queries.length ? queries.join(';') : null;
},
this
);
},
function findAffectedTables(err, sql) {
assert.ifError(err);
if ( ! sql ) {
throw new Error("this request doesn't need an X-Cache-Channel generated");
}
queryTablesApi.getAffectedTablesInQuery(cdbRequest.userByReq(req), sql, this); // in addCacheChannel
},
function buildCacheChannel(err, tableNames) {
assert.ifError(err);
if (req.profiler) {
req.profiler.done('affectedTables');
}
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
me.channelCache[cacheKey] = cacheChannel;
return cacheChannel;
},
function finish(err, cacheChannel) {
callback(err, cacheChannel);
}
);
};
// Set the cache chanel info to invalidate the cache on the frontend server
//
// @param req The request object.
// The function will have no effect unless req.res exists.
// It is expected that req.params contains 'table' and 'dbname'
//
// @param cb function(err, channel) will be called when ready.
// the channel parameter will be null if nothing was added
//
me.addCacheChannel = function(app, req, cb) {
// skip non-GET requests, or requests for which there's no response
if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
if (req.profiler) {
req.profiler.start('addCacheChannel');
}
var res = req.res;
if ( req.params.token ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
var ttl = global.environment.varnish.ttl || 86400;
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
}
// Set Last-Modified header
var lastUpdated;
if ( req.params.cache_buster ) {
// Assuming cache_buster is a timestamp
// FIXME: store lastModified in the cache channel instead
lastUpdated = new Date(parseInt(req.params.cache_buster));
} else {
lastUpdated = new Date();
}
res.header('Last-Modified', lastUpdated.toUTCString());
me.generateCacheChannel(app, req, function(err, channel){
if (req.profiler) {
req.profiler.done('generateCacheChannel');
req.profiler.end();
}
if ( ! err ) {
res.header('X-Cache-Channel', channel);
cb(null, channel);
} else {
console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
// TODO: evaluate if we should bubble up the error instead
cb(null, 'ERROR');
}
});
};
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
me.renderer.onTileErrorStrategy = function(err, tile, headers, stats, format, callback) {
if (err && err.message === 'Render timed out' && format === 'png') {
return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
} else {
return callback(err, tile, headers, stats);
}
};
}
me.renderCache.beforeRendererCreate = function(req, callback) {
var user = cdbRequest.userByReq(req);
var rendererOptions = {};
step(
function getLimits(err) {
assert.ifError(err);
cartoData.getTilerRenderLimit(user, this);
},
function handleTilerLimits(err, renderLimit) {
assert.ifError(err);
rendererOptions.limits = {
cacheOnTimeout: rendererConfig.mapnik.limits.cacheOnTimeout || false,
render: renderLimit || rendererConfig.mapnik.limits.render || 0
};
return null;
},
function finish(err) {
if (err) {
return callback(err);
}
return callback(null, rendererOptions);
}
);
};
me.beforeLayergroupCreate = function(req, requestMapConfig, callback) {
mapConfigNamedLayersAdapter.getLayers(cdbRequest.userByReq(req), requestMapConfig.layers, pgConnection,
function(err, layers, datasource) {
if (err) {
return callback(err);
}
requestMapConfig.layers = layers;
return callback(null, requestMapConfig, datasource);
}
);
};
me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
var token = response.layergroupid;
var username = cdbRequest.userByReq(req);
var tasksleft = 2; // redis key and affectedTables
var errors = [];
var done = function(err) {
if ( err ) {
errors.push('' + err);
}
if ( ! --tasksleft ) {
err = errors.length ? new Error(errors.join('\n')) : null;
callback(err);
}
};
// include in layergroup response the variables in serverMedata
// those variables are useful to send to the client information
// about how to reach this server or information about it
var serverMetadata = global.environment.serverMetadata;
if (serverMetadata) {
_.extend(response, serverMetadata);
}
// Don't wait for the mapview count increment to
// take place before proceeding. Error will be logged
// asyncronously
cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
if (req.profiler) {
req.profiler.done('incMapviewCount');
}
if ( err ) {
console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
}
done();
});
var sql = mapconfig.layers.map(function(layer) {
return layer.options.sql;
}).join(';');
var dbName = req.params.dbname;
var cacheKey = dbName + ':' + token;
step(
function getAffectedTablesAndLastUpdatedTime() {
queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) {
req.profiler.done('queryTablesAndLastUpdated');
}
assert.ifError(err);
var cacheChannel = me.buildCacheChannel(dbName, result.affectedTables);
me.channelCache[cacheKey] = cacheChannel;
// last update for layergroup cache buster
response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime;
response.last_updated = new Date(result.lastUpdatedTime).toISOString();
var res = req.res;
if (res) {
if (req.method === 'GET') {
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
res.header('Last-Modified', (new Date()).toUTCString());
res.header('X-Cache-Channel', cacheChannel);
}
res.header('X-Layergroup-Id', response.layergroupid);
}
return null;
},
function finish(err) {
done(err);
}
);
};
/* X-Cache-Channel generation } */
// Check if a request is authorized by a signer
//
// @param req express request object
// @param callback function(err, signed_by) signed_by will be
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
me.authorizedBySigner = function(req, callback) {
if ( ! req.params.token || ! req.params.signer ) {
return callback(null, null); // no signer requested
}
var signer = req.params.signer;
var layergroup_id = req.params.token;
var auth_token = req.params.auth_token;
var mapStore = req.app.mapStore;
if (!mapStore) {
throw new Error('Unable to retrieve map configuration token');
}
mapStore.load(layergroup_id, function(err, mapConfig) {
if (err) {
return callback(err);
}
var authorized = me.templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
return callback(null, authorized ? signer : null);
});
};
// Check if a request is authorized by api_key
//
// @param req express request object
// @param callback function(err, authorized)
// NOTE: authorized is expected to be 0 or 1 (integer)
//
me.authorizedByAPIKey = function(req, callback)
{
var givenKey = req.query.api_key || req.query.map_key;
if ( ! givenKey && req.body ) {
// check also in request body
givenKey = req.body.api_key || req.body.map_key;
}
if ( ! givenKey ) {
callback(null, 0); // no api key, no authorization...
return;
}
//console.log("given ApiKey: " + givenKey);
var user = cdbRequest.userByReq(req);
step(
function (){
cartoData.getUserMapKey(user, this);
},
function checkApiKey(err, val){
assert.ifError(err);
return ( val && givenKey == val ) ? 1 : 0;
},
function finish(err, authorized) {
callback(err, authorized);
}
);
};
/**
* Check access authorization
*
* @param req - standard req object. Importantly contains table and host information
* @param callback function(err, allowed) is access allowed not?
*/
me.authorize = function(req, callback) {
var that = this;
var user = cdbRequest.userByReq(req);
step(
function (){
that.authorizedByAPIKey(req, this);
},
function checkApiKey(err, authorized){
if (req.profiler) {
req.profiler.done('authorizedByAPIKey');
}
assert.ifError(err);
// if not authorized by api_key, continue
if (authorized !== 1) {
// not authorized by api_key,
// check if authorized by signer
that.authorizedBySigner(req, this);
return;
}
// authorized by api key, login as the given username and stop
pgConnection.setDBAuth(user, req.params, function(err) {
callback(err, true); // authorized (or error)
});
},
function checkSignAuthorized(err, signed_by){
if (err) {
return callback(err);
}
if ( ! signed_by ) {
// request not authorized by signer.
// if no signer name was given, let dbparams and
// PostgreSQL do the rest.
//
if ( ! req.params.signer ) {
callback(null, true); // authorized so far
return;
}
// if signer name was given, return no authorization
callback(null, false);
return;
}
pgConnection.setDBAuth(signed_by, req.params, function(err) {
if (req.profiler) {
req.profiler.done('setDBAuth');
}
callback(err, true); // authorized (or error)
});
}
);
};
me.setDBParams = function(cdbuser, params, callback) {
step(
function setAuth() {
pgConnection.setDBAuth(cdbuser, params, this);
},
function setConn(err) {
if ( err ) throw err;
pgConnection.setDBConn(cdbuser, params, this);
},
function finish(err) {
callback(err);
}
);
};
// jshint maxcomplexity:10
/**
* Whitelist input and get database name & default geometry type from
* subdomain/user metadata held in CartoDB Redis
* @param req - standard express request obj. Should have host & table
* @param callback
*/
me.req2params = function(req, callback){
if ( req.query.lzma ) {
// Decode (from base64)
var lzma = new Buffer(req.query.lzma, 'base64')
.toString('binary')
.split('')
.map(function(c) {
return c.charCodeAt(0) - 128;
});
// Decompress
lzmaWorker.decompress(
lzma,
function(result) {
if (req.profiler) {
req.profiler.done('lzma');
}
try {
delete req.query.lzma;
_.extend(req.query, JSON.parse(result));
me.req2params(req, callback);
} catch (err) {
callback(new Error('Error parsing lzma as JSON: ' + err));
}
}
);
return;
}
req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
var user = cdbRequest.userByReq(req);
if ( req.params.token ) {
//console.log("Request parameters include token " + req.params.token);
var tksplit = req.params.token.split(':');
req.params.token = tksplit[0];
if ( tksplit.length > 1 ) {
req.params.cache_buster= tksplit[1];
}
tksplit = req.params.token.split('@');
if ( tksplit.length > 1 ) {
req.params.signer = tksplit.shift();
if ( ! req.params.signer ) {
req.params.signer = user;
}
else if ( req.params.signer !== user ) {
var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' +
user + '"');
err.http_status = 403;
callback(err);
return;
}
if ( tksplit.length > 1 ) {
/*var template_hash = */tksplit.shift(); // unused
}
req.params.token = tksplit.shift();
//console.log("Request for token " + req.params.token + " with signature from " + req.params.signer);
}
}
// bring all query values onto req.params object
_.extend(req.params, req.query);
if (req.profiler) {
req.profiler.done('req2params.setup');
}
step(
function getPrivacy(){
me.authorize(req, this);
},
function gatekeep(err, authorized){
if (req.profiler) {
req.profiler.done('authorize');
}
assert.ifError(err);
if(!authorized) {
err = new Error("Sorry, you are unauthorized (permission denied)");
err.http_status = 403;
throw err;
}
return null;
},
function getDatabase(err){
assert.ifError(err);
pgConnection.setDBConn(user, req.params, this);
},
function finishSetup(err) {
if ( err ) { callback(err, req); return; }
// Add default database connection parameters
// if none given
_.defaults(req.params, {
dbuser: global.environment.postgres.user,
dbpassword: global.environment.postgres.password,
dbhost: global.environment.postgres.host,
dbport: global.environment.postgres.port
});
callback(null, req);
}
);
};
return me;
};
package.json
+1
View File
@@ -22,6 +22,7 @@
"Sandro Santilli <strk@vizzuality.com>"
],
"dependencies": {
"express": "~2.5.11",
"underscore" : "~1.6.0",
"dot": "~1.0.2",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/backend-foundations",
test/acceptance/cache/surrogate_keys_invalidation.js
Vendored
+14 -13
View File
@@ -6,23 +6,25 @@ var step = require('step');
var FastlyPurge = require('fastly-purge');
var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry');
var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/cartodb_windshaft');
var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/server');
describe('templates surrogate keys', function() {
var redisClient = redis.createClient(global.environment.redis.port);
// Enable Varnish purge for tests
var varnishHost = global.environment.varnish.host;
global.environment.varnish.host = '127.0.0.1';
var varnishPurgeEnabled = global.environment.varnish.purge_enabled;
global.environment.varnish.purge_enabled = true;
var serverOptions = require('../../../lib/cartodb/server_options');
var fastlyConfig = global.environment.fastly;
// Enable Varnish purge for tests
var varnishHost = serverOptions.varnish_host;
serverOptions.varnish_host = '127.0.0.1';
var varnishPurgeEnabled = serverOptions.varnish_purge_enabled;
serverOptions.varnish_purge_enabled = true;
var fastlyConfig = serverOptions.fastly;
var FAKE_FASTLY_API_KEY = 'fastly-api-key';
var FAKE_FASTLY_SERVICE_ID = 'fake-service-id';
global.environment.fastly = {
serverOptions.fastly = {
enabled: true,
// the fastly api key
apiKey: FAKE_FASTLY_API_KEY,
@@ -30,7 +32,6 @@ describe('templates surrogate keys', function() {
serviceId: FAKE_FASTLY_SERVICE_ID
};
var serverOptions = require('../../../lib/cartodb/server_options')();
var server = new CartodbWindshaft(serverOptions);
var templateOwner = 'localhost',
@@ -58,7 +59,7 @@ describe('templates surrogate keys', function() {
expectedBody = { template_id: expectedTemplateId };
var varnishHttpUrl = [
'http://', global.environment.varnish.host, ':', global.environment.varnish.http_port
'http://', serverOptions.varnish_host, ':', serverOptions.varnish_http_port
].join('');
var cacheEntryKey = new NamedMapsCacheEntry(templateOwner, templateName).key();
@@ -70,10 +71,10 @@ describe('templates surrogate keys', function() {
after(function(done) {
serverOptions.varnish_purge_enabled = false;
global.environment.varnish.host = varnishHost;
global.environment.varnish.purge_enabled = varnishPurgeEnabled;
serverOptions.varnish_host = varnishHost;
serverOptions.varnish_purge_enabled = varnishPurgeEnabled;
global.environment.fastly = fastlyConfig;
serverOptions.fastly = fastlyConfig;
nock.restore();
done();
test/acceptance/health_check.js
+2 -2
View File
@@ -1,8 +1,8 @@
require(__dirname + '/../support/test_helper');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var metadataBackend = {};
test/acceptance/limits.js
+2 -2
View File
@@ -4,7 +4,7 @@ var assert = require('../support/assert');
var _ = require('underscore');
var redis = require('redis');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
describe.skip('render limits', function() {
@@ -22,7 +22,7 @@ describe.skip('render limits', function() {
var server;
beforeEach(function() {
server = new CartodbWindshaft(serverOptions());
server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
});
test/acceptance/multilayer.js
+4 -4
View File
@@ -12,9 +12,9 @@ var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixture
var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20;
var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25;
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions());
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
@@ -859,7 +859,7 @@ suite(suiteName, function() {
function do_restart_server(err/*, res*/) {
if ( err ) throw err;
// hack simulating restart...
server = new CartodbWindshaft(serverOptions());
server = new CartodbWindshaft(serverOptions);
return null;
},
function do_get1(err)
test/acceptance/multilayer_server.js
+3 -3
View File
@@ -7,8 +7,8 @@ var _ = require('underscore');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
@@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() {
};
// reset internal cacheChannel cache
serverOptions.channelCache = {};
server.channelCache = {};
assert.response(server,
{
test/acceptance/named_layers.js
+2 -2
View File
@@ -1,8 +1,8 @@
var test_helper = require('../support/test_helper');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var RedisPool = require('redis-mpool');
test/acceptance/named_static_maps.js
+2 -2
View File
@@ -3,8 +3,8 @@ var RedisPool = require('redis-mpool');
var querystring = require('querystring');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
test/acceptance/server.js
+2 -2
View File
@@ -4,8 +4,8 @@ var assert = require('../support/assert');
var querystring = require('querystring');
var step = require('step');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
test/acceptance/templates.js
+4 -4
View File
@@ -14,13 +14,13 @@ process.env.PGHOST = 'fake';
var helper = require(__dirname + '/../support/test_helper');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
describe('template_api', function() {
serverOptions.channelCache = {};
server.channelCache = {};
var redis_client = redis.createClient(global.environment.redis.port);
@@ -1154,7 +1154,7 @@ describe('template_api', function() {
assert.ok(cc);
assert.ok(cc.match, /ciao/, cc);
// hack simulating restart...
serverOptions.channelCache = {}; // need to clean channel cache
server.channelCache = {}; // need to clean channel cache
var get_request = {
url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1',
method: 'GET',
test/acceptance/x_cache_channel.js
+2 -2
View File
@@ -3,8 +3,8 @@ require('../support/test_helper');
var assert = require('../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
test/support/assert.js
+124 -100
View File
@@ -9,64 +9,6 @@ var exec = require('child_process').exec,
var assert = module.exports = exports = require('assert');
// @param tolerance number of tolerated grid cell differences
assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
var err = null;
var Celldiff = function(x, y, ev, ov) {
this.x = x;
this.y = y;
this.ev = ev;
this.ov = ov;
};
Celldiff.prototype.toString = function() {
return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
};
try {
var obtained_json = JSON.parse(buffer);
// compare grid
var obtained_grid = obtained_json.grid;
var expected_grid = expected_json.grid;
var nrows = obtained_grid.length
if (nrows != expected_grid.length) {
throw new Error( "Obtained grid rows (" + nrows +
") != expected grid rows (" + expected_grid.length + ")" );
}
var celldiff = [];
for (var i=0; i<nrows; ++i) {
var ocols = obtained_grid[i];
var ecols = expected_grid[i];
var ncols = ocols.length;
if ( ncols != ecols.length ) {
throw new Error( "Obtained grid cols (" + ncols +
") != expected grid cols (" + ecols.length +
") on row " + i );
}
for (var j=0; j<ncols; ++j) {
var ocell = ocols[j];
var ecell = ecols[j];
if ( ocell !== ecell ) {
celldiff.push(new Celldiff(i, j, ecell, ocell));
}
}
}
if ( celldiff.length > tolerance ) {
throw new Error( celldiff.length + " cell differences: " + celldiff );
}
assert.deepEqual(obtained_json.keys, expected_json.keys);
} catch (e) { err = e; }
callback(err);
};
/**
* Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
* similarity is not within the tolerance limit it will callback with an error.
@@ -78,15 +20,38 @@ assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
* @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
*/
assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
if (!callback) callback = function(err) { if (err) throw err; };
callback = callback || function(err) { assert.ifError(err); };
var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
testImageFilePath = '/tmp/windshaft-test-image-' + (Math.random() * 1e16); // TODO: make predictable
var err = fs.writeFileSync(testImageFilePath, buffer, 'binary');
if (err) throw err;
testImageFilePath = createImageFromBuffer(buffer, 'test');
imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, function(err) {
fs.unlinkSync(testImageFilePath);
callback(err);
});
};
assert.imageBuffersAreEqual = function(bufferA, bufferB, tolerance, callback) {
var randStr = (Math.random() * 1e16).toString().substring(0, 8);
var imageFilePathA = createImageFromBuffer(bufferA, randStr + '-a'),
imageFilePathB = createImageFromBuffer(bufferB, randStr + '-b');
imageFilesAreEqual(imageFilePathA, imageFilePathB, tolerance, function(err, similarity) {
callback(err, [imageFilePathA, imageFilePathB], similarity);
});
};
function createImageFromBuffer(buffer, nameHint) {
var imageFilePath = path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
var err = fs.writeFileSync(imageFilePath, buffer, 'binary');
assert.ifError(err);
return imageFilePath;
}
function imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, callback) {
var resultFilePath = path.resolve(util.format('/tmp/windshaft-result-%s-diff.png', Date.now()));
var imageMagickCmd = util.format(
'compare -metric fuzz "%s" "%s" /dev/null',
testImageFilePath, referenceImageFilePath
'compare -metric fuzz "%s" "%s" "%s"',
testImageFilePath, referenceImageFilePath, resultFilePath
);
exec(imageMagickCmd, function(err, stdout, stderr) {
@@ -104,29 +69,29 @@ assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolera
tolerancePerMil = (tolerance / 1000);
if (similarity > tolerancePerMil) {
err = new Error(util.format(
'Images %s and %s are not equal (got %d similarity, expected %d)',
testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil)
'Images %s and %s are not equal (got %d similarity, expected %d). Result %s',
testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil, resultFilePath)
);
err.similarity = similarity;
callback(err);
callback(err, similarity);
} else {
fs.unlinkSync(testImageFilePath);
callback(null);
fs.unlinkSync(resultFilePath);
callback(null, similarity);
}
}
});
};
}
/**
* Assert response from `server` with
* the given `req` object and `res` assertions object.
*
* @param {Server} server
* @param {Server|Object} server
* @param {Object} req
* @param {Object|Function} res
* @param {String} msg
* @param {String|Function} msg
*/
// jshint maxcomplexity:12
assert.response = function(server, req, res, msg){
var port = 5555;
function check(){
@@ -161,14 +126,12 @@ assert.response = function(server, req, res, msg){
}
// Callback as third or fourth arg
var callback = typeof res === 'function'
? res
: typeof msg === 'function'
? msg
: function(){};
var callback = typeof res === 'function' ? res : (typeof msg === 'function' ? msg : function(){});
// Default messate to test title
if (typeof msg === 'function') msg = null;
if (typeof msg === 'function') {
msg = null;
}
msg = msg || assert.testTitle;
msg += '. ';
@@ -183,6 +146,7 @@ assert.response = function(server, req, res, msg){
issue();
}
// jshint maxcomplexity:8
function issue(){
// Issue request
@@ -217,27 +181,29 @@ assert.response = function(server, req, res, msg){
}, requestTimeout);
}
if (data) request.write(data);
if (data) {
request.write(data);
}
request.on('response', function(response){
response.body = '';
response.setEncoding(encoding);
response.on('data', function(chunk){ response.body += chunk; });
response.on('end', function(){
if (timer) clearTimeout(timer);
if (timer) {
clearTimeout(timer);
}
check();
// Assert response body
if (res.body !== undefined) {
var eql = res.body instanceof RegExp
? res.body.test(response.body)
: res.body === response.body;
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
assert.ok(
eql,
msg + 'Invalid response body.\n'
+ ' Expected: ' + res.body + '\n'
+ ' Got: ' + response.body
msg + colorize('[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}')
);
}
@@ -246,10 +212,10 @@ assert.response = function(server, req, res, msg){
assert.equal(
response.statusCode,
status,
msg + colorize('Invalid response status code.\n'
+ ' Expected: [green]{' + status + '}\n'
+ ' Got: [red]{' + response.statusCode + '}\n'
+ ' Response body: ' + response.body)
msg + colorize('[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body)
);
}
@@ -260,14 +226,12 @@ assert.response = function(server, req, res, msg){
var name = keys[i],
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
eql = expected instanceof RegExp
? expected.test(actual)
: expected == actual;
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
assert.ok(
eql,
msg + 'Invalid response header [bold]{' + name + '}.\n'
+ ' Expected: [green]{' + expected + '}\n'
+ ' Got: [red]{' + actual + '}'
headerEql,
msg + colorize('Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}')
);
}
}
@@ -278,9 +242,69 @@ assert.response = function(server, req, res, msg){
});
request.end();
}
}
};
// @param tolerance number of tolerated grid cell differences
// jshint maxcomplexity:8
assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
var err = null;
var Celldiff = function(x, y, ev, ov) {
this.x = x;
this.y = y;
this.ev = ev;
this.ov = ov;
};
Celldiff.prototype.toString = function() {
return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
};
try {
var obtained_json = JSON.parse(buffer);
// compare grid
var obtained_grid = obtained_json.grid;
var expected_grid = expected_json.grid;
var nrows = obtained_grid.length;
if (nrows !== expected_grid.length) {
throw new Error( "Obtained grid rows (" + nrows +
") != expected grid rows (" + expected_grid.length + ")" );
}
var celldiff = [];
for (var i=0; i<nrows; ++i) {
var ocols = obtained_grid[i];
var ecols = expected_grid[i];
var ncols = ocols.length;
if ( ncols !== ecols.length ) {
throw new Error( "Obtained grid cols (" + ncols +
") != expected grid cols (" + ecols.length +
") on row " + i );
}
for (var j=0; j<ncols; ++j) {
var ocell = ocols[j];
var ecell = ecols[j];
if ( ocell !== ecell ) {
celldiff.push(new Celldiff(i, j, ecell, ocell));
}
}
}
if ( celldiff.length > tolerance ) {
throw new Error( celldiff.length + " cell differences: " + celldiff );
}
assert.deepEqual(obtained_json.keys, expected_json.keys);
} catch (e) { err = e; }
callback(err);
};
/**
* Colorize the given string using ansi-escape sequences.
* Disabled when --boring is set.
@@ -293,4 +317,4 @@ function colorize(str) {
return str.replace(/\[(\w+)\]\{([^]*?)\}/g, function(_, color, str) {
return '\x1B[' + colors[color] + 'm' + str + '\x1B[0m';
});
}
}
test/support/test_helper.js
+17 -3
View File
@@ -5,7 +5,6 @@
* Desc: Loads test specific variables
*/
var _ = require('underscore');
var assert = require('assert');
var LZMA = require('lzma').LZMA;
@@ -20,11 +19,11 @@ process.env.NODE_ENV = 'test';
function lzma_compress_to_base64(payload, mode, callback) {
lzmaWorker.compress(payload, mode,
function(ints) {
ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
ints = ints.map(function(c) { return String.fromCharCode(c + 128); }).join('');
var base64 = new Buffer(ints, 'binary').toString('base64');
callback(null, base64);
},
function(percent) {
function(/*percent*/) {
//console.log("Compressing: " + percent + "%");
}
);
@@ -55,6 +54,21 @@ function checkSurrogateKey(res, expectedKey) {
assert.equal(res.headers['surrogate-key'], expectedKey);
}
//var _ = require('underscore');
//var redis = require('redis');
// global afterEach to capture tests that leave keys in redis
//afterEach(function(done) {
// var redisClient = redis.createClient(global.environment.redis.port);
// // Check that we start with an empty redis db
// redisClient.keys("*", function(err, keys) {
// if ( err ) {
// return done(err);
// }
// assert.equal(keys.length, 0, "test left objects in redis:\n" + keys.join("\n"));
// redisClient.flushall(done);
// });
//});
module.exports = {
lzma_compress_to_base64: lzma_compress_to_base64,
test/unit/cartodb/req2params.test.js
+9 -7
View File
@@ -5,7 +5,9 @@ var test_helper = require('../../support/test_helper');
suite('req2params', function() {
// configure redis pool instance to use in tests
var opts = require('../../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../../lib/cartodb/server');
var serverOptions = require('../../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
var test_pubuser = global.environment.postgres.user;
@@ -13,11 +15,11 @@ suite('req2params', function() {
test('can be found in server_options', function(){
assert.ok(_.isFunction(opts.req2params));
assert.ok(_.isFunction(server.req2params));
});
test('cleans up request', function(done){
opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
if ( err ) { done(err); return; }
assert.ok(_.isObject(req.query), 'request has query');
assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
@@ -30,7 +32,7 @@ suite('req2params', function() {
});
test('sets dbname from redis metadata', function(done){
opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
if ( err ) { done(err); return; }
//console.dir(req);
assert.ok(_.isObject(req.query), 'request has query');
@@ -44,7 +46,7 @@ suite('req2params', function() {
});
test('sets also dbuser for authenticated requests', function(done){
opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
if ( err ) { done(err); return; }
//console.dir(req);
assert.ok(_.isObject(req.query), 'request has query');
@@ -54,7 +56,7 @@ suite('req2params', function() {
assert.equal(req.params.dbname, test_database);
assert.equal(req.params.dbuser, test_user);
opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
// wrong key resets params to no user
assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
done();
@@ -80,7 +82,7 @@ suite('req2params', function() {
lzma: data
}
};
opts.req2params(req, function(err, req) {
server.req2params(req, function(err, req) {
if ( err ) {
return done(err);
}