cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Standalone server initial implementation

Browse Source 
- no dependency over Windshaft.Server
This commit is contained in:
Raul Ochoa
2015-07-04 20:41:22 +02:00
parent ed90cadd75
commit 2d3f2667ca
24 changed files with 1598 additions and 991 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/acceptance/cache/surrogate_keys_invalidation.js
Vendored
+14 -13
View File
@@ -6,23 +6,25 @@ var step = require('step');
var FastlyPurge = require('fastly-purge');
var NamedMapsCacheEntry = require(__dirname + '/../../../lib/cartodb/cache/model/named_maps_entry');
var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/cartodb_windshaft');
var CartodbWindshaft = require(__dirname + '/../../../lib/cartodb/server');
describe('templates surrogate keys', function() {
var redisClient = redis.createClient(global.environment.redis.port);
// Enable Varnish purge for tests
var varnishHost = global.environment.varnish.host;
global.environment.varnish.host = '127.0.0.1';
var varnishPurgeEnabled = global.environment.varnish.purge_enabled;
global.environment.varnish.purge_enabled = true;
var serverOptions = require('../../../lib/cartodb/server_options');
var fastlyConfig = global.environment.fastly;
// Enable Varnish purge for tests
var varnishHost = serverOptions.varnish_host;
serverOptions.varnish_host = '127.0.0.1';
var varnishPurgeEnabled = serverOptions.varnish_purge_enabled;
serverOptions.varnish_purge_enabled = true;
var fastlyConfig = serverOptions.fastly;
var FAKE_FASTLY_API_KEY = 'fastly-api-key';
var FAKE_FASTLY_SERVICE_ID = 'fake-service-id';
global.environment.fastly = {
serverOptions.fastly = {
enabled: true,
// the fastly api key
apiKey: FAKE_FASTLY_API_KEY,
@@ -30,7 +32,6 @@ describe('templates surrogate keys', function() {
serviceId: FAKE_FASTLY_SERVICE_ID
};
var serverOptions = require('../../../lib/cartodb/server_options')();
var server = new CartodbWindshaft(serverOptions);
var templateOwner = 'localhost',
@@ -58,7 +59,7 @@ describe('templates surrogate keys', function() {
expectedBody = { template_id: expectedTemplateId };
var varnishHttpUrl = [
'http://', global.environment.varnish.host, ':', global.environment.varnish.http_port
'http://', serverOptions.varnish_host, ':', serverOptions.varnish_http_port
].join('');
var cacheEntryKey = new NamedMapsCacheEntry(templateOwner, templateName).key();
@@ -70,10 +71,10 @@ describe('templates surrogate keys', function() {
after(function(done) {
serverOptions.varnish_purge_enabled = false;
global.environment.varnish.host = varnishHost;
global.environment.varnish.purge_enabled = varnishPurgeEnabled;
serverOptions.varnish_host = varnishHost;
serverOptions.varnish_purge_enabled = varnishPurgeEnabled;
global.environment.fastly = fastlyConfig;
serverOptions.fastly = fastlyConfig;
nock.restore();
done();
test/acceptance/health_check.js
+2 -2
View File
@@ -1,8 +1,8 @@
require(__dirname + '/../support/test_helper');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var metadataBackend = {};
test/acceptance/limits.js
+2 -2
View File
@@ -4,7 +4,7 @@ var assert = require('../support/assert');
var _ = require('underscore');
var redis = require('redis');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
describe.skip('render limits', function() {
@@ -22,7 +22,7 @@ describe.skip('render limits', function() {
var server;
beforeEach(function() {
server = new CartodbWindshaft(serverOptions());
server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
});
test/acceptance/multilayer.js
+4 -4
View File
@@ -12,9 +12,9 @@ var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixture
var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20;
var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25;
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions());
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
@@ -859,7 +859,7 @@ suite(suiteName, function() {
function do_restart_server(err/*, res*/) {
if ( err ) throw err;
// hack simulating restart...
server = new CartodbWindshaft(serverOptions());
server = new CartodbWindshaft(serverOptions);
return null;
},
function do_get1(err)
test/acceptance/multilayer_server.js
+3 -3
View File
@@ -7,8 +7,8 @@ var _ = require('underscore');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
@@ -346,7 +346,7 @@ describe('tests from old api translated to multilayer', function() {
};
// reset internal cacheChannel cache
serverOptions.channelCache = {};
server.channelCache = {};
assert.response(server,
{
test/acceptance/named_layers.js
+2 -2
View File
@@ -1,8 +1,8 @@
var test_helper = require('../support/test_helper');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var RedisPool = require('redis-mpool');
test/acceptance/named_static_maps.js
+2 -2
View File
@@ -3,8 +3,8 @@ var RedisPool = require('redis-mpool');
var querystring = require('querystring');
var assert = require('../support/assert');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
test/acceptance/server.js
+2 -2
View File
@@ -4,8 +4,8 @@ var assert = require('../support/assert');
var querystring = require('querystring');
var step = require('step');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
test/acceptance/templates.js
+4 -4
View File
@@ -14,13 +14,13 @@ process.env.PGHOST = 'fake';
var helper = require(__dirname + '/../support/test_helper');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
describe('template_api', function() {
serverOptions.channelCache = {};
server.channelCache = {};
var redis_client = redis.createClient(global.environment.redis.port);
@@ -1154,7 +1154,7 @@ describe('template_api', function() {
assert.ok(cc);
assert.ok(cc.match, /ciao/, cc);
// hack simulating restart...
serverOptions.channelCache = {}; // need to clean channel cache
server.channelCache = {}; // need to clean channel cache
var get_request = {
url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1',
method: 'GET',
test/acceptance/x_cache_channel.js
+2 -2
View File
@@ -3,8 +3,8 @@ require('../support/test_helper');
var assert = require('../support/assert');
var qs = require('querystring');
var CartodbWindshaft = require('../../lib/cartodb/cartodb_windshaft');
var serverOptions = require('../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
test/support/assert.js
+124 -100
View File
@@ -9,64 +9,6 @@ var exec = require('child_process').exec,
var assert = module.exports = exports = require('assert');
// @param tolerance number of tolerated grid cell differences
assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
var err = null;
var Celldiff = function(x, y, ev, ov) {
this.x = x;
this.y = y;
this.ev = ev;
this.ov = ov;
};
Celldiff.prototype.toString = function() {
return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
};
try {
var obtained_json = JSON.parse(buffer);
// compare grid
var obtained_grid = obtained_json.grid;
var expected_grid = expected_json.grid;
var nrows = obtained_grid.length
if (nrows != expected_grid.length) {
throw new Error( "Obtained grid rows (" + nrows +
") != expected grid rows (" + expected_grid.length + ")" );
}
var celldiff = [];
for (var i=0; i<nrows; ++i) {
var ocols = obtained_grid[i];
var ecols = expected_grid[i];
var ncols = ocols.length;
if ( ncols != ecols.length ) {
throw new Error( "Obtained grid cols (" + ncols +
") != expected grid cols (" + ecols.length +
") on row " + i );
}
for (var j=0; j<ncols; ++j) {
var ocell = ocols[j];
var ecell = ecols[j];
if ( ocell !== ecell ) {
celldiff.push(new Celldiff(i, j, ecell, ocell));
}
}
}
if ( celldiff.length > tolerance ) {
throw new Error( celldiff.length + " cell differences: " + celldiff );
}
assert.deepEqual(obtained_json.keys, expected_json.keys);
} catch (e) { err = e; }
callback(err);
};
/**
* Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
* similarity is not within the tolerance limit it will callback with an error.
@@ -78,15 +20,38 @@ assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
* @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
*/
assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
if (!callback) callback = function(err) { if (err) throw err; };
callback = callback || function(err) { assert.ifError(err); };
var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
testImageFilePath = '/tmp/windshaft-test-image-' + (Math.random() * 1e16); // TODO: make predictable
var err = fs.writeFileSync(testImageFilePath, buffer, 'binary');
if (err) throw err;
testImageFilePath = createImageFromBuffer(buffer, 'test');
imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, function(err) {
fs.unlinkSync(testImageFilePath);
callback(err);
});
};
assert.imageBuffersAreEqual = function(bufferA, bufferB, tolerance, callback) {
var randStr = (Math.random() * 1e16).toString().substring(0, 8);
var imageFilePathA = createImageFromBuffer(bufferA, randStr + '-a'),
imageFilePathB = createImageFromBuffer(bufferB, randStr + '-b');
imageFilesAreEqual(imageFilePathA, imageFilePathB, tolerance, function(err, similarity) {
callback(err, [imageFilePathA, imageFilePathB], similarity);
});
};
function createImageFromBuffer(buffer, nameHint) {
var imageFilePath = path.resolve('test/results/png/image-' + nameHint + '-' + Date.now() + '.png');
var err = fs.writeFileSync(imageFilePath, buffer, 'binary');
assert.ifError(err);
return imageFilePath;
}
function imageFilesAreEqual(testImageFilePath, referenceImageFilePath, tolerance, callback) {
var resultFilePath = path.resolve(util.format('/tmp/windshaft-result-%s-diff.png', Date.now()));
var imageMagickCmd = util.format(
'compare -metric fuzz "%s" "%s" /dev/null',
testImageFilePath, referenceImageFilePath
'compare -metric fuzz "%s" "%s" "%s"',
testImageFilePath, referenceImageFilePath, resultFilePath
);
exec(imageMagickCmd, function(err, stdout, stderr) {
@@ -104,29 +69,29 @@ assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolera
tolerancePerMil = (tolerance / 1000);
if (similarity > tolerancePerMil) {
err = new Error(util.format(
'Images %s and %s are not equal (got %d similarity, expected %d)',
testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil)
'Images %s and %s are not equal (got %d similarity, expected %d). Result %s',
testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil, resultFilePath)
);
err.similarity = similarity;
callback(err);
callback(err, similarity);
} else {
fs.unlinkSync(testImageFilePath);
callback(null);
fs.unlinkSync(resultFilePath);
callback(null, similarity);
}
}
});
};
}
/**
* Assert response from `server` with
* the given `req` object and `res` assertions object.
*
* @param {Server} server
* @param {Server|Object} server
* @param {Object} req
* @param {Object|Function} res
* @param {String} msg
* @param {String|Function} msg
*/
// jshint maxcomplexity:12
assert.response = function(server, req, res, msg){
var port = 5555;
function check(){
@@ -161,14 +126,12 @@ assert.response = function(server, req, res, msg){
}
// Callback as third or fourth arg
var callback = typeof res === 'function'
? res
: typeof msg === 'function'
? msg
: function(){};
var callback = typeof res === 'function' ? res : (typeof msg === 'function' ? msg : function(){});
// Default messate to test title
if (typeof msg === 'function') msg = null;
if (typeof msg === 'function') {
msg = null;
}
msg = msg || assert.testTitle;
msg += '. ';
@@ -183,6 +146,7 @@ assert.response = function(server, req, res, msg){
issue();
}
// jshint maxcomplexity:8
function issue(){
// Issue request
@@ -217,27 +181,29 @@ assert.response = function(server, req, res, msg){
}, requestTimeout);
}
if (data) request.write(data);
if (data) {
request.write(data);
}
request.on('response', function(response){
response.body = '';
response.setEncoding(encoding);
response.on('data', function(chunk){ response.body += chunk; });
response.on('end', function(){
if (timer) clearTimeout(timer);
if (timer) {
clearTimeout(timer);
}
check();
// Assert response body
if (res.body !== undefined) {
var eql = res.body instanceof RegExp
? res.body.test(response.body)
: res.body === response.body;
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
assert.ok(
eql,
msg + 'Invalid response body.\n'
+ ' Expected: ' + res.body + '\n'
+ ' Got: ' + response.body
msg + colorize('[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}')
);
}
@@ -246,10 +212,10 @@ assert.response = function(server, req, res, msg){
assert.equal(
response.statusCode,
status,
msg + colorize('Invalid response status code.\n'
+ ' Expected: [green]{' + status + '}\n'
+ ' Got: [red]{' + response.statusCode + '}\n'
+ ' Response body: ' + response.body)
msg + colorize('[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body)
);
}
@@ -260,14 +226,12 @@ assert.response = function(server, req, res, msg){
var name = keys[i],
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
eql = expected instanceof RegExp
? expected.test(actual)
: expected == actual;
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
assert.ok(
eql,
msg + 'Invalid response header [bold]{' + name + '}.\n'
+ ' Expected: [green]{' + expected + '}\n'
+ ' Got: [red]{' + actual + '}'
headerEql,
msg + colorize('Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}')
);
}
}
@@ -278,9 +242,69 @@ assert.response = function(server, req, res, msg){
});
request.end();
}
}
};
// @param tolerance number of tolerated grid cell differences
// jshint maxcomplexity:8
assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
var err = null;
var Celldiff = function(x, y, ev, ov) {
this.x = x;
this.y = y;
this.ev = ev;
this.ov = ov;
};
Celldiff.prototype.toString = function() {
return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
};
try {
var obtained_json = JSON.parse(buffer);
// compare grid
var obtained_grid = obtained_json.grid;
var expected_grid = expected_json.grid;
var nrows = obtained_grid.length;
if (nrows !== expected_grid.length) {
throw new Error( "Obtained grid rows (" + nrows +
") != expected grid rows (" + expected_grid.length + ")" );
}
var celldiff = [];
for (var i=0; i<nrows; ++i) {
var ocols = obtained_grid[i];
var ecols = expected_grid[i];
var ncols = ocols.length;
if ( ncols !== ecols.length ) {
throw new Error( "Obtained grid cols (" + ncols +
") != expected grid cols (" + ecols.length +
") on row " + i );
}
for (var j=0; j<ncols; ++j) {
var ocell = ocols[j];
var ecell = ecols[j];
if ( ocell !== ecell ) {
celldiff.push(new Celldiff(i, j, ecell, ocell));
}
}
}
if ( celldiff.length > tolerance ) {
throw new Error( celldiff.length + " cell differences: " + celldiff );
}
assert.deepEqual(obtained_json.keys, expected_json.keys);
} catch (e) { err = e; }
callback(err);
};
/**
* Colorize the given string using ansi-escape sequences.
* Disabled when --boring is set.
@@ -293,4 +317,4 @@ function colorize(str) {
return str.replace(/\[(\w+)\]\{([^]*?)\}/g, function(_, color, str) {
return '\x1B[' + colors[color] + 'm' + str + '\x1B[0m';
});
}
}
test/support/test_helper.js
+17 -3
View File
@@ -5,7 +5,6 @@
* Desc: Loads test specific variables
*/
var _ = require('underscore');
var assert = require('assert');
var LZMA = require('lzma').LZMA;
@@ -20,11 +19,11 @@ process.env.NODE_ENV = 'test';
function lzma_compress_to_base64(payload, mode, callback) {
lzmaWorker.compress(payload, mode,
function(ints) {
ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
ints = ints.map(function(c) { return String.fromCharCode(c + 128); }).join('');
var base64 = new Buffer(ints, 'binary').toString('base64');
callback(null, base64);
},
function(percent) {
function(/*percent*/) {
//console.log("Compressing: " + percent + "%");
}
);
@@ -55,6 +54,21 @@ function checkSurrogateKey(res, expectedKey) {
assert.equal(res.headers['surrogate-key'], expectedKey);
}
//var _ = require('underscore');
//var redis = require('redis');
// global afterEach to capture tests that leave keys in redis
//afterEach(function(done) {
// var redisClient = redis.createClient(global.environment.redis.port);
// // Check that we start with an empty redis db
// redisClient.keys("*", function(err, keys) {
// if ( err ) {
// return done(err);
// }
// assert.equal(keys.length, 0, "test left objects in redis:\n" + keys.join("\n"));
// redisClient.flushall(done);
// });
//});
module.exports = {
lzma_compress_to_base64: lzma_compress_to_base64,
test/unit/cartodb/req2params.test.js
+9 -7
View File
@@ -5,7 +5,9 @@ var test_helper = require('../../support/test_helper');
suite('req2params', function() {
// configure redis pool instance to use in tests
var opts = require('../../../lib/cartodb/server_options')();
var CartodbWindshaft = require('../../../lib/cartodb/server');
var serverOptions = require('../../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
var test_pubuser = global.environment.postgres.user;
@@ -13,11 +15,11 @@ suite('req2params', function() {
test('can be found in server_options', function(){
assert.ok(_.isFunction(opts.req2params));
assert.ok(_.isFunction(server.req2params));
});
test('cleans up request', function(done){
opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
if ( err ) { done(err); return; }
assert.ok(_.isObject(req.query), 'request has query');
assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
@@ -30,7 +32,7 @@ suite('req2params', function() {
});
test('sets dbname from redis metadata', function(done){
opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
if ( err ) { done(err); return; }
//console.dir(req);
assert.ok(_.isObject(req.query), 'request has query');
@@ -44,7 +46,7 @@ suite('req2params', function() {
});
test('sets also dbuser for authenticated requests', function(done){
opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
if ( err ) { done(err); return; }
//console.dir(req);
assert.ok(_.isObject(req.query), 'request has query');
@@ -54,7 +56,7 @@ suite('req2params', function() {
assert.equal(req.params.dbname, test_database);
assert.equal(req.params.dbuser, test_user);
opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
server.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
// wrong key resets params to no user
assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
done();
@@ -80,7 +82,7 @@ suite('req2params', function() {
lzma: data
}
};
opts.req2params(req, function(err, req) {
server.req2params(req, function(err, req) {
if ( err ) {
return done(err);
}