cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source 
Conflicts:
	NEWS.md
This commit is contained in:
Luis Bosque
2013-03-05 14:15:07 +01:00
parent fea30dcea4 e038aa7522
commit 4995011f1e
21 changed files with 525 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,3 +1,4 @@
node_modules
config/environments/*.js
.idea
tools/munin/windshaft.conf
Makefile
+11 -1
View File
@@ -7,9 +7,19 @@ clean:
config/environments/test.js: config/environments/test.js.example
./configure
check: config/environments/test.js
check-local: config/environments/test.js
./run_tests.sh ${RUNTESTFLAGS} \
test/unit/cartodb/redis_pool.test.js \
test/unit/cartodb/req2params.test.js \
test/acceptance/cache_validator.js \
test/acceptance/server.js
check-submodules:
for sub in windshaft grainstore mapnik; do \
test -e node_modules/$${sub} && make -C node_modules/$${sub} check; \
done
check-full: check-local check-submodules
check: check-local
NEWS.md
+20 -1
View File
@@ -1,6 +1,25 @@
1.1.8
-----
* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param
1.1.7 (DD//MM//YY)
-----
* Do not let /etc/services confuse FD checker (munin plugin)
* Multilayer support (#72)
* Expose renderer settings in the environment config files
1.1.6 (19//02//13)
-----
* Require windshaft 0.8.5, fixing some stability issues
and providing cache info on request
* Require grainstore 0.10.9, fixing an issue with multi-geom markers
* Enhance run_tests.sh to allow running single tests and skipping preparation
* Fix async throws in getGeometryType, getInfoWindow and getMapMetadata
* Survive connection refusals from redis
* Add maxConnection environment configuration, default to 128
1.1.5 (DD//MM//YY)
-----
* Enhance run_tests.sh to allow running single tests and skipping preparation
* Fix bogus cached return of utf grid for fully contained tiles (#67)
1.1.4 (DD//MM//YY)
app.js
+14 -2
View File
@@ -10,9 +10,9 @@
// sanity check
var ENV = process.argv[2]
if (ENV != 'development' && ENV != 'production'){
if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
console.error("\nnode app.js [environment]");
console.error("environments: [development, production]\n");
console.error("environments: [development, production, staging]\n");
process.exit(1);
}
@@ -33,7 +33,19 @@ var Windshaft = require('windshaft');
var serverOptions = require('./lib/cartodb/server_options');
ws = CartodbWindshaft(serverOptions);
// Maximum number of connections for one process
// 128 is a good number if you have up to 1024 filedescriptors
// 4 is good if you have max 32 filedescriptors
// 1 is good if you have max 16 filedescriptors
ws.maxConnections = global.environment.maxConnections || 128;
ws.listen(global.environment.port, global.environment.host);
ws.on('listening', function() {
console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
});
process.on('SIGUSR1', function() {
ws.dumpCacheStats();
});
cluster.js
+8
View File
@@ -36,6 +36,9 @@ var serverOptions = require('./lib/cartodb/server_options');
var ws = CartodbWindshaft(serverOptions);
// Maximum number of connections for one process (see apps.js)
ws.maxConnections = global.environment.maxConnections || 128;
//.use(cluster.logger('logs'))
//.use(cluster.stats())
//.use(cluster.pidfiles('pids'))
@@ -48,9 +51,14 @@ var cluster = new Cluster({
noWorkers: 1
});
console.log(new Date().toISOString() + " - " + process.pid + " - Windshaft cluster starting");
cluster.listen(function(cb) {
cb(ws);
}, function() {
console.log("Windshaft tileserver started on port " + global.environment.port);
});
process.on('SIGUSR1', function() {
ws.dumpCacheStats();
});
config/environments/development.js.example
+20 -3
View File
@@ -2,11 +2,14 @@ var config = {
environment: 'development'
,port: 8181
,host: '127.0.0.1'
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
,maxConnections:128
// idle socket timeout, in miliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: false
,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'
,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
@@ -26,14 +29,28 @@ var config = {
max_size: 500
}
,mapnik_version: undefined
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64
}
,millstone: {
cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
}
,redis: {
host: '127.0.0.1',
port: 6379,
idleTimeoutMillis: 1,
reapIntervalMillis: 1
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 1, // idle time before dropping connection
reapIntervalMillis: 1 // time between cleanups
}
,sqlapi: {
protocol: 'http',
config/environments/production.js.example
+21 -2
View File
@@ -2,11 +2,14 @@ var config = {
environment: 'production'
,port: 8181
,host: '127.0.0.1'
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
,maxConnections:128
// idle socket timeout, in miliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'
,postgres_auth_user: 'cartodb_user_<%= user_id %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
@@ -20,12 +23,28 @@ var config = {
max_size: 500
}
,mapnik_version: undefined
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64
}
,millstone: {
cache_basedir: '/home/ubuntu/tile_assets/'
}
,redis: {
host: '127.0.0.1',
port: 6379
port: 6379,
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 30000, // idle time before dropping connection
reapIntervalMillis: 1000 // time between cleanups
}
,sqlapi: {
protocol: 'https',
config/environments/staging.js.example
+21 -2
View File
@@ -2,11 +2,14 @@ var config = {
environment: 'production'
,port: 8181
,host: '127.0.0.1'
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
,maxConnections:128
// idle socket timeout, in miliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'
,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
@@ -20,12 +23,28 @@ var config = {
max_size: 500
}
,mapnik_version: undefined
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64
}
,millstone: {
cache_basedir: '/home/ubuntu/tile_assets/'
}
,redis: {
host: '127.0.0.1',
port: 6379
port: 6379,
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 30000, // idle time before dropping connection
reapIntervalMillis: 1000 // time between cleanups
}
,sqlapi: {
protocol: 'https',
config/environments/test.js.example
+19 -2
View File
@@ -2,6 +2,9 @@ var config = {
environment: 'test'
,port: 8888
,host: '127.0.0.1'
// Maximum number of connections for one process
// 128 is a good value with a limit of 1024 open file descriptors
,maxConnections:128
// idle socket timeout, in miliseconds
,socket_timeout: 600000
,enable_cors: true
@@ -20,14 +23,28 @@ var config = {
max_size: 500
}
,mapnik_version: '2.0.2'
,renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
metatile: 4,
bufferSize: 64
}
,millstone: {
cache_basedir: '/tmp/cdb-tiler-test/millstone'
}
,redis: {
host: '127.0.0.1',
port: 6333,
idleTimeoutMillis: 1,
reapIntervalMillis: 1
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 1, // idle time before dropping connection
reapIntervalMillis: 1 // time between cleanups
}
,sqlapi: {
protocol: 'http',
lib/cartodb/carto_data.js
+3 -7
View File
@@ -106,7 +106,7 @@ module.exports = function() {
if (check_result === 1) {
// authorized by key, login as db owner
that.getId(req, function(err, user_id) {
if (err) throw new Error(err);
if (err) throw err;
var dbuser = _.template(global.settings.postgres_auth_user, {user_id: user_id});
_.extend(req, {dbuser:dbuser});
callback(err, true);
@@ -126,7 +126,6 @@ module.exports = function() {
that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
},
function(err, data){
//if (err) throw err;
callback(err, data);
}
);
@@ -152,7 +151,6 @@ module.exports = function() {
that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
},
function(err, data){
if (err) throw err;
callback(err, data);
}
);
@@ -172,7 +170,6 @@ module.exports = function() {
that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
},
function(err, data){
if (err) throw err;
callback(err, data);
}
);
@@ -193,7 +190,6 @@ module.exports = function() {
that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
},
function(err, data){
if (err) throw err;
callback(err, data);
}
);
@@ -227,13 +223,13 @@ module.exports = function() {
redis_pool.acquire(db, this);
},
function executeQuery(err, data) {
if ( err ) throw err;
redisClient = data;
redisArgs.push(this);
redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
},
function releaseRedisClient(err, data) {
if (err) throw err;
redis_pool.release(db, redisClient);
if ( ! _.isUndefined(redisClient) ) redis_pool.release(db, redisClient);
callback(err, data);
}
);
lib/cartodb/redis_pool.js
+4 -1
View File
@@ -58,8 +58,11 @@ var RedisPool = function(opts){
client.send_anyway = true;
client.select(database);
client.send_anyway = false;
callback(null, client);
});
return callback(null, client);
client.on('error', function (err) {
callback(err, null);
});
},
destroy: function(client) {
return client.quit();
lib/cartodb/server_options.js
+15
View File
@@ -6,13 +6,28 @@ var _ = require('underscore')
;
module.exports = function(){
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000,
metatile: 4,
bufferSize: 64
});
var me = {
base_url: '/tiles/:table',
base_url_notable: '/tiles',
grainstore: {
datasource: global.environment.postgres,
cachedir: global.environment.millstone.cache_basedir,
mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik
},
mapnik: {
metatile: rendererConfig.metatile,
bufferSize: rendererConfig.bufferSize
},
renderCache: {
ttl: rendererConfig.cache_ttl
},
redis: global.environment.redis,
enable_cors: global.environment.enable_cors,
varnish_host: global.environment.varnish.host,
npm-shrinkwrap.json
Generated
+37 -34
View File
@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.1.5",
"version": "1.1.7",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb02",
@@ -13,7 +13,7 @@
"version": "1.9.2",
"dependencies": {
"formidable": {
"version": "1.0.11"
"version": "1.0.12"
}
}
},
@@ -32,22 +32,22 @@
"version": "0.8.3"
},
"npm": {
"version": "1.1.69",
"version": "1.2.12",
"dependencies": {
"semver": {
"version": "1.1.1"
"version": "1.1.3"
},
"ini": {
"version": "1.0.5"
"version": "1.1.0"
},
"slide": {
"version": "1.1.3"
},
"abbrev": {
"version": "1.0.3"
"version": "1.0.4"
},
"graceful-fs": {
"version": "1.1.14"
"version": "1.2.0"
},
"minimatch": {
"version": "0.2.9",
@@ -58,10 +58,10 @@
}
},
"nopt": {
"version": "2.0.0"
"version": "2.1.1"
},
"rimraf": {
"version": "2.0.2"
"version": "2.1.4"
},
"request": {
"version": "2.9.203",
@@ -71,10 +71,10 @@
"version": "1.0.5"
},
"tar": {
"version": "0.1.14"
"version": "0.1.16"
},
"fstream": {
"version": "0.1.19"
"version": "0.1.22"
},
"block-stream": {
"version": "0.0.6"
@@ -98,13 +98,13 @@
"version": "2.0.4"
},
"node-gyp": {
"version": "0.7.3"
"version": "0.8.4"
},
"fstream-npm": {
"version": "0.1.3",
"dependencies": {
"fstream-ignore": {
"version": "0.0.5"
"version": "0.0.6"
}
}
},
@@ -124,7 +124,7 @@
"version": "0.1.2"
},
"npm-registry-client": {
"version": "0.2.10",
"version": "0.2.17",
"dependencies": {
"couch-login": {
"version": "0.1.15"
@@ -132,13 +132,13 @@
}
},
"read-package-json": {
"version": "0.1.11"
"version": "0.2.0"
},
"read-installed": {
"version": "0.0.3"
"version": "0.1.1"
},
"glob": {
"version": "3.1.14"
"version": "3.1.20"
},
"init-package-json": {
"version": "0.0.6",
@@ -152,7 +152,7 @@
"version": "0.0.3"
},
"lockfile": {
"version": "0.2.1"
"version": "0.3.0"
},
"retry": {
"version": "0.6.0"
@@ -161,10 +161,10 @@
"version": "1.1.1"
},
"npmconf": {
"version": "0.0.16",
"version": "0.0.23",
"dependencies": {
"config-chain": {
"version": "1.1.3",
"version": "1.1.5",
"dependencies": {
"proto-list": {
"version": "1.2.2"
@@ -187,28 +187,31 @@
"version": "1.3.3"
},
"grainstore": {
"version": "0.10.8",
"version": "0.11.1",
"dependencies": {
"carto": {
"version": "0.9.3-cdb2",
"from": "git://github.com/CartoDB/carto.git#cdb-0.9",
"dependencies": {
"mapnik-reference": {
"version": "5.0.0-cdb1",
"from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0"
},
"xml2js": {
"version": "0.1.14",
"dependencies": {
"sax": {
"version": "0.4.2"
"version": "0.5.2"
}
}
}
}
},
"mapnik-reference": {
"version": "5.0.0-cdb1",
"from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0"
"version": "5.0.4"
},
"millstone": {
"version": "0.5.13",
"version": "0.5.15",
"dependencies": {
"request": {
"version": "2.11.4",
@@ -235,26 +238,26 @@
}
},
"srs": {
"version": "0.2.18"
"version": "0.2.20"
},
"zipfile": {
"version": "0.3.3"
"version": "0.3.4"
},
"sqlite3": {
"version": "2.1.5"
},
"mime": {
"version": "1.2.7"
"version": "1.2.9"
},
"mkdirp": {
"version": "0.3.4"
"version": "0.3.5"
}
}
}
}
},
"windshaft": {
"version": "0.8.2",
"version": "0.9.1",
"dependencies": {
"express": {
"version": "2.5.11",
@@ -263,7 +266,7 @@
"version": "1.9.2",
"dependencies": {
"formidable": {
"version": "1.0.11"
"version": "1.0.12"
}
}
},
@@ -324,10 +327,10 @@
"version": "2.9.202"
},
"mapnik": {
"version": "0.7.18"
"version": "0.7.22"
},
"semver": {
"version": "1.1.1"
"version": "1.1.3"
},
"mocha": {
"version": "1.2.1",
@@ -350,7 +353,7 @@
"version": "1.0.2"
},
"debug": {
"version": "0.7.0"
"version": "0.7.2"
}
}
}
package.json
+4 -4
View File
@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.1.5",
"version": "1.1.8",
"description": "A map tile server for CartoDB",
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
"licenses": [{
@@ -21,10 +21,10 @@
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"grainstore" : "~0.10.8",
"windshaft" : "~0.8.2",
"grainstore" : "~0.11.1",
"windshaft" : "~0.9.1",
"step": "0.0.x",
"generic-pool": "1.0.x",
"generic-pool": "~1.0.12",
"redis": "0.7.2",
"hiredis": "~0.1.14",
"request": "2.9.202",
test/acceptance/multilayer.js
+148
View File
@@ -0,0 +1,148 @@
var assert = require('../support/assert');
var tests = module.exports = {};
var _ = require('underscore');
var redis = require('redis');
var querystring = require('querystring');
var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
require(__dirname + '/../support/test_helper');
var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('multilayer', function() {
var redis_client = redis.createClient(global.environment.redis.port);
suiteSetup(function(){
});
test("layergroup with 2 layers, each with its style", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.0.1'
} },
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2',
cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
cartocss_version: '2.0.2'
} }
]
};
var expected_token = "d442ca6d3ece793b9c16c02a1d1ea5f2";
Step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/tiles/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
var expectedBody = { layergroupid: expected_token };
// TODO: check last modified
//expectedBody.layercount = 2;
if ( expected_token ) assert.deepEqual(parsedBody, expectedBody);
else expected_token = parsedBody.layergroupid;
next(null, res);
});
},
function do_get_tile(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
headers: {host: 'localhost' },
encoding: 'binary'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
function(err, similarity) {
next(err);
});
});
},
function do_get_grid_layer0(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer0/0/0/0.grid.json?interactivity=cartodb_id',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json', 2,
function(err, similarity) {
next(err);
});
});
},
function do_get_grid_layer1(err)
{
if ( err ) throw err;
var next = this;
assert.response(server, {
url: '/tiles/layergroup/' + expected_token
+ '/layer1/0/0/0.grid.json?interactivity=cartodb_id',
headers: {host: 'localhost' },
method: 'GET'
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json', 2,
function(err, similarity) {
next(err);
});
});
},
function finish(err) {
var errors = [];
if ( err ) {
errors.push(err.message);
console.log("Error: " + err);
}
redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
});
});
}
);
});
suiteTeardown(function(done) {
// This test will add map_style records, like
// 'map_style|null|publicuser|my_table',
redis_client.keys("map_style|*", function(err, matches) {
_.each(matches, function(k) { redis_client.del(k); });
done();
});
});
});
test/acceptance/server.js
+8 -8
View File
@@ -110,7 +110,7 @@ suite('server', function() {
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.equal(res.statusCode, 400, res.body);
assert.deepEqual(JSON.parse(res.body),
{error: 'Sorry, you are unauthorized (permission denied)'});
done();
@@ -127,7 +127,7 @@ suite('server', function() {
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"});
done();
@@ -175,7 +175,7 @@ suite('server', function() {
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: '#my_table3{backgxxxxxround-color:#fff;}'})
},{
status: 500, // FIXME: should be 400 !
status: 400,
body: /Unrecognized rule: backgxxxxxround-color/
}, function() { done(); });
});
@@ -187,7 +187,7 @@ suite('server', function() {
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: '#my_table3{'})
},{
status: 500, // FIXME: should be 400 !
status: 400,
body: /Missing closing/
}, function() { done(); });
});
@@ -199,7 +199,7 @@ suite('server', function() {
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: '#my_table4{backgxxxxxround-color:#fff;foo:bar}'})
},{
status: 500, // FIXME: should be 400 !
status: 400,
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.length, 2);
@@ -252,7 +252,7 @@ suite('server', function() {
data: querystring.stringify({style: 'Map { background-color:#aaa; }'})
},{}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
assert.response(server, {
@@ -464,7 +464,7 @@ suite('server', function() {
method: 'GET'
},{}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
done();
});
});
@@ -479,7 +479,7 @@ suite('server', function() {
},{
}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"});
done();
test/support/assert.js
+58
View File
@@ -8,6 +8,64 @@ var exec = require('child_process').exec;
var assert = module.exports = exports = require('assert');
// @param tolerance number of tolerated grid cell differences
assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
var err = null;
var Celldiff = function(x, y, ev, ov) {
this.x = x;
this.y = y;
this.ev = ev;
this.ov = ov;
};
Celldiff.prototype.toString = function() {
return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
};
try {
var obtained_json = JSON.parse(buffer);
// compare grid
var obtained_grid = obtained_json.grid;
var expected_grid = expected_json.grid;
var nrows = obtained_grid.length
if (nrows != expected_grid.length) {
throw new Error( "Obtained grid rows (" + nrows +
") != expected grid rows (" + expected_grid.length + ")" );
}
var celldiff = [];
for (var i=0; i<nrows; ++i) {
var ocols = obtained_grid[i];
var ecols = expected_grid[i];
var ncols = ocols.length;
if ( ncols != ecols.length ) {
throw new Error( "Obtained grid cols (" + ncols +
") != expected grid cols (" + ecols.length +
") on row " + i );
}
for (var j=0; j<ncols; ++j) {
var ocell = ocols[j];
var ecell = ecols[j];
if ( ocell !== ecell ) {
celldiff.push(new Celldiff(i, j, ecell, ocell));
}
}
}
if ( celldiff.length > tolerance ) {
throw new Error( celldiff.length + " cell differences: " + celldiff );
}
assert.deepEqual(obtained_json.keys, expected_json.keys);
} catch (e) { err = e; }
callback(err);
};
//
// @param tol tolerated color distance as a percent over max channel value
// by default this is zero. For meaningful values, see
test/support/prepare_db.sh
+2 -1
View File
@@ -33,4 +33,5 @@ echo "SADD rails:users:localhost:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0
echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0
echo "Finished preparing data. Run tests with expresso."
echo "Finished preparing data. Ready to run tests"
tools/munin/Makefile
+19
View File
@@ -0,0 +1,19 @@
MUNIN_PLUGINS_DIR=/etc/munin/plugins
MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
PWD=$(shell pwd)
all: windshaft.conf
windshaft.conf: windshaft.conf.in
sed 's#@PWD@#$(PWD)#' < $< > $@
install-munin-plugin-conf: windshaft.conf
install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
install-munin-plugin: windshaft
install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
install: install-munin-plugin install-munin-plugin-conf
clean:
rm -f windshaft.conf
tools/munin/windshaft
Executable
+87
View File
@@ -0,0 +1,87 @@
#!/bin/sh
envnik=$(basename "${TILER_ENVIRONMENT}" .js)
if test "$1" = "config"; then
echo "graph_title fd usage (${envnik})"
cat <<"EOM"
graph_vlabel number of file descriptors
graph_category windshaft
graph_scale no
procs.label Number of tiler processes
pgsql.label PostgreSQL connections (max)
redis.label Redis connections (max)
http.label Incoming http requests (max)
caches.label Number of renderer caches (max)
nfd.label Total file descriptors (max)
EOM
exit 0
fi
if test x"$1" != x; then
TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
fi
if test -z "$TILER_ENVIRONMENT"; then
echo "Usage: $0 [<environment>]" >&2
echo " or: [TILER_ENVIRONMENT=<environment>] $0" >&2
exit 1
fi
http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
nworkers=$(echo "${pids}" | wc -l)
pids=$(echo "${pids}" | paste -sd ' ')
if test -z "${pids}"; then
echo "No processes found listening on tcp port '${http_port}'" >&2
exit 1
fi
tmpreport="/tmp/checkfd.$$.txt"
lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
maxdb=0
maxredis=0
maxhttp=0
maxtot=0
maxcache=0
for pid in ${pids}; do
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
if test $cnt -gt $maxdb; then maxdb=$cnt; fi
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
if test $cnt -gt $maxredis; then maxredis=$cnt; fi
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
if test $cnt -gt $maxtot; then maxtot=$cnt; fi
log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
if test -e "${log}"; then
kill -USR1 "${pid}"
cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
if test $cnt -gt $maxcache; then maxcache=$cnt; fi
else
# report the error...
maxcache=-1
fi
done
echo "procs.value ${nworkers}"
echo "pgsql.value ${maxdb}"
echo "redis.value ${maxredis}"
echo "http.value ${maxhttp}"
echo "caches.value ${maxcache}"
echo "nfd.value ${maxtot}"
rm -f "${tmpreport}"
tools/munin/windshaft.conf.in
+5
View File
@@ -0,0 +1,5 @@
# Configuration file for munin plugin
[windshaft]
user root
env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js