diff --git a/.gitignore b/.gitignore index fc482c42..96b65b68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules config/environments/*.js .idea +tools/munin/windshaft.conf diff --git a/Makefile b/Makefile index 424a3c91..c6eee208 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,19 @@ clean: config/environments/test.js: config/environments/test.js.example ./configure -check: config/environments/test.js +check-local: config/environments/test.js ./run_tests.sh ${RUNTESTFLAGS} \ test/unit/cartodb/redis_pool.test.js \ test/unit/cartodb/req2params.test.js \ test/acceptance/cache_validator.js \ test/acceptance/server.js + +check-submodules: + for sub in windshaft grainstore mapnik; do \ + test -e node_modules/$${sub} && make -C node_modules/$${sub} check; \ + done + +check-full: check-local check-submodules + +check: check-local + diff --git a/NEWS.md b/NEWS.md index 3cef2d0a..57db4db3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,25 @@ +1.1.8 +----- +* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param + +1.1.7 (DD//MM//YY) +----- +* Do not let /etc/services confuse FD checker (munin plugin) +* Multilayer support (#72) +* Expose renderer settings in the environment config files + +1.1.6 (19//02//13) +----- +* Require windshaft 0.8.5, fixing some stability issues + and providing cache info on request +* Require grainstore 0.10.9, fixing an issue with multi-geom markers +* Enhance run_tests.sh to allow running single tests and skipping preparation +* Fix async throws in getGeometryType, getInfoWindow and getMapMetadata +* Survive connection refusals from redis +* Add maxConnection environment configuration, default to 128 + 1.1.5 (DD//MM//YY) ----- -* Enhance run_tests.sh to allow running single tests and skipping preparation * Fix bogus cached return of utf grid for fully contained tiles (#67) 1.1.4 (DD//MM//YY) diff --git a/app.js b/app.js index f432a617..845a3cd6 100755 --- a/app.js +++ b/app.js @@ -10,9 +10,9 @@ // sanity check var ENV = process.argv[2] -if (ENV != 'development' && ENV != 'production'){ +if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){ console.error("\nnode app.js [environment]"); - console.error("environments: [development, production]\n"); + console.error("environments: [development, production, staging]\n"); process.exit(1); } @@ -33,7 +33,19 @@ var Windshaft = require('windshaft'); var serverOptions = require('./lib/cartodb/server_options'); ws = CartodbWindshaft(serverOptions); + +// Maximum number of connections for one process +// 128 is a good number if you have up to 1024 filedescriptors +// 4 is good if you have max 32 filedescriptors +// 1 is good if you have max 16 filedescriptors +ws.maxConnections = global.environment.maxConnections || 128; + ws.listen(global.environment.port, global.environment.host); + ws.on('listening', function() { console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port); }); + +process.on('SIGUSR1', function() { + ws.dumpCacheStats(); +}); diff --git a/cluster.js b/cluster.js index d5501d17..84ac8040 100755 --- a/cluster.js +++ b/cluster.js @@ -36,6 +36,9 @@ var serverOptions = require('./lib/cartodb/server_options'); var ws = CartodbWindshaft(serverOptions); +// Maximum number of connections for one process (see apps.js) +ws.maxConnections = global.environment.maxConnections || 128; + //.use(cluster.logger('logs')) //.use(cluster.stats()) //.use(cluster.pidfiles('pids')) @@ -48,9 +51,14 @@ var cluster = new Cluster({ noWorkers: 1 }); +console.log(new Date().toISOString() + " - " + process.pid + " - Windshaft cluster starting"); + cluster.listen(function(cb) { cb(ws); }, function() { console.log("Windshaft tileserver started on port " + global.environment.port); }); +process.on('SIGUSR1', function() { + ws.dumpCacheStats(); +}); diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 050cfc45..9c8c10f8 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -2,11 +2,14 @@ var config = { environment: 'development' ,port: 8181 ,host: '127.0.0.1' + // Maximum number of connections for one process + // 128 is a good value with a limit of 1024 open file descriptors + ,maxConnections:128 // idle socket timeout, in miliseconds ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: false - ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' + ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]' ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>' ,postgres: { // Parameters to pass to datasource plugin of mapnik @@ -26,14 +29,28 @@ var config = { max_size: 500 } ,mapnik_version: undefined + ,renderer: { + // Milliseconds since last access before renderer cache item expires + cache_ttl: 60000, + metatile: 4, + bufferSize: 64 + } ,millstone: { cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' } ,redis: { host: '127.0.0.1', port: 6379, - idleTimeoutMillis: 1, - reapIntervalMillis: 1 + // Max number of connections in each pool. + // Users will be put on a queue when the limit is hit. + // Set to maxConnection to have no possible queues. + // There are currently 3 pools involved in serving + // windshaft-cartodb requests so multiply this number + // by 3 to know how many possible connections will be + // kept open by the server. The default is 50. + max: 50, + idleTimeoutMillis: 1, // idle time before dropping connection + reapIntervalMillis: 1 // time between cleanups } ,sqlapi: { protocol: 'http', diff --git a/config/environments/production.js.example b/config/environments/production.js.example index e7a76715..5ee29407 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -2,11 +2,14 @@ var config = { environment: 'production' ,port: 8181 ,host: '127.0.0.1' + // Maximum number of connections for one process + // 128 is a good value with a limit of 1024 open file descriptors + ,maxConnections:128 // idle socket timeout, in miliseconds ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: true - ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' + ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]' ,postgres_auth_user: 'cartodb_user_<%= user_id %>' ,postgres: { // Parameters to pass to datasource plugin of mapnik @@ -20,12 +23,28 @@ var config = { max_size: 500 } ,mapnik_version: undefined + ,renderer: { + // Milliseconds since last access before renderer cache item expires + cache_ttl: 60000, + metatile: 4, + bufferSize: 64 + } ,millstone: { cache_basedir: '/home/ubuntu/tile_assets/' } ,redis: { host: '127.0.0.1', - port: 6379 + port: 6379, + // Max number of connections in each pool. + // Users will be put on a queue when the limit is hit. + // Set to maxConnection to have no possible queues. + // There are currently 3 pools involved in serving + // windshaft-cartodb requests so multiply this number + // by 3 to know how many possible connections will be + // kept open by the server. The default is 50. + max: 50, + idleTimeoutMillis: 30000, // idle time before dropping connection + reapIntervalMillis: 1000 // time between cleanups } ,sqlapi: { protocol: 'https', diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index dedb5001..7fc660c5 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -2,11 +2,14 @@ var config = { environment: 'production' ,port: 8181 ,host: '127.0.0.1' + // Maximum number of connections for one process + // 128 is a good value with a limit of 1024 open file descriptors + ,maxConnections:128 // idle socket timeout, in miliseconds ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: true - ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' + ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]' ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>' ,postgres: { // Parameters to pass to datasource plugin of mapnik @@ -20,12 +23,28 @@ var config = { max_size: 500 } ,mapnik_version: undefined + ,renderer: { + // Milliseconds since last access before renderer cache item expires + cache_ttl: 60000, + metatile: 4, + bufferSize: 64 + } ,millstone: { cache_basedir: '/home/ubuntu/tile_assets/' } ,redis: { host: '127.0.0.1', - port: 6379 + port: 6379, + // Max number of connections in each pool. + // Users will be put on a queue when the limit is hit. + // Set to maxConnection to have no possible queues. + // There are currently 3 pools involved in serving + // windshaft-cartodb requests so multiply this number + // by 3 to know how many possible connections will be + // kept open by the server. The default is 50. + max: 50, + idleTimeoutMillis: 30000, // idle time before dropping connection + reapIntervalMillis: 1000 // time between cleanups } ,sqlapi: { protocol: 'https', diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 096acb03..b97b9a6d 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -2,6 +2,9 @@ var config = { environment: 'test' ,port: 8888 ,host: '127.0.0.1' + // Maximum number of connections for one process + // 128 is a good value with a limit of 1024 open file descriptors + ,maxConnections:128 // idle socket timeout, in miliseconds ,socket_timeout: 600000 ,enable_cors: true @@ -20,14 +23,28 @@ var config = { max_size: 500 } ,mapnik_version: '2.0.2' + ,renderer: { + // Milliseconds since last access before renderer cache item expires + cache_ttl: 60000, + metatile: 4, + bufferSize: 64 + } ,millstone: { cache_basedir: '/tmp/cdb-tiler-test/millstone' } ,redis: { host: '127.0.0.1', port: 6333, - idleTimeoutMillis: 1, - reapIntervalMillis: 1 + // Max number of connections in each pool. + // Users will be put on a queue when the limit is hit. + // Set to maxConnection to have no possible queues. + // There are currently 3 pools involved in serving + // windshaft-cartodb requests so multiply this number + // by 3 to know how many possible connections will be + // kept open by the server. The default is 50. + max: 50, + idleTimeoutMillis: 1, // idle time before dropping connection + reapIntervalMillis: 1 // time between cleanups } ,sqlapi: { protocol: 'http', diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js index 5ad4723c..e43b7233 100644 --- a/lib/cartodb/carto_data.js +++ b/lib/cartodb/carto_data.js @@ -106,7 +106,7 @@ module.exports = function() { if (check_result === 1) { // authorized by key, login as db owner that.getId(req, function(err, user_id) { - if (err) throw new Error(err); + if (err) throw err; var dbuser = _.template(global.settings.postgres_auth_user, {user_id: user_id}); _.extend(req, {dbuser:dbuser}); callback(err, true); @@ -126,7 +126,6 @@ module.exports = function() { that.retrieve(that.table_metadata_db, redisKey, 'privacy', this); }, function(err, data){ - //if (err) throw err; callback(err, data); } ); @@ -152,7 +151,6 @@ module.exports = function() { that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this); }, function(err, data){ - if (err) throw err; callback(err, data); } ); @@ -172,7 +170,6 @@ module.exports = function() { that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this); }, function(err, data){ - if (err) throw err; callback(err, data); } ); @@ -193,7 +190,6 @@ module.exports = function() { that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this); }, function(err, data){ - if (err) throw err; callback(err, data); } ); @@ -227,13 +223,13 @@ module.exports = function() { redis_pool.acquire(db, this); }, function executeQuery(err, data) { + if ( err ) throw err; redisClient = data; redisArgs.push(this); redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs); }, function releaseRedisClient(err, data) { - if (err) throw err; - redis_pool.release(db, redisClient); + if ( ! _.isUndefined(redisClient) ) redis_pool.release(db, redisClient); callback(err, data); } ); diff --git a/lib/cartodb/redis_pool.js b/lib/cartodb/redis_pool.js index ea5c1bd5..d2c40cc2 100644 --- a/lib/cartodb/redis_pool.js +++ b/lib/cartodb/redis_pool.js @@ -58,8 +58,11 @@ var RedisPool = function(opts){ client.send_anyway = true; client.select(database); client.send_anyway = false; + callback(null, client); }); - return callback(null, client); + client.on('error', function (err) { + callback(err, null); + }); }, destroy: function(client) { return client.quit(); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 79141a2c..ac9c458f 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -6,13 +6,28 @@ var _ = require('underscore') ; module.exports = function(){ + + var rendererConfig = _.defaults(global.environment.renderer || {}, { + cache_ttl: 60000, + metatile: 4, + bufferSize: 64 + }); + var me = { base_url: '/tiles/:table', + base_url_notable: '/tiles', grainstore: { datasource: global.environment.postgres, cachedir: global.environment.millstone.cache_basedir, mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik }, + mapnik: { + metatile: rendererConfig.metatile, + bufferSize: rendererConfig.bufferSize + }, + renderCache: { + ttl: rendererConfig.cache_ttl + }, redis: global.environment.redis, enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 8c631486..3ef03a44 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "1.1.5", + "version": "1.1.7", "dependencies": { "cluster2": { "version": "0.3.5-cdb02", @@ -13,7 +13,7 @@ "version": "1.9.2", "dependencies": { "formidable": { - "version": "1.0.11" + "version": "1.0.12" } } }, @@ -32,22 +32,22 @@ "version": "0.8.3" }, "npm": { - "version": "1.1.69", + "version": "1.2.12", "dependencies": { "semver": { - "version": "1.1.1" + "version": "1.1.3" }, "ini": { - "version": "1.0.5" + "version": "1.1.0" }, "slide": { "version": "1.1.3" }, "abbrev": { - "version": "1.0.3" + "version": "1.0.4" }, "graceful-fs": { - "version": "1.1.14" + "version": "1.2.0" }, "minimatch": { "version": "0.2.9", @@ -58,10 +58,10 @@ } }, "nopt": { - "version": "2.0.0" + "version": "2.1.1" }, "rimraf": { - "version": "2.0.2" + "version": "2.1.4" }, "request": { "version": "2.9.203", @@ -71,10 +71,10 @@ "version": "1.0.5" }, "tar": { - "version": "0.1.14" + "version": "0.1.16" }, "fstream": { - "version": "0.1.19" + "version": "0.1.22" }, "block-stream": { "version": "0.0.6" @@ -98,13 +98,13 @@ "version": "2.0.4" }, "node-gyp": { - "version": "0.7.3" + "version": "0.8.4" }, "fstream-npm": { "version": "0.1.3", "dependencies": { "fstream-ignore": { - "version": "0.0.5" + "version": "0.0.6" } } }, @@ -124,7 +124,7 @@ "version": "0.1.2" }, "npm-registry-client": { - "version": "0.2.10", + "version": "0.2.17", "dependencies": { "couch-login": { "version": "0.1.15" @@ -132,13 +132,13 @@ } }, "read-package-json": { - "version": "0.1.11" + "version": "0.2.0" }, "read-installed": { - "version": "0.0.3" + "version": "0.1.1" }, "glob": { - "version": "3.1.14" + "version": "3.1.20" }, "init-package-json": { "version": "0.0.6", @@ -152,7 +152,7 @@ "version": "0.0.3" }, "lockfile": { - "version": "0.2.1" + "version": "0.3.0" }, "retry": { "version": "0.6.0" @@ -161,10 +161,10 @@ "version": "1.1.1" }, "npmconf": { - "version": "0.0.16", + "version": "0.0.23", "dependencies": { "config-chain": { - "version": "1.1.3", + "version": "1.1.5", "dependencies": { "proto-list": { "version": "1.2.2" @@ -187,28 +187,31 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.10.8", + "version": "0.11.1", "dependencies": { "carto": { "version": "0.9.3-cdb2", "from": "git://github.com/CartoDB/carto.git#cdb-0.9", "dependencies": { + "mapnik-reference": { + "version": "5.0.0-cdb1", + "from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0" + }, "xml2js": { "version": "0.1.14", "dependencies": { "sax": { - "version": "0.4.2" + "version": "0.5.2" } } } } }, "mapnik-reference": { - "version": "5.0.0-cdb1", - "from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0" + "version": "5.0.4" }, "millstone": { - "version": "0.5.13", + "version": "0.5.15", "dependencies": { "request": { "version": "2.11.4", @@ -235,26 +238,26 @@ } }, "srs": { - "version": "0.2.18" + "version": "0.2.20" }, "zipfile": { - "version": "0.3.3" + "version": "0.3.4" }, "sqlite3": { "version": "2.1.5" }, "mime": { - "version": "1.2.7" + "version": "1.2.9" }, "mkdirp": { - "version": "0.3.4" + "version": "0.3.5" } } } } }, "windshaft": { - "version": "0.8.2", + "version": "0.9.1", "dependencies": { "express": { "version": "2.5.11", @@ -263,7 +266,7 @@ "version": "1.9.2", "dependencies": { "formidable": { - "version": "1.0.11" + "version": "1.0.12" } } }, @@ -324,10 +327,10 @@ "version": "2.9.202" }, "mapnik": { - "version": "0.7.18" + "version": "0.7.22" }, "semver": { - "version": "1.1.1" + "version": "1.1.3" }, "mocha": { "version": "1.2.1", @@ -350,7 +353,7 @@ "version": "1.0.2" }, "debug": { - "version": "0.7.0" + "version": "0.7.2" } } } diff --git a/package.json b/package.json index 85e3afe1..fab65f18 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.1.5", + "version": "1.1.8", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -21,10 +21,10 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.10.8", - "windshaft" : "~0.8.2", + "grainstore" : "~0.11.1", + "windshaft" : "~0.9.1", "step": "0.0.x", - "generic-pool": "1.0.x", + "generic-pool": "~1.0.12", "redis": "0.7.2", "hiredis": "~0.1.14", "request": "2.9.202", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js new file mode 100644 index 00000000..91e37b34 --- /dev/null +++ b/test/acceptance/multilayer.js @@ -0,0 +1,148 @@ +var assert = require('../support/assert'); +var tests = module.exports = {}; +var _ = require('underscore'); +var redis = require('redis'); +var querystring = require('querystring'); +var semver = require('semver'); +var mapnik = require('mapnik'); +var Step = require('step'); + +require(__dirname + '/../support/test_helper'); + +var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures'; + +var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); +var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); +server.setMaxListeners(0); + +suite('multilayer', function() { + + var redis_client = redis.createClient(global.environment.redis.port); + + suiteSetup(function(){ + }); + + test("layergroup with 2 layers, each with its style", function(done) { + + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.0.1' + } }, + { options: { + sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2', + cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', + cartocss_version: '2.0.2' + } } + ] + }; + + var expected_token = "d442ca6d3ece793b9c16c02a1d1ea5f2"; + Step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/tiles/layergroup', + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); + var expectedBody = { layergroupid: expected_token }; + // TODO: check last modified + //expectedBody.layercount = 2; + if ( expected_token ) assert.deepEqual(parsedBody, expectedBody); + else expected_token = parsedBody.layergroupid; + next(null, res); + }); + }, + function do_get_tile(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + '/0/0/0.png', + method: 'GET', + headers: {host: 'localhost' }, + encoding: 'binary' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "image/png"); + assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function do_get_grid_layer0(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/layer0/0/0/0.grid.json?interactivity=cartodb_id', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8"); + assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function do_get_grid_layer1(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/layer1/0/0/0.grid.json?interactivity=cartodb_id', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8"); + assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function finish(err) { + var errors = []; + if ( err ) { + errors.push(err.message); + console.log("Error: " + err); + } + redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) { + if ( err ) errors.push(err.message); + assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); + redis_client.del(matches, function(err) { + if ( err ) errors.push(err.message); + if ( errors.length ) done(new Error(errors)); + else done(null); + }); + }); + } + ); + }); + + suiteTeardown(function(done) { + // This test will add map_style records, like + // 'map_style|null|publicuser|my_table', + redis_client.keys("map_style|*", function(err, matches) { + _.each(matches, function(k) { redis_client.del(k); }); + done(); + }); + }); + +}); + diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 7f5ccf03..3e419015 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -110,7 +110,7 @@ suite('server', function() { },{ }, function(res) { // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); + assert.equal(res.statusCode, 400, res.body); assert.deepEqual(JSON.parse(res.body), {error: 'Sorry, you are unauthorized (permission denied)'}); done(); @@ -127,7 +127,7 @@ suite('server', function() { },{ }, function(res) { // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(JSON.parse(res.body), {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); done(); @@ -175,7 +175,7 @@ suite('server', function() { headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' }, data: querystring.stringify({style: '#my_table3{backgxxxxxround-color:#fff;}'}) },{ - status: 500, // FIXME: should be 400 ! + status: 400, body: /Unrecognized rule: backgxxxxxround-color/ }, function() { done(); }); }); @@ -187,7 +187,7 @@ suite('server', function() { headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' }, data: querystring.stringify({style: '#my_table3{'}) },{ - status: 500, // FIXME: should be 400 ! + status: 400, body: /Missing closing/ }, function() { done(); }); }); @@ -199,7 +199,7 @@ suite('server', function() { headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' }, data: querystring.stringify({style: '#my_table4{backgxxxxxround-color:#fff;foo:bar}'}) },{ - status: 500, // FIXME: should be 400 ! + status: 400, }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.length, 2); @@ -252,7 +252,7 @@ suite('server', function() { data: querystring.stringify({style: 'Map { background-color:#aaa; }'}) },{}, function(res) { // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body); assert.response(server, { @@ -464,7 +464,7 @@ suite('server', function() { method: 'GET' },{}, function(res) { // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); + assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body); done(); }); }); @@ -479,7 +479,7 @@ suite('server', function() { },{ }, function(res) { // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); + assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body); assert.deepEqual(JSON.parse(res.body), {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); done(); diff --git a/test/support/assert.js b/test/support/assert.js index 82fcf99d..26d3221f 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -8,6 +8,64 @@ var exec = require('child_process').exec; var assert = module.exports = exports = require('assert'); +// @param tolerance number of tolerated grid cell differences +assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) { + fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update + var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8')); + + var err = null; + + var Celldiff = function(x, y, ev, ov) { + this.x = x; + this.y = y; + this.ev = ev; + this.ov = ov; + }; + + Celldiff.prototype.toString = function() { + return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]'; + }; + + try { + var obtained_json = JSON.parse(buffer); + + // compare grid + var obtained_grid = obtained_json.grid; + var expected_grid = expected_json.grid; + var nrows = obtained_grid.length + if (nrows != expected_grid.length) { + throw new Error( "Obtained grid rows (" + nrows + + ") != expected grid rows (" + expected_grid.length + ")" ); + } + var celldiff = []; + for (var i=0; i tolerance ) { + throw new Error( celldiff.length + " cell differences: " + celldiff ); + } + + assert.deepEqual(obtained_json.keys, expected_json.keys); + } catch (e) { err = e; } + + callback(err); +}; + // // @param tol tolerated color distance as a percent over max channel value // by default this is zero. For meaningful values, see diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index c5a39d7e..567b4d1e 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -33,4 +33,5 @@ echo "SADD rails:users:localhost:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5 echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0 echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0 -echo "Finished preparing data. Run tests with expresso." +echo "Finished preparing data. Ready to run tests" + diff --git a/tools/munin/Makefile b/tools/munin/Makefile new file mode 100644 index 00000000..a1aa98b2 --- /dev/null +++ b/tools/munin/Makefile @@ -0,0 +1,19 @@ +MUNIN_PLUGINS_DIR=/etc/munin/plugins +MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d +PWD=$(shell pwd) + +all: windshaft.conf + +windshaft.conf: windshaft.conf.in + sed 's#@PWD@#$(PWD)#' < $< > $@ + +install-munin-plugin-conf: windshaft.conf + install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf + +install-munin-plugin: windshaft + install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft + +install: install-munin-plugin install-munin-plugin-conf + +clean: + rm -f windshaft.conf diff --git a/tools/munin/windshaft b/tools/munin/windshaft new file mode 100755 index 00000000..77adcb23 --- /dev/null +++ b/tools/munin/windshaft @@ -0,0 +1,87 @@ +#!/bin/sh + +envnik=$(basename "${TILER_ENVIRONMENT}" .js) + +if test "$1" = "config"; then +echo "graph_title fd usage (${envnik})" +cat <<"EOM" +graph_vlabel number of file descriptors +graph_category windshaft +graph_scale no +procs.label Number of tiler processes +pgsql.label PostgreSQL connections (max) +redis.label Redis connections (max) +http.label Incoming http requests (max) +caches.label Number of renderer caches (max) +nfd.label Total file descriptors (max) +EOM +exit 0 +fi + +if test x"$1" != x; then + TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js +fi + +if test -z "$TILER_ENVIRONMENT"; then + echo "Usage: $0 []" >&2 + echo " or: [TILER_ENVIRONMENT=] $0" >&2 + exit 1 +fi + +http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1 +pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1 +redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1 + +pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}') +nworkers=$(echo "${pids}" | wc -l) +pids=$(echo "${pids}" | paste -sd ' ') + +if test -z "${pids}"; then + echo "No processes found listening on tcp port '${http_port}'" >&2 + exit 1 +fi + +tmpreport="/tmp/checkfd.$$.txt" + +lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}" + +maxdb=0 +maxredis=0 +maxhttp=0 +maxtot=0 +maxcache=0 + +for pid in ${pids}; do + + cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l); + if test $cnt -gt $maxdb; then maxdb=$cnt; fi + + cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l); + if test $cnt -gt $maxredis; then maxredis=$cnt; fi + + cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l); + if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi + + cnt=$(grep "${pid}" "${tmpreport}" | wc -l); + if test $cnt -gt $maxtot; then maxtot=$cnt; fi + + log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}') + if test -e "${log}"; then + kill -USR1 "${pid}" + cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l) + if test $cnt -gt $maxcache; then maxcache=$cnt; fi + else + # report the error... + maxcache=-1 + fi + +done + +echo "procs.value ${nworkers}" +echo "pgsql.value ${maxdb}" +echo "redis.value ${maxredis}" +echo "http.value ${maxhttp}" +echo "caches.value ${maxcache}" +echo "nfd.value ${maxtot}" + +rm -f "${tmpreport}" diff --git a/tools/munin/windshaft.conf.in b/tools/munin/windshaft.conf.in new file mode 100644 index 00000000..0e71f467 --- /dev/null +++ b/tools/munin/windshaft.conf.in @@ -0,0 +1,5 @@ +# Configuration file for munin plugin + +[windshaft] +user root +env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js