From 34863765ed625d73be301bb17dc2907afb8fd924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Tue, 8 Mar 2016 10:05:21 +0000 Subject: [PATCH 1/2] Freeze cartodb-query-tables version --- npm-shrinkwrap.json | 118 ++++++++++++++++++++++---------------------- package.json | 2 +- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 373ce17c..8f717453 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -84,14 +84,14 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" }, "mime-types": { - "version": "2.1.9", + "version": "2.1.10", "from": "mime-types@>=2.1.9 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } } @@ -123,6 +123,11 @@ } } }, + "cartodb-query-tables": { + "version": "0.1.0", + "from": "cartodb-query-tables@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.1.0.tgz" + }, "cartodb-redis": { "version": "0.13.0", "from": "cartodb-redis@>=0.13.0 <0.14.0", @@ -192,14 +197,14 @@ "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz", "dependencies": { "mime-types": { - "version": "2.1.9", + "version": "2.1.10", "from": "mime-types@>=2.1.6 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } }, @@ -381,14 +386,14 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" }, "mime-types": { - "version": "2.1.9", + "version": "2.1.10", "from": "mime-types@>=2.1.6 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } } @@ -470,11 +475,6 @@ "from": "lzma@>=1.3.7 <1.4.0", "resolved": "https://registry.npmjs.org/lzma/-/lzma-1.3.7.tgz" }, - "cartodb-query-tables": { - "version": "0.1.0", - "from": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master", - "resolved": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" - }, "node-statsd": { "version": "0.0.7", "from": "node-statsd@>=0.0.7 <0.1.0", @@ -525,9 +525,9 @@ "resolved": "https://registry.npmjs.org/request/-/request-2.62.0.tgz", "dependencies": { "bl": { - "version": "1.0.2", + "version": "1.0.3", "from": "bl@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz", "dependencies": { "readable-stream": { "version": "2.0.5", @@ -601,14 +601,14 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" }, "mime-types": { - "version": "2.1.9", + "version": "2.1.10", "from": "mime-types@>=2.1.2 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { - "version": "1.21.0", - "from": "mime-db@>=1.21.0 <1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz" + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" } } }, @@ -729,14 +729,21 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz", "dependencies": { "ansi-styles": { - "version": "2.1.0", + "version": "2.2.0", "from": "ansi-styles@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.0.tgz", + "dependencies": { + "color-convert": { + "version": "1.0.0", + "from": "color-convert@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.0.0.tgz" + } + } }, "escape-string-regexp": { - "version": "1.0.4", + "version": "1.0.5", "from": "escape-string-regexp@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" }, "has-ansi": { "version": "2.0.0", @@ -751,9 +758,9 @@ } }, "strip-ansi": { - "version": "3.0.0", + "version": "3.0.1", "from": "strip-ansi@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "dependencies": { "ansi-regex": { "version": "2.0.0", @@ -782,9 +789,9 @@ } }, "is-my-json-valid": { - "version": "2.12.4", + "version": "2.13.0", "from": "is-my-json-valid@>=2.12.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.4.tgz", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.0.tgz", "dependencies": { "generate-function": { "version": "2.0.0", @@ -3748,9 +3755,9 @@ "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.1.2.tgz" }, "less": { - "version": "2.6.0", + "version": "2.6.1", "from": "less@>=2.5.3 <3.0.0", - "resolved": "https://registry.npmjs.org/less/-/less-2.6.0.tgz", + "resolved": "https://registry.npmjs.org/less/-/less-2.6.1.tgz", "dependencies": { "errno": { "version": "0.1.4", @@ -3765,14 +3772,14 @@ } }, "graceful-fs": { - "version": "3.0.8", - "from": "graceful-fs@>=3.0.5 <4.0.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.8.tgz" + "version": "4.1.3", + "from": "graceful-fs@>=4.1.2 <5.0.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz" }, "image-size": { - "version": "0.3.5", - "from": "image-size@>=0.3.5 <0.4.0", - "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.3.5.tgz" + "version": "0.4.0", + "from": "image-size@>=0.4.0 <0.5.0", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.4.0.tgz" }, "mime": { "version": "1.3.4", @@ -3792,28 +3799,21 @@ } }, "promise": { - "version": "6.1.0", - "from": "promise@>=6.0.1 <7.0.0", - "resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz", + "version": "7.1.1", + "from": "promise@>=7.1.1 <8.0.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.1.1.tgz", "dependencies": { "asap": { - "version": "1.0.0", - "from": "asap@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz" + "version": "2.0.3", + "from": "asap@>=2.0.3 <2.1.0", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.3.tgz" } } }, "source-map": { - "version": "0.4.4", - "from": "source-map@>=0.4.2 <0.5.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", - "dependencies": { - "amdefine": { - "version": "1.0.0", - "from": "amdefine@>=0.0.4", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.0.tgz" - } - } + "version": "0.5.3", + "from": "source-map@>=0.5.3 <0.6.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" } } }, diff --git a/package.json b/package.json index 60eb890e..ff706b08 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "lru-cache": "2.6.5", "lzma": "~1.3.7", "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb", - "cartodb-query-tables": "https://github.com/CartoDB/node-cartodb-query-tables/tarball/master" + "cartodb-query-tables": "~0.1.0" }, "devDependencies": { "istanbul": "~0.3.6", From b56d110f5094e6eb8f39dbd875a4d6cc750584b9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 8 Mar 2016 15:06:06 +0100 Subject: [PATCH 2/2] Remove deprecated tools directory --- NEWS.md | 3 + tools/README.md | 8 -- tools/convert_database_styles | 81 ------------ tools/create_multilayer | 40 ------ tools/create_template | 49 ------- tools/delete_template | 45 ------- tools/examples/mapconfig_simple.js | 11 -- tools/examples/mapconfig_slow.js | 11 -- tools/examples/mapconfig_torque.js | 10 -- tools/examples/template_simple.js | 17 --- tools/instanciate_template | 53 -------- tools/list_templates | 45 ------- tools/munin/Makefile | 19 --- tools/munin/windshaft | 87 ------------- tools/munin/windshaft.conf.in | 5 - tools/performance/stresstest_templates.sh | 54 -------- tools/reset_styles | 105 --------------- tools/show_style | 148 ---------------------- tools/update_template | 47 ------- 19 files changed, 3 insertions(+), 835 deletions(-) delete mode 100644 tools/README.md delete mode 100755 tools/convert_database_styles delete mode 100755 tools/create_multilayer delete mode 100755 tools/create_template delete mode 100755 tools/delete_template delete mode 100644 tools/examples/mapconfig_simple.js delete mode 100644 tools/examples/mapconfig_slow.js delete mode 100644 tools/examples/mapconfig_torque.js delete mode 100644 tools/examples/template_simple.js delete mode 100755 tools/instanciate_template delete mode 100755 tools/list_templates delete mode 100644 tools/munin/Makefile delete mode 100755 tools/munin/windshaft delete mode 100644 tools/munin/windshaft.conf.in delete mode 100755 tools/performance/stresstest_templates.sh delete mode 100755 tools/reset_styles delete mode 100755 tools/show_style delete mode 100755 tools/update_template diff --git a/NEWS.md b/NEWS.md index 8fa300d8..5725bb77 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ Released 2016-mm-dd +Announcements: + - Remove deprecated tools directory + ## 2.26.3 diff --git a/tools/README.md b/tools/README.md deleted file mode 100644 index 5d777333..00000000 --- a/tools/README.md +++ /dev/null @@ -1,8 +0,0 @@ -Deprecated tools -================ - -All tools and scripts found in this directory are deprecated and no longer maintained. - -Use at your own peril. - -In future releases they might get removed. diff --git a/tools/convert_database_styles b/tools/convert_database_styles deleted file mode 100755 index e69fa37c..00000000 --- a/tools/convert_database_styles +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env node - -var path = require('path'); - -var grainstore = require('../node_modules/windshaft/node_modules/grainstore'); -var mapnik = require('mapnik'); -var redis = require('redis'); - -function usage(me, exitcode) { - console.log("Usage: " + me + " []"); - process.exit(exitcode); -} - -var node_path = process.argv.shift(); -var script_path = process.argv.shift(); -var me = path.basename(script_path); - -var database_name = process.argv.shift() -var table_name = process.argv.shift() -var MAPNIK_VERSION = process.argv.shift() - - -if ( ! MAPNIK_VERSION ) { - MAPNIK_VERSION = mapnik.versions.mapnik; -} - -if ( ! database_name || ! table_name) { - usage(me, 1); -} - -var REDIS_PORT = 6379; // TODO: make a command line parameter - -var dbnum = 0; - -var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION}); - -var failures = []; - -var client = redis.createClient(REDIS_PORT, 'localhost'); -client.on('connect', function() { - client.select(dbnum); - client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) { - - processNext = function() { - if ( ! matches.length ) process.exit(failures.length); - var k = matches.shift(); - - if ( /map_style\|.*\|.*\|/.test(k) ) { - //console.warn("Key " + k + " is EXTENDED, skipping"); - processNext(); - } - - var out = 'map_style|' + database_name + '|' + table_name + ': '; - - var mml_builder = mml_store.mml_builder({ - dbname:database_name, - table:table_name}, - function(err, payload) { - if ( err ) { - console.warn(out + err.message); - failures.push(k); processNext(); - } - else { - mml_builder.resetStyle(function(err, data) { - if ( err ) { - console.warn(out + err.message); - failures.push(k); - } - else console.log(out + 'OK'); - processNext(); - }, true); - } - }); - - }; - - processNext(); - - }); - -}); diff --git a/tools/create_multilayer b/tools/create_multilayer deleted file mode 100755 index dde24321..00000000 --- a/tools/create_multilayer +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/layergroup - -# This is for direct windshaft connection -#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup - -while test -n "$1"; do - if test "$1" = "-v"; then - verbose=yes - elif test -z "$cfg"; then - cfg="$1" - else - tiler_url="$1" - fi - shift -done - -if test -z "$cfg"; then - echo "Usage: $0 [-v] []" >&2 - echo "Default is ${tiler_url}" >&2 - exit 1 -fi - -cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}" -if test x${verbose} = xyes; then - cmd="${cmd} -v" -fi -res=`cat ${cfg} | tr '\n' ' ' | ${cmd}` -if test $? -gt 0; then - echo "curl command failed: ${cmd}" -fi - -if test x${verbose} = xyes; then - echo "${res}" -fi - -tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'` -echo $tok diff --git a/tools/create_template b/tools/create_template deleted file mode 100755 index 36661836..00000000 --- a/tools/create_template +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/template -apikey=${CDB_APIKEY} - -while test -n "$1"; do - if test "$1" = "-v"; then - verbose=yes - elif test "$1" = "-k"; then - shift - apikey="$1" - elif test "$1" = "-u"; then - shift - tiler_url="$1" - elif test -z "$cfg"; then - cfg="$1" - else - echo "Unused parameter $1" >&2 - fi - shift -done - -if test -z "$cfg"; then - echo "Usage: $0 [-v] [-k ] [-u ] " >&2 - echo "Default is ${tiler_url}" >&2 - echo "Default is read from CDB_APIKEY env variable" >&2 - exit 1 -fi - -cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}" -if test x${verbose} = xyes; then - cmd="${cmd} -v" -fi -res=`cat ${cfg} | tr '\n' ' ' | ${cmd}` -if test $? -gt 0; then - echo "curl command failed: ${cmd}" -fi - -if test x${verbose} = xyes; then - echo "${res}" -fi - -# Successful response contains no space -echo "$res" | grep " " && { echo $res && exit 1; } - -tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'` -echo $tok - diff --git a/tools/delete_template b/tools/delete_template deleted file mode 100755 index 1dc90968..00000000 --- a/tools/delete_template +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/template -apikey=${CDB_APIKEY} - -while test -n "$1"; do - if test "$1" = "-v"; then - verbose=yes - elif test "$1" = "-k"; then - shift - apikey="$1" - elif test "$1" = "-u"; then - shift - tiler_url="$1" - elif test -z "$tpl"; then - tpl="$1" - else - echo "Unused parameter $1" >&2 - fi - shift -done - -if test -z "$tpl"; then - echo "Usage: $0 [-v] [-k ] [-u ] " >&2 - echo "Default is ${tiler_url}" >&2 - echo "Default is read from CDB_APIKEY env variable" >&2 - exit 1 -fi - -cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}" -if test x${verbose} = xyes; then - cmd="${cmd} -v" -fi -res=`${cmd}` -if test $? -gt 0; then - echo "curl command failed: ${cmd}" -fi - -if test x${verbose} = xyes; then - echo "${res}" -fi - -tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'` -echo $tok diff --git a/tools/examples/mapconfig_simple.js b/tools/examples/mapconfig_simple.js deleted file mode 100644 index e1fc1f43..00000000 --- a/tools/examples/mapconfig_simple.js +++ /dev/null @@ -1,11 +0,0 @@ -{"version":"1.0.1", - "layers":[{ - "type":"cartodb", - "options":{ - "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator", - "cartocss":"#style{ marker-width: 12;}", - "cartocss_version":"2.1.1", - "Interactivity":"id" - } - }] -} diff --git a/tools/examples/mapconfig_slow.js b/tools/examples/mapconfig_slow.js deleted file mode 100644 index 77a92370..00000000 --- a/tools/examples/mapconfig_slow.js +++ /dev/null @@ -1,11 +0,0 @@ -{"version":"1.0.1", - "layers":[{ - "type":"cartodb", - "options":{ - "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x", - "cartocss":"#style{ marker-width: 12;}", - "cartocss_version":"2.1.1", - "Interactivity":"id" - } - }] -} diff --git a/tools/examples/mapconfig_torque.js b/tools/examples/mapconfig_torque.js deleted file mode 100644 index fdff333c..00000000 --- a/tools/examples/mapconfig_torque.js +++ /dev/null @@ -1,10 +0,0 @@ -{"version":"1.0.1", - "layers":[{ - "type":"torque", - "options":{ - "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator", - "cartocss":"Map{ -torque-time-attribute:'id'; -torque-aggregation-function:'count(id)'; -torque-frame-count:2; -torque-resolution:2}", - "cartocss_version": "2.1.1" - } - }] -} diff --git a/tools/examples/template_simple.js b/tools/examples/template_simple.js deleted file mode 100644 index 5402fe5d..00000000 --- a/tools/examples/template_simple.js +++ /dev/null @@ -1,17 +0,0 @@ -{ - "version":"0.0.1", - "name":"simple", - "placeholders":{}, - "auth":{ "method":"open" }, - "layergroup":{ - "version":"1.0.1", - "layers":[{ - "type":"cartodb", - "options":{ - "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator", - "cartocss":"#s{ marker-width: 12;}", - "cartocss_version":"2.1.1" - } - }] - } -} diff --git a/tools/instanciate_template b/tools/instanciate_template deleted file mode 100755 index d367913c..00000000 --- a/tools/instanciate_template +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/template -apikey=${CDB_APIKEY} - -while test -n "$1"; do - if test "$1" = "-v"; then - verbose=yes - elif test "$1" = "-k"; then - shift - apikey="$1" - elif test "$1" = "-u"; then - shift - tiler_url="$1" - elif test -z "$tpl"; then - tpl="$1" - elif test -z "$cfg"; then - cfg="$1" - else - echo "Unused parameter $1" >&2 - fi - shift -done - -if test -z "$tpl"; then - echo "Usage: $0 [-v] [-k ] [-u ] []" >&2 - echo "Default is ${tiler_url}" >&2 - echo "Default is read from CDB_APIKEY env variable" >&2 - exit 1 -fi - -if test -z "$cfg"; then - cfg="/dev/null" -fi - -tiler_url="${tiler_url}/${tpl}" - -cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}" -if test x${verbose} = xyes; then - cmd="${cmd} -v" -fi -res=`cat ${cfg} | tr '\n' ' ' | ${cmd}` -if test $? -gt 0; then - echo "curl command failed: ${cmd}" -fi - -if test x${verbose} = xyes; then - echo "${res}" -fi - -tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'` -echo $tok diff --git a/tools/list_templates b/tools/list_templates deleted file mode 100755 index 1fc887b9..00000000 --- a/tools/list_templates +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/template -apikey=${CDB_APIKEY} - -while test -n "$1"; do - if test "$1" = "-v"; then - verbose=yes - elif test "$1" = "-k"; then - shift - apikey="$1" - elif test "$1" = "-u"; then - shift - tiler_url="$1" - elif test "$1" = "-h" -o "$1" = "-?"; then - echo "Usage: $0 [-v] [-k ] [-u ]" >&2 - echo "Default is ${tiler_url}" >&2 - echo "Default is read from CDB_APIKEY env variable" >&2 - exit 0 - else - echo "Unused parameter $1" >&2 - fi - shift -done - - -cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}" -if test x${verbose} = xyes; then - cmd="${cmd} -v" -fi -res=`${cmd}` -if test $? -gt 0; then - echo "curl command failed: ${cmd}" -fi - -if test x${verbose} = xyes; then - echo "${res}" -fi - -node < $@ - -install-munin-plugin-conf: windshaft.conf - install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf - -install-munin-plugin: windshaft - install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft - -install: install-munin-plugin install-munin-plugin-conf - -clean: - rm -f windshaft.conf diff --git a/tools/munin/windshaft b/tools/munin/windshaft deleted file mode 100755 index e0913447..00000000 --- a/tools/munin/windshaft +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh - -envnik=$(basename "${TILER_ENVIRONMENT}" .js) - -if test "$1" = "config"; then -echo "graph_title fd usage (${envnik})" -cat <<"EOM" -graph_vlabel number of file descriptors -graph_category windshaft -graph_scale no -procs.label Number of tiler processes -pgsql.label PostgreSQL connections (max) -redis.label Redis connections (max) -http.label Incoming http requests (max) -caches.label Number of renderer caches (max) -nfd.label Total file descriptors (max) -EOM -exit 0 -fi - -if test x"$1" != x; then - TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js -fi - -if test -z "$TILER_ENVIRONMENT"; then - echo "Usage: $0 []" >&2 - echo " or: [TILER_ENVIRONMENT=] $0" >&2 - exit 1 -fi - -http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1 -pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1 -redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1 - -pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}') -nworkers=$(echo "${pids}" | wc -l) -pids=$(echo "${pids}" | paste -sd ' ') - -if test -z "${pids}"; then - echo "No processes found listening on tcp port '${http_port}'" >&2 - exit 1 -fi - -tmpreport="/tmp/checkfd.$$.txt" - -lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}" - -maxdb=0 -maxredis=0 -maxhttp=0 -maxtot=0 -maxcache=0 - -for pid in ${pids}; do - - cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l); - if test $cnt -gt $maxdb; then maxdb=$cnt; fi - - cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l); - if test $cnt -gt $maxredis; then maxredis=$cnt; fi - - cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l); - if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi - - cnt=$(grep "${pid}" "${tmpreport}" | wc -l); - if test $cnt -gt $maxtot; then maxtot=$cnt; fi - - log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}') - if test -e "${log}"; then - kill -USR2 "${pid}" - cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l) - if test $cnt -gt $maxcache; then maxcache=$cnt; fi - else - # report the error... - maxcache=-1 - fi - -done - -echo "procs.value ${nworkers}" -echo "pgsql.value ${maxdb}" -echo "redis.value ${maxredis}" -echo "http.value ${maxhttp}" -echo "caches.value ${maxcache}" -echo "nfd.value ${maxtot}" - -rm -f "${tmpreport}" diff --git a/tools/munin/windshaft.conf.in b/tools/munin/windshaft.conf.in deleted file mode 100644 index 0e71f467..00000000 --- a/tools/munin/windshaft.conf.in +++ /dev/null @@ -1,5 +0,0 @@ -# Configuration file for munin plugin - -[windshaft] -user root -env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js diff --git a/tools/performance/stresstest_templates.sh b/tools/performance/stresstest_templates.sh deleted file mode 100755 index 56ac4ef9..00000000 --- a/tools/performance/stresstest_templates.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/template -apikey=${CDB_APIKEY} - -while test -n "$1"; do - if test "$1" = "-k"; then - shift - apikey="$1" - elif test "$1" = "-u"; then - shift - tiler_url="$1" - elif test -z "$tpl"; then - tpl="$1" - else - echo "Unused parameter $1" >&2 - fi - shift -done - -if test -z "$tpl"; then - echo "Usage: $0 [-v] [-k ] [-u ] " >&2 - echo "Default is ${tiler_url}" >&2 - echo "Default is read from CDB_APIKEY env variable" >&2 - exit 1 -fi - -basedir=$(cd $(dirname $0); cd ..; pwd) -export CDB_APIKEY=${apikey} -max=3000000 -i=0 -while test "$i" -le "$max"; do - tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"` - tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin` - if test $? -ne 0; then - echo $tpl_id >&2 - break - fi - tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin` - if test $? -ne 0; then - echo $tpl_id >&2 - break - fi - out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}` - if test $? -ne 0; then - echo $out >&2 - break - fi - i=$((i+1)) - if test `expr $i % 100` -eq 0; then - echo -n "." - fi -done diff --git a/tools/reset_styles b/tools/reset_styles deleted file mode 100755 index 9f770f0c..00000000 --- a/tools/reset_styles +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env node - -/* - -This scripts drops all extended map_style keys in redis and regenerates -the XML caches in all the base ones to target the configured mapnik_version. - -Optionally (with --convert) it also re-writes the CartoCSS if needed -to target the configured mapnik_version. - -It is recommended to make a backup of the redis database before using -this script. - -*/ - -var path = require('path'); - -// Reset all styles in the store -var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore'); -var mapnik = require('mapnik'); -var redis = require('redis'); - -function usage(me, exitcode) { - console.log("Usage: " + me + " [--convert] "); - process.exit(exitcode); -} - -var doConvert = false; - -var node_path = process.argv.shift(); -var script_path = process.argv.shift(); -var me = path.basename(script_path); -var ENV; -var arg; -while ( arg = process.argv.shift() ) { - if ( arg == '--convert' ) { - doConvert = true; - } - else if ( ! ENV ) { - ENV = arg; - } - else { - usage(me, 1); - } -} - -if ( ! ENV ) usage(me, 1); - -global.environment = require('../config/environments/' + ENV); -var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment - -var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik; - -console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION); - -var dbnum = 0; - -var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore); - -var failures = []; - -var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host); -client.on('connect', function() { - client.select(dbnum); - client.keys('map_style|*', function(err, matches) { - - processNext = function() { - if ( ! matches.length ) process.exit(failures.length); - var k = matches.shift(); - - if ( /map_style\|.*\|.*\|/.test(k) ) { - // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58 - //console.warn("Key " + k + " is EXTENDED, dropping"); - client.del(k, function(err) { - if ( err ) console.warn("Error dropping key " + k); - processNext(); - }); - return; - } - - var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k); - var db = params[1]; - var tab = params[2]; - var out = 'map_style|' + db + '|' + tab + ': '; - - var mml_builder = mml_store.mml_builder({dbname:db, table:tab}, - function(err, payload) { - - if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); } - else { - mml_builder.resetStyle(function(err, data) { - if ( err ) { console.warn(out + err.message); failures.push(k); } - else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' )); - processNext(); - }, doConvert); - } - }); - - }; - - processNext(); - - }); - -}); diff --git a/tools/show_style b/tools/show_style deleted file mode 100755 index 66ae5924..00000000 --- a/tools/show_style +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env node - -var path = require('path'); -var redis = require('redis'); -var Step = require('step'); - - -function usage(me, exitcode) { - console.log("Usage: " + me + " [--env ] [|~]"); - process.exit(exitcode); -} - -var node_path = process.argv.shift(); -var script_path = process.argv.shift(); -var basedir = path.dirname(script_path); -var me = path.basename(script_path); - -var ENV = 'development.js'; -var username, token; -var arg; -while ( arg = process.argv.shift() ) { - if ( arg == '--env' ) { - ENV = process.argv.shift(); - } - else if ( ! username ) { - username = arg; - } - else if ( ! token ) { - token = arg; - } - else { - console.warn("Unused parameter " + arg); - } -} - -if ( ! username ) usage(me, 1); - -console.log("Using environment " + ENV); - -global.environment = require('../config/environments/' + ENV); -// _after_ setting global.environment -var serverOptions = require('../lib/cartodb/server_options')(); - -var client; -var dbname; -Step( - function getClient() { - client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host); - client.on('connect', this); - }, - function getUserMeta(err) { - if ( err ) throw err; - client.select(5); - client.hgetall('rails:users:' + username, this); - }, - function readDB(err, data) { - if ( err ) throw err; - if ( ! data ) - throw new Error('Username ' + username + ' unknown by redis on port ' - + serverOptions.redis.port + ' (try CARTODB/script/restore_redis?)'); - //console.log("Data:"); console.dir(data); - dbname = data['database_name']; - console.log("Database name for user " + username + ": " + dbname); - client.select(0); - return null; - }, - function showTokens(err) { - if ( err ) throw err; - if ( token ) return null; - var next = this; - Step( - function getTokens() { - client.keys('map_style|' + dbname + '|*', this); - }, - function showTokens(err, data) { - if (err) throw err; - if ( data ) console.log(data.join('\n')); - return null; - }, - function showTokensFinish(err) { - next(err); - } - ); - }, - function showStyle(err) { - if ( err ) throw err; - if ( ! token ) return null; - var next = this; - Step( - function getStyle() { - client.get('map_style|' + dbname + '|' + token, this); - }, - function showStyle(err, data) { - if ( err ) throw err; - if ( ! data ) { - throw new Error(token + ': no such map style known by redis on port ' - + serverOptions.redis.port); - } -//console.log("data: " + data); - var x=JSON.parse(data); - printMapnikStyle(x, this); - }, - function showStyleFinish(err) { - next(err); - } - ); - }, - function finish(err) { - if ( err ) { - console.error(err.message) - process.exit(1); - } - process.exit(0); - } -); - -function printMapnikStyle(x, callback) { - console.log('style: ' + x.style); - console.log('version: ' + x.version); - var grainstore = require(basedir + '/../node_modules/windshaft/node_modules/grainstore/lib/grainstore'); - var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore); - var builderconfig = {dbname:dbname}; - if ( token.match(/^~/) ) { - builderconfig.token = token.substring(1); - } else { - builderconfig.table = token; - } - var mml_builder; - Step( - function getBuilder() { - mml_builder = mml_store.mml_builder(builderconfig, this); - }, - function getXML(err, builder) { - if ( err ) throw err; - mml_builder.toXML(this); - }, - function showXML(err, xml) { - if ( err ) throw err; - console.log('- XML - '); - console.log(xml); - return null; - }, - function finish(err) { - callback(err); - } - ); -} - diff --git a/tools/update_template b/tools/update_template deleted file mode 100755 index 6fcdb181..00000000 --- a/tools/update_template +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -verbose=no -tiler_url=http://dev.localhost.lan:8181/tiles/template -apikey=${CDB_APIKEY} - -while test -n "$1"; do - if test "$1" = "-v"; then - verbose=yes - elif test "$1" = "-k"; then - shift - apikey="$1" - elif test "$1" = "-u"; then - shift - tiler_url="$1" - elif test -z "$tpl"; then - tpl="$1" - elif test -z "$cfg"; then - cfg="$1" - else - echo "Unused parameter $1" >&2 - fi - shift -done - -if test -z "$cfg"; then - echo "Usage: $0 [-v] [-k ] [-u ] " >&2 - echo "Default is ${tiler_url}" >&2 - echo "Default is read from CDB_APIKEY env variable" >&2 - exit 1 -fi - -cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}" -if test x${verbose} = xyes; then - cmd="${cmd} -v" -fi -res=`cat ${cfg} | tr '\n' ' ' | ${cmd}` -if test $? -gt 0; then - echo "curl command failed: ${cmd}" -fi - -if test x${verbose} = xyes; then - echo "${res}" -fi - -tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'` -echo $tok