first commit

tools/cdbsql

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+
+// Command line tool for CartoDB SQL API
+//
+// https://github.com/Vizzuality/CartoDB-SQL-API
+//
+
+var http = require('http');
+
+var nodevers = process.versions.node.split('.');
+
+// NOTE: readline is also available in 0.4 but doesn't work
+var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
+//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
+
+var readline = hasReadline ? require('readline') : null;
+var tty = require('tty');
+
+var me = process.argv[1];
+
+process.argv.shift(); // this will be "node" (argv[0])
+process.argv.shift(); // this will be "benchmark.js" (argv[1])
+
+var batch_mode = false;
+var format = 'json';
+var username = 'dev';
+var domain = 'localhost';
+var port = 8080;
+var api_version = 2;
+var api_key = process.env['CDBSQL_APIKEY'];
+var sql;
+var decimal_places;
+var echo_queries = false;
+var skipfields;
+var verbose = 0;
+
+function usage(exit_code) {
+  console.log("Usage: " + me + " [OPTIONS] <query>");
+  if ( hasReadline ) {
+    console.log("       " + me + " [OPTIONS]");
+  }
+  console.log("Options:");
+  console.log(" -v                      verbose operations (off)");
+  console.log(" --help                  print this help");
+  console.log(" --user <string>         cartodb username (" + username + ")");
+  console.log(" --domain <string>       service domain (" + domain + ")");
+  console.log(" --port <num>            service tcp port number (" + port + ")");
+  console.log(" --api-version <num>     API version (" + api_version +")");
+  console.log(" --key <string>          API authentication key (CDBSQL_APIKEY)");
+  console.log(" --format <string>       Response format (json)");
+  console.log(" --dp <num>              Decimal places in geojson format (unspecified)");
+  console.log(" --skipfields <string>   Comma-separated list of fields to skip (none)");
+  console.log(" --echo-queries          echo commands sent to server");
+  if ( hasReadline ) {
+  console.log(" --batch                 Send all read queries at once (off)");
+  }
+  process.exit(exit_code);
+}
+
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '-v' ) {
+    ++verbose;
+  }
+  else if ( arg == '--help' ) {
+    usage(0);
+  }
+  else if ( arg == '--key' ) {
+    api_key = process.argv.shift();
+  }
+  else if ( arg == '--domain' ) {
+    domain = process.argv.shift();
+  }
+  else if ( arg == '--user' ) {
+    username = process.argv.shift();
+  }
+  else if ( arg == '--port' ) {
+    port = process.argv.shift();
+  }
+  else if ( arg == '--api-version' ) {
+    api_version = process.argv.shift();
+  }
+  else if ( arg == '--format' ) {
+    format = process.argv.shift();
+  }
+  else if ( arg == '--dp' ) {
+    decimal_places = process.argv.shift();
+  }
+  else if ( arg == '--batch' ) {
+    batch_mode = true;
+  }
+  else if ( arg == '--skipfields' ) {
+    skipfields = process.argv.shift();
+  }
+  else if ( arg == '--echo-queries' ) {
+    echo_queries = true;
+  }
+  else if ( ! sql ) {
+    sql = arg;
+  }
+  else {
+    usage(1);
+  }
+}
+
+var hostname = username + '.' + domain;
+
+if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) {
+  batch_mode = true;
+}
+
+if ( ! sql ) {
+
+  if ( ! batch_mode ) {
+
+    if ( readline ) {
+
+      var rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+      });
+
+      rl.setPrompt(hostname + '> ');
+      rl.prompt();
+
+      sql = '';
+      rl.on('line', function(line) {
+        sql += line;
+        // TODO: some sanity checking, like trim the line or check if it ends with semicolon
+        if ( sql.length ) {
+          processQuery(sql, function() {
+            sql = '';
+            rl.prompt();
+          });
+        } else rl.prompt();
+      }).on('close', function() {
+        if ( sql.length ) {
+          console.warn("Unprocessed sql left: [" + sql + "]");
+        }
+        console.log("Good bye");
+      }).on('SIGCONT', function() {
+        // this is needed so not to exit on stop/resume
+        rl.prompt();
+      });
+    } else {
+      usage(1);
+    }
+  } else { // batch mode
+    process.stdin.resume();
+    process.stdin.setEncoding('utf8');
+    sql = '';
+    process.stdin.on('data', function(chunk) {
+      // TODO: some sanity checking, like trim the line or check if it ends with semicolon
+      processQuery(chunk);
+    });
+  }
+} else {
+  processQuery(sql);
+}
+
+// -- Perform the request
+
+function processQuery(sql, callback)
+{
+
+  var post_data = 'q=' + encodeURIComponent(sql);
+
+  var opt = {
+    host: hostname,
+    port: port,
+    path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format),
+    method: 'POST',
+    headers: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+          'Content-Length': post_data.length
+    }
+  };
+
+  if ( api_key ) opt.path += '&api_key=' + api_key;
+  if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places;
+  if ( typeof(skipfields) != 'undefined' ) opt.path += '&skipfields=' + encodeURIComponent(skipfields);
+
+  var body = '';
+  var request = 'http://' + opt.host + ':' + opt.port + opt.path;
+  if ( verbose ) {
+    console.log("Posting " + post_data + " to ", request);
+  }
+
+  var req = http.request(opt, function(res) {
+    //console.log("Response status: " + res.statusCode);
+    res.on('data', function(chunk) {
+      body += chunk;
+      //console.log("data: "); console.dir(json);
+    });
+
+    res.on('end', function() {
+      console.log("Request:", request);
+      var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
+      sqlprint = sqlprint.split('\n').join(' ');
+      if ( echo_queries ) console.log("Query:", sqlprint);
+      console.log("Response status: " + res.statusCode);
+      console.log('Response body:');
+      console.dir(body);
+      if ( callback ) callback();
+    });
+
+  }).on('error', function(e) {
+    console.log("Request:", request);
+    console.log("Error: " + e.message);
+    if ( callback ) callback();
+  });
+
+  req.write(post_data);
+  req.end();
+
+}

tools/munin/Makefile

@@ -0,0 +1,19 @@
+MUNIN_PLUGINS_DIR=/etc/munin/plugins
+MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
+PWD=$(shell pwd)
+
+all: cdbsqlapi.conf
+
+cdbsqlapi.conf: cdbsqlapi.conf.in
+	sed 's#@PWD@#$(PWD)#' < $< > $@
+
+install-munin-plugin-conf: cdbsqlapi.conf 
+	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/cdbsqlapi.conf
+
+install-munin-plugin: cdbsqlapi
+	install -m 755 $< $(MUNIN_PLUGINS_DIR)/cdbsqlapi
+
+install: install-munin-plugin install-munin-plugin-conf
+
+clean:
+	rm -f cdbsqlapi.conf

tools/munin/cdbsqlapi

@@ -0,0 +1,74 @@
+#!/bin/sh
+
+envnik=$(basename "${SQLAPI_ENVIRONMENT}" .js)
+
+if test "$1" = "config"; then
+echo "graph_title fd usage (${envnik})"
+cat <<"EOM"
+graph_vlabel number of file descriptors
+graph_category sqlapi
+graph_scale no
+procs.label Number of worker processes
+pgsql.label PostgreSQL connections (max)
+redis.label Redis connections (max)
+http.label Incoming http requests (max)
+nfd.label Total file descriptors (max)
+EOM
+exit 0
+fi
+
+if test x"$1" != x; then
+  SQLAPI_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
+fi
+
+if test -z "$SQLAPI_ENVIRONMENT"; then
+  echo "Usage: $0 [<environment>]" >&2
+  echo "   or: [SQLAPI_ENVIRONMENT=<environment>] $0" >&2
+  exit 1
+fi
+
+http_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').node_port)" | node) || exit 1
+pgsql_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').db_port)" | node) || exit 1
+redis_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').redis_port)" | node) || exit 1
+
+pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
+nworkers=$(echo "${pids}" | wc -l)
+pids=$(echo "${pids}" | paste -sd ' ')
+
+if test -z "${pids}"; then
+  echo "No processes found listening on tcp port '${http_port}'" >&2
+  exit 1
+fi
+
+tmpreport="/tmp/checkfd.$$.txt"
+
+lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
+
+maxdb=0
+maxredis=0
+maxhttp=0
+maxtot=0
+
+for pid in ${pids}; do
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
+  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
+  
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
+  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
+  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
+  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
+
+done
+
+echo "procs.value ${nworkers}"
+echo "pgsql.value ${maxdb}"
+echo "redis.value ${maxredis}"
+echo "http.value ${maxhttp}"
+echo "nfd.value ${maxtot}"
+
+rm -f "${tmpreport}"

tools/munin/cdbsqlapi.conf.in

@@ -0,0 +1,5 @@
+# Configuration file for munin plugin
+
+[cdbsqlapi]
+user root
+env.SQLAPI_ENVIRONMENT @PWD@/../../config/environments/production.js