first commit

app/models/bin_encoder.js

@@ -0,0 +1,166 @@
+'use strict';
+
+function ArrayBufferSer(type, data, options) {
+  if(type === undefined) {
+      throw "ArrayBufferSer should be created with a type";
+  }
+  this.options = options || {};
+  this._initFunctions();
+  this.headerSize = 8;
+  this.data = data;
+  this.type = type = Math.min(type, ArrayBufferSer.BUFFER);
+  var size = this._sizeFor(this.headerSize, data);
+  this.buffer = new Buffer(this.headerSize + size);
+  this.buffer.writeUInt32BE(type, 0); // this could be one byte but for byte padding is better to be 4 bytes
+  this.buffer.writeUInt32BE(size, 4);
+  this.offset = this.headerSize;
+
+  var w = this.writeFn[type];
+
+  var i;
+  if(!this.options.delta) {
+    for(i = 0; i < data.length; ++i) {
+      this[w](data[i]);
+    }
+  } else {
+    this[w](data[0]);
+    for(i = 1; i < data.length; ++i) {
+      this[w](data[i] - data[i - 1]);
+    }
+  }
+}
+
+//
+// constants
+//
+ArrayBufferSer.INT8 = 1;
+ArrayBufferSer.UINT8 = 2;
+ArrayBufferSer.UINT8_CLAMP = 3;
+ArrayBufferSer.INT16 = 4;
+ArrayBufferSer.UINT16 = 5;
+ArrayBufferSer.INT32 = 6;
+ArrayBufferSer.UINT32 = 7;
+ArrayBufferSer.FLOAT32 = 8;
+//ArrayBufferSer.FLOAT64 = 9; not supported
+ArrayBufferSer.STRING = 10;
+ArrayBufferSer.BUFFER = 11;
+
+ArrayBufferSer.MAX_PADDING = ArrayBufferSer.INT32;
+
+
+ArrayBufferSer.typeNames = {
+  'int8': ArrayBufferSer.INT8,
+  'uint8': ArrayBufferSer.UINT8,
+  'uintclamp': ArrayBufferSer.UINT8_CLAMP,
+  'int16': ArrayBufferSer.INT16,
+  'uint16': ArrayBufferSer.UINT16,
+  'int32': ArrayBufferSer.INT32,
+  'uint32': ArrayBufferSer.UINT32,
+  'float32': ArrayBufferSer.FLOAT32,
+  'string': ArrayBufferSer.STRING,
+  'buffer': ArrayBufferSer.BUFFER
+};
+
+ArrayBufferSer.prototype = {
+
+  // 0 not used
+  sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8],
+
+  _paddingFor: function(off, type) {
+    var s = this.sizes[type];
+    if(s) {
+      var r = off % s;
+      return r === 0 ? 0 : s - r;
+    }
+    return 0;
+  },
+
+  _sizeFor: function(offset, t) {
+    var self = this;
+    var s = this.sizes[this.type];
+    if(s) {
+      return s*t.length;
+    }
+    s = 0;
+    if(this.type === ArrayBufferSer.STRING) {
+      // calculate size with padding
+      t.forEach(function(arr) {
+        var pad = self._paddingFor(offset, ArrayBufferSer.MAX_PADDING);
+        s += pad;
+        offset += pad;
+        var len = (self.headerSize + arr.length*2);
+        s += len;
+        offset += len;
+      });
+    } else {
+      t.forEach(function(arr) {
+        var pad = self._paddingFor(offset, ArrayBufferSer.MAX_PADDING);
+        s += pad;
+        offset += pad;
+        s += arr.getSize();
+        offset += arr.getSize();
+      });
+    }
+    return s;
+  },
+
+  getDataSize: function() {
+    return this._sizeFor(0, this.data);
+  },
+
+  getSize: function() {
+    return this.headerSize + this._sizeFor(this.headerSize, this.data);
+  },
+
+  writeFn: [
+      '',
+      'writeInt8',
+      'writeUInt8',
+      'writeUInt8Clamp',
+      'writeInt16LE',
+      'writeUInt16LE',
+      'writeUInt32LE',
+      'writeUInt32LE',
+      'writeFloatLE',
+      'writeDoubleLE',
+      'writeString',
+      'writteBuffer'
+  ],
+
+  _initFunctions: function() {
+    var self = this;
+    this.writeFn.forEach(function(fn) {
+      if(self[fn] === undefined) {
+        self[fn] = function(d) {
+          self.buffer[fn](d, self.offset);
+          self.offset += self.sizes[self.type];
+        };
+      }
+    });
+  },
+
+  writeUInt8Clamp: function(c) {
+    this.buffer.writeUInt8(Math.min(255, c), this.offset);
+    this.offset += 1;
+  },
+
+  writeString: function(s) {
+    var arr = [];
+    for(var i = 0, len = s.length; i < len; ++i) {
+      arr.push(s.charCodeAt(i));
+    }
+    var str = new ArrayBufferSer(ArrayBufferSer.UINT16, arr);
+    this.writteBuffer(str);
+  },
+
+  writteBuffer: function(b) {
+    this.offset += this._paddingFor(this.offset, ArrayBufferSer.MAX_PADDING);
+    // copy header
+    b.buffer.copy(this.buffer, this.offset);
+    this.offset += b.buffer.length;
+  }
+
+};
+
+
+module.exports = ArrayBufferSer;

app/models/cartodb_request.js

@@ -0,0 +1,41 @@
+'use strict';
+
+/**
+ * this module provides cartodb-specific interpretation
+ * of request headers
+ */
+
+function CartodbRequest() {
+}
+
+module.exports = CartodbRequest;
+
+/**
+ * If the request contains the user use it, if not guess from the host
+ */
+CartodbRequest.prototype.userByReq = function(req) {
+    if (req.params.user) {
+        return req.params.user;
+    }
+    return userByHostName(req.headers.host);
+};
+
+var re_userFromHost = new RegExp(
+    global.settings.user_from_host || '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
+);
+
+function userByHostName(host) {
+    var mat = host.match(re_userFromHost);
+    if (!mat) {
+        console.error("ERROR: user pattern '" + re_userFromHost + "' does not match hostname '" + host + "'");
+        return;
+    }
+
+    if (mat.length !== 2) {
+        console.error(
+            "ERROR: pattern '" + re_userFromHost + "' gave unexpected matches against '" + host + "': " + mat
+        );
+        return;
+    }
+    return mat[1];
+}

app/models/formats/README

@@ -0,0 +1,18 @@
+Format classes are required to expose a constructor with no arguments,
+a getFileExtension() and a sendResponse(opts, callback) method.
+
+The ``opts`` parameter contains:
+
+ sink        Output stream to send the reponse to
+ sql         SQL query requested by the user
+ skipfields  Comma separate list of fields to skip from output
+             really only needed with "SELECT *" queries
+ gn          Name of the geometry column (for formats requiring one)
+ dp          Number of decimal points of precision for geometries (if used)
+ database    Name of the database to connect to
+ user_id     Identifier of the user
+ filename    Name to use for attachment disposition
+ 
+The ``callback`` parameter is a function that is invoked when the
+format object finished with sending the result to the sink.
+If an error occurs the callback is invoked with an Error argument.

app/models/formats/index.js

@@ -0,0 +1,22 @@
+'use strict';
+
+var fs = require("fs");
+var formats = {};
+
+function formatFilesWithPath(dir) {
+    var formatDir = __dirname + '/' + dir;
+    return fs.readdirSync(formatDir).map(function(formatFile) {
+        return formatDir + '/' + formatFile;
+    });
+}
+
+var formatFilesPaths = []
+    .concat(formatFilesWithPath('ogr'))
+    .concat(formatFilesWithPath('pg'));
+
+formatFilesPaths.forEach(function(file) {
+    var format = require(file);
+    formats[format.prototype.id] = format;
+});
+
+module.exports = formats;

app/models/formats/ogr.js

@@ -0,0 +1,346 @@
+'use strict';
+
+var crypto = require('crypto');
+var step = require('step');
+var fs = require('fs');
+var _ = require('underscore');
+var PSQL = require('cartodb-psql');
+var spawn = require('child_process').spawn;
+
+// Keeps track of what's waiting baking for export
+var bakingExports = {};
+
+function OgrFormat(id) {
+  this.id = id;
+}
+
+OgrFormat.prototype = {
+
+  id: "ogr",
+
+  is_file: true,
+
+  getQuery: function(/*sql, options*/) {
+    return null; // dont execute the query
+  },
+
+  transform: function(/*result, options, callback*/) {
+    throw "should not be called for file formats";
+  },
+
+  getContentType: function(){ return this._contentType; },
+
+  getFileExtension: function(){ return this._fileExtension; },
+
+  getKey: function(options) {
+    return [this.id,
+        options.dbopts.dbname,
+        options.dbopts.user,
+        options.gn,
+        this.generateMD5(options.filename),
+        this.generateMD5(options.sql)].concat(options.skipfields).join(':');
+  },
+
+  generateMD5: function (data){
+    var hash = crypto.createHash('md5');
+    hash.update(data);
+    return hash.digest('hex');
+  }
+
+};
+
+// Internal function usable by all OGR-driven outputs
+OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback) {
+
+  //var gcol = options.gn;
+  var sql = options.sql;
+  var skipfields = options.skipfields;
+  var out_layername = options.filename;
+
+  var dbopts = options.dbopts;
+
+  var ogr2ogr = global.settings.ogr2ogrCommand || 'ogr2ogr';
+  var dbhost = dbopts.host;
+  var dbport = dbopts.port;
+  var dbuser = dbopts.user;
+  var dbpass = dbopts.pass;
+  var dbname = dbopts.dbname;
+
+  var timeout = options.timeout;
+
+  var that = this;
+
+  var columns = [];
+  var geocol;
+  var pg;
+  // Drop ending semicolon (ogr doens't like it)
+  sql = sql.replace(/;\s*$/, '');
+
+  const theGeomFirst = (fieldA, fieldB) => {
+    if (fieldA.name === 'the_geom') {
+      return -1;
+    }
+    if (fieldB.name === 'the_geom') {
+      return 1;
+    }
+    return 0;
+  };
+
+  step (
+
+    function fetchColumns() {
+      var colsql = 'SELECT * FROM (' + sql + ') as _cartodbsqlapi LIMIT 0';
+      pg = new PSQL(dbopts);
+      pg.query(colsql, this);
+    },
+    function findSRS(err, result) {
+      if (err) {
+        throw err;
+      }
+
+      var needSRS = that._needSRS;
+
+      columns = result.fields
+        // skip columns
+        .filter(field => skipfields.indexOf(field.name) === -1)
+        // put "the_geom" first (if exists)
+        .sort(theGeomFirst)
+        // get first geometry to calculate SRID ("the_geom" if exists)
+        .map(field => {
+          if (needSRS && !geocol && pg.typeName(field.dataTypeID) === 'geometry') {
+            geocol = field.name;
+          }
+
+          return field;
+        })
+        // apply quotes to columns
+        .map(field => out_format === 'CSV' ? pg.quoteIdentifier(field.name)+'::text' : pg.quoteIdentifier(field.name));
+
+      if ( ! needSRS || ! geocol ) {
+          return null;
+      }
+
+      var next = this;
+
+      var qgeocol = pg.quoteIdentifier(geocol);
+      var sridsql = 'SELECT ST_Srid(' + qgeocol + ') as srid, GeometryType(' +
+                   qgeocol + ') as type FROM (' + sql + ') as _cartodbsqlapi WHERE ' +
+                   qgeocol + ' is not null limit 1';
+
+      pg.query(sridsql, function(err, result) {
+        if ( err ) { next(err); return; }
+        if ( result.rows.length ) {
+          var srid = result.rows[0].srid;
+          var type = result.rows[0].type;
+          next(null, srid, type);
+        } else {
+          // continue as srid and geom type are not critical when there are no results
+          next(null);
+        }
+      });
+    },
+    function spawnDumper(err, srid, type) {
+      if (err) {
+        throw err;
+      }
+
+      var next = this;
+
+      var ogrsql = 'SELECT ' + columns.join(',') + ' FROM (' + sql + ') as _cartodbsqlapi';
+
+      var ogrargs = [
+        '-f', out_format,
+        '-lco', 'RESIZE=YES',
+        '-lco', 'ENCODING=UTF-8',
+        '-lco', 'LINEFORMAT=CRLF',
+        out_filename,
+        "PG:host=" + dbhost + " port=" + dbport + " user=" + dbuser + " dbname=" + dbname + " password=" + dbpass,
+        '-sql', ogrsql
+      ];
+
+      if ( srid ) {
+        ogrargs.push('-a_srs', 'EPSG:'+srid);
+      }
+
+      if ( type ) {
+        ogrargs.push('-nlt', type);
+      }
+
+      if (options.cmd_params){
+        ogrargs = ogrargs.concat(options.cmd_params);
+      }
+
+      ogrargs.push('-nln', out_layername);
+
+      // TODO: research if `exec` could fit better than `spawn`
+      var child = spawn(ogr2ogr, ogrargs);
+
+      var timedOut = false;
+      var ogrTimeout;
+      if (timeout > 0) {
+        ogrTimeout = setTimeout(function () {
+          timedOut = true;
+          child.kill();
+        }, timeout);
+      }
+
+      child.on('error', function (err) {
+        clearTimeout(ogrTimeout);
+        next(err);
+      });
+
+      var stderrData = [];
+      child.stderr.setEncoding('utf8');
+      child.stderr.on('data', function (data) {
+        stderrData.push(data);
+      });
+
+      child.on('exit', function(code) {
+        clearTimeout(ogrTimeout);
+
+        if (timedOut) {
+          return next(new Error('statement timeout'));
+        }
+
+        if (code !== 0) {
+          var errMessage = 'ogr2ogr command return code ' + code;
+          if (stderrData.length > 0) {
+            errMessage += ', Error: ' + stderrData.join('\n');
+          }
+
+          return next(new Error(errMessage));
+        }
+
+        return next();
+      });
+
+    },
+    function finish(err) {
+      callback(err, out_filename);
+    }
+  );
+};
+
+OgrFormat.prototype.toOGR_SingleFile = function(options, fmt, callback) {
+
+  var dbname = options.dbopts.dbname;
+  var user_id = options.dbopts.user;
+  var gcol = options.gcol;
+  var sql = options.sql;
+  var skipfields = options.skipfields;
+  var ext = this._fileExtension;
+  var layername = options.filename;
+
+  var tmpdir = global.settings.tmpDir || '/tmp';
+  var reqKey = [
+      fmt,
+      dbname,
+      user_id,
+      gcol,
+      this.generateMD5(layername),
+      this.generateMD5(sql)
+  ].concat(skipfields).join(':');
+  var outdirpath = tmpdir + '/sqlapi-' + process.pid + '-' + reqKey;
+  var dumpfile = outdirpath + ':cartodb-query.' + ext;
+
+  // TODO: following tests:
+  //  - fetch query with no "the_geom" column
+  this.toOGR(options, fmt, dumpfile, callback);
+};
+
+OgrFormat.prototype.sendResponse = function(opts, callback) {
+  //var next = callback;
+  var reqKey = this.getKey(opts);
+  var qElem = new ExportRequest(opts.sink, callback, opts.beforeSink);
+  var baking = bakingExports[reqKey];
+  if ( baking ) {
+    baking.req.push( qElem );
+  } else {
+    baking = bakingExports[reqKey] = { req: [ qElem ] };
+    this.generate(opts, function(err, dumpfile) {
+      if ( opts.profiler ) {
+          opts.profiler.done('generate');
+      }
+      step (
+        function sendResults() {
+          var nextPipe = function(finish) {
+            var r = baking.req.shift();
+            if ( ! r ) { finish(null); return; }
+            r.sendFile(err, dumpfile, function() {
+              nextPipe(finish);
+            });
+          };
+
+          if ( ! err ) {
+              nextPipe(this);
+          } else {
+            _.each(baking.req, function(r) {
+              r.cb(err);
+            });
+            return true;
+          }
+        },
+        function cleanup(/*err*/) {
+          delete bakingExports[reqKey];
+
+          // unlink dump file (sync to avoid race condition)
+          console.log("removing", dumpfile);
+          try { fs.unlinkSync(dumpfile); }
+          catch (e) {
+            if ( e.code !== 'ENOENT' ) {
+              console.log("Could not unlink dumpfile " + dumpfile + ": " + e);
+            }
+          }
+        }
+      );
+    });
+  }
+};
+
+// TODO: put in an ExportRequest.js ----- {
+
+function ExportRequest(ostream, callback, beforeSink) {
+  this.cb = callback;
+  this.beforeSink = beforeSink;
+  this.ostream = ostream;
+  this.istream = null;
+  this.canceled = false;
+
+  var that = this;
+
+  this.ostream.on('close', function() {
+    //console.log("Request close event, qElem.stream is " + qElem.stream);
+    that.canceled = true;
+    if ( that.istream ) {
+      that.istream.destroy();
+    }
+  });
+}
+
+ExportRequest.prototype.sendFile = function (err, filename, callback) {
+  var that = this;
+  if ( ! this.canceled ) {
+    //console.log("Creating readable stream out of dumpfile");
+    this.istream = fs.createReadStream(filename)
+    .on('open', function(/*fd*/) {
+      if ( that.beforeSink ) {
+          that.beforeSink();
+      }
+      that.istream.pipe(that.ostream);
+      callback();
+    })
+    .on('error', function(e) {
+      console.log("Can't send response: " + e);
+      that.ostream.end();
+      callback();
+    });
+  } else {
+    //console.log("Response was canceled, not streaming the file");
+    callback();
+  }
+  this.cb();
+};
+
+//------ }
+
+module.exports = OgrFormat;

app/models/formats/ogr/csv.js

@@ -0,0 +1,16 @@
+'use strict';
+
+var ogr = require('./../ogr');
+
+function CsvFormat() {}
+
+CsvFormat.prototype = new ogr('csv');
+
+CsvFormat.prototype._contentType = "text/csv; charset=utf-8; header=present";
+CsvFormat.prototype._fileExtension = "csv";
+
+CsvFormat.prototype.generate = function(options, callback) {
+    this.toOGR_SingleFile(options, 'CSV', callback);
+};
+
+module.exports = CsvFormat;

app/models/formats/ogr/geopackage.js

@@ -0,0 +1,25 @@
+'use strict';
+
+var ogr = require('./../ogr');
+
+function GeoPackageFormat() {}
+
+GeoPackageFormat.prototype = new ogr('gpkg');
+
+GeoPackageFormat.prototype._contentType = "application/x-sqlite3; charset=utf-8";
+GeoPackageFormat.prototype._fileExtension = "gpkg";
+// As of GDAL 1.10.1 SRID detection is bogus, so we use
+// our own method. See:
+//  http://trac.osgeo.org/gdal/ticket/5131
+//  http://trac.osgeo.org/gdal/ticket/5287
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/110
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/116
+// Bug was fixed in GDAL 1.10.2
+GeoPackageFormat.prototype._needSRS = true;
+
+GeoPackageFormat.prototype.generate = function(options, callback) {
+    options.cmd_params = ['-lco', 'FID=cartodb_id'];
+    this.toOGR_SingleFile(options, 'GPKG', callback);
+};
+
+module.exports = GeoPackageFormat;

app/models/formats/ogr/kml.js

@@ -0,0 +1,24 @@
+'use strict';
+
+var ogr = require('./../ogr');
+
+function KmlFormat() {}
+
+KmlFormat.prototype = new ogr('kml');
+
+KmlFormat.prototype._contentType = "application/kml; charset=utf-8";
+KmlFormat.prototype._fileExtension = "kml";
+// As of GDAL 1.10.1 SRID detection is bogus, so we use
+// our own method. See:
+//  http://trac.osgeo.org/gdal/ticket/5131
+//  http://trac.osgeo.org/gdal/ticket/5287
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/110
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/116
+// Bug was fixed in GDAL 1.10.2
+KmlFormat.prototype._needSRS = true;
+
+KmlFormat.prototype.generate = function(options, callback) {
+    this.toOGR_SingleFile(options, 'KML', callback);
+};
+
+module.exports = KmlFormat;

app/models/formats/ogr/shp.js

@@ -0,0 +1,135 @@
+'use strict';
+
+var step = require('step');
+var fs = require('fs');
+var spawn = require('child_process').spawn;
+
+var ogr = require('./../ogr');
+
+function ShpFormat() {
+}
+
+ShpFormat.prototype = new ogr('shp');
+
+ShpFormat.prototype._contentType = "application/zip; charset=utf-8";
+ShpFormat.prototype._fileExtension = "zip";
+// As of GDAL 1.10 SRID detection is bogus, so we use
+// our own method. See:
+//  http://trac.osgeo.org/gdal/ticket/5131
+//  http://trac.osgeo.org/gdal/ticket/5287
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/110
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/116
+ShpFormat.prototype._needSRS = true;
+
+ShpFormat.prototype.generate = function(options, callback) {
+  this.toSHP(options, callback);
+};
+
+ShpFormat.prototype.toSHP = function (options, callback) {
+  var dbname = options.database;
+  var user_id = options.user_id;
+  var gcol = options.gn;
+  var sql = options.sql;
+  var skipfields = options.skipfields;
+  var filename = options.filename;
+
+  var fmtObj = this;
+  var zip = global.settings.zipCommand || 'zip';
+  var zipOptions = '-qrj';
+  var tmpdir = global.settings.tmpDir || '/tmp';
+  var reqKey = [ 'shp', dbname, user_id, gcol, this.generateMD5(sql) ].concat(skipfields).join(':');
+  var outdirpath = tmpdir + '/sqlapi-' + process.pid + '-' + reqKey;
+  var zipfile = outdirpath + '.zip';
+  var shapefile = outdirpath + '/' + filename + '.shp';
+
+  // TODO: following tests:
+  //  - fetch query with no "the_geom" column
+
+  step (
+    function createOutDir() {
+      fs.mkdir(outdirpath, 0o777, this);
+    },
+    function spawnDumper(err) {
+      if (err) {
+        throw err;
+      }
+
+      fmtObj.toOGR(options, 'ESRI Shapefile', shapefile, this);
+    },
+    function doZip(err) {
+      if (err) {
+        throw err;
+      }
+
+      var next = this;
+
+      var child = spawn(zip, [zipOptions, zipfile, outdirpath ]);
+
+      child.on('error', function (err) {
+        next(new Error('Error executing zip command,  ' + err));
+      });
+
+      var stderrData = [];
+      child.stderr.setEncoding('utf8');
+      child.stderr.on('data', function (data) {
+        stderrData.push(data);
+      });
+
+      child.on('exit', function(code) {
+        if (code !== 0) {
+          var errMessage = 'Zip command return code ' + code;
+          if (stderrData.length) {
+            errMessage += ', Error: ' + stderrData.join('\n');
+          }
+
+          return next(new Error(errMessage));
+        }
+
+        return next();
+      });
+    },
+    function cleanupDir(topError) {
+
+      var next = this;
+
+      // Unlink the dir content
+      var unlinkall = function(dir, files, finish) {
+        var f = files.shift();
+        if ( ! f ) { finish(null); return; }
+        var fn = dir + '/' + f;
+        fs.unlink(fn, function(err) {
+          if ( err ) {
+            console.log("Unlinking " + fn + ": " + err);
+            finish(err);
+          } else {
+              unlinkall(dir, files, finish);
+          }
+        });
+      };
+      fs.readdir(outdirpath, function(err, files) {
+        if ( err ) {
+          if ( err.code !== 'ENOENT' ) {
+            next(new Error([topError, err].join('\n')));
+          } else {
+            next(topError);
+          }
+        } else {
+          unlinkall(outdirpath, files, function(/*err*/) {
+            fs.rmdir(outdirpath, function(err) {
+              if ( err ) {
+                  console.log("Removing dir " + outdirpath + ": " + err);
+              }
+              next(topError, zipfile);
+            });
+          });
+        }
+      });
+    },
+    function finalStep(err, zipfile) {
+      callback(err, zipfile);
+    }
+  );
+};
+
+
+module.exports = ShpFormat;

app/models/formats/ogr/spatialite.js

@@ -0,0 +1,25 @@
+'use strict';
+
+var ogr = require('./../ogr');
+
+function SpatiaLiteFormat() {}
+
+SpatiaLiteFormat.prototype = new ogr('spatialite');
+
+SpatiaLiteFormat.prototype._contentType = "application/x-sqlite3; charset=utf-8";
+SpatiaLiteFormat.prototype._fileExtension = "sqlite";
+// As of GDAL 1.10.1 SRID detection is bogus, so we use
+// our own method. See:
+//  http://trac.osgeo.org/gdal/ticket/5131
+//  http://trac.osgeo.org/gdal/ticket/5287
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/110
+//  http://github.com/CartoDB/CartoDB-SQL-API/issues/116
+// Bug was fixed in GDAL 1.10.2
+SpatiaLiteFormat.prototype._needSRS = true;
+
+SpatiaLiteFormat.prototype.generate = function(options, callback) {
+    this.toOGR_SingleFile(options, 'SQLite', callback);
+    options.cmd_params = ['SPATIALITE=yes'];
+};
+
+module.exports = SpatiaLiteFormat;

app/models/formats/pg.js

@@ -0,0 +1,164 @@
+'use strict';
+
+var step = require('step');
+var PSQL = require('cartodb-psql');
+
+function PostgresFormat(id) {
+    this.id = id;
+}
+
+PostgresFormat.prototype = {
+
+  getQuery: function(sql/*, options*/) {
+    return sql;
+  },
+
+  getContentType: function(){
+    return this._contentType;
+  },
+
+  getFileExtension: function() {
+    return this.id;
+  }
+
+};
+
+PostgresFormat.prototype.handleQueryRow = function(row, result) {
+    result.addRow(row);
+};
+
+PostgresFormat.prototype.handleQueryRowWithSkipFields = function(row, result) {
+    var sf = this.opts.skipfields;
+    for ( var j=0; j<sf.length; ++j ) {
+        delete row[sf[j]];
+    }
+    this.handleQueryRow(row, result);
+};
+
+PostgresFormat.prototype.handleNotice = function(msg, result) {
+    if ( ! result.notices ) {
+        result.notices = [];
+    }
+    for (var i=0; i<msg.length; i++) {
+        result.notices.push(msg[i]);
+    }
+};
+
+PostgresFormat.prototype.handleQueryEnd = function(result) {
+  this.queryCanceller = undefined;
+
+  if ( this.error ) {
+    this.callback(this.error);
+    return;
+  }
+
+  if ( this.opts.profiler ) {
+      this.opts.profiler.done('gotRows');
+  }
+
+  this.opts.total_time = (Date.now() - this.start_time)/1000;
+
+  // Drop field description for skipped fields
+  if (this.hasSkipFields) {
+    var sf = this.opts.skipfields;
+    var newfields = [];
+    for ( var j=0; j<result.fields.length; ++j ) {
+      var f = result.fields[j];
+      if ( sf.indexOf(f.name) === -1 ) {
+          newfields.push(f);
+      }
+    }
+    result.fields = newfields;
+  }
+
+  var that = this;
+
+  step (
+    function packageResult() {
+      if ( that.opts.abortChecker ) {
+        that.opts.abortChecker('packageResult');
+      }
+      that.transform(result, that.opts, this);
+    },
+    function sendResults(err, out){
+        if (err) {
+            throw err;
+        }
+
+        // return to browser
+        if ( out ) {
+          if ( that.opts.beforeSink ) {
+              that.opts.beforeSink();
+          }
+          that.opts.sink.send(out);
+        } else {
+          console.error("No output from transform, doing nothing ?!");
+        }
+    },
+    function errorHandle(err){
+        that.callback(err);
+    }
+  );
+};
+
+PostgresFormat.prototype.sendResponse = function(opts, callback) {
+  if ( this.callback ) {
+    callback(new Error("Invalid double call to .sendResponse on a pg formatter"));
+    return;
+  }
+  this.callback = callback;
+  this.opts = opts;
+
+  this.hasSkipFields = opts.skipfields.length;
+
+  var sql = this.getQuery(opts.sql, {
+    gn: opts.gn,
+    dp: opts.dp,
+    skipfields: opts.skipfields
+  });
+
+  var that = this;
+
+  this.start_time = Date.now();
+
+  this.client = new PSQL(opts.dbopts);
+  this.client.eventedQuery(sql, function(err, query, queryCanceller) {
+      that.queryCanceller = queryCanceller;
+      if (err) {
+        callback(err);
+        return;
+      }
+      if ( that.opts.profiler ) {
+          that.opts.profiler.done('eventedQuery');
+      }
+
+      if (that.hasSkipFields) {
+          query.on('row', that.handleQueryRowWithSkipFields.bind(that));
+      } else {
+          query.on('row', that.handleQueryRow.bind(that));
+      }
+      query.on('end', that.handleQueryEnd.bind(that));
+      query.on('error', function(err) {
+          that.error = err;
+          if (err.message && err.message.match(/row too large, was \d* bytes/i)) {
+              return console.error(JSON.stringify({
+                  username: opts.username,
+                  type: 'row_size_limit_exceeded',
+                  error: err.message
+              }));
+          }
+          that.handleQueryEnd();
+      });
+      query.on('notice', function(msg) {
+          that.handleNotice(msg, query._result);
+      });
+  });
+};
+
+PostgresFormat.prototype.cancel = function() {
+    if (this.queryCanceller) {
+        this.queryCanceller.call();
+    }
+};
+
+module.exports = PostgresFormat;

app/models/formats/pg/arraybuffer.js

@@ -0,0 +1,87 @@
+'use strict';
+
+var _ = require('underscore');
+
+var pg  = require('./../pg');
+var ArrayBufferSer = require("../../bin_encoder");
+
+function BinaryFormat() {}
+
+BinaryFormat.prototype = new pg('arraybuffer');
+
+BinaryFormat.prototype._contentType = "application/octet-stream";
+
+BinaryFormat.prototype._extractTypeFromName = function(name) {
+  var g = name.match(/.*__(uintclamp|uint|int|float)(8|16|32)/i);
+  if(g && g.length === 3) {
+    var typeName = g[1] + g[2];
+    return ArrayBufferSer.typeNames[typeName];
+  }
+};
+
+// jshint maxcomplexity:12
+BinaryFormat.prototype.transform = function(result, options, callback) {
+  var total_rows = result.rowCount;
+  var rows = result.rows;
+
+  // get headers
+  if(!total_rows) {
+    callback(null, new Buffer(0));
+    return;
+  }
+
+  var headersNames = Object.keys(rows[0]);
+  var headerTypes = [];
+
+  if(_.contains(headersNames, 'the_geom')) {
+    callback(new Error("geometry types are not supported"), null);
+    return;
+  }
+
+  try {
+    var i;
+    var t;
+    // get header types (and guess from name)
+    for(i = 0; i < headersNames.length; ++i) {
+      r = rows[0];
+      n = headersNames[i];
+      if(typeof(r[n]) === 'string') {
+        headerTypes.push(ArrayBufferSer.STRING);
+      } else if(typeof(r[n]) === 'object') {
+        t = this._extractTypeFromName(n);
+        t = t || ArrayBufferSer.FLOAT32;
+        headerTypes.push(ArrayBufferSer.BUFFER + t);
+      } else {
+        t = this._extractTypeFromName(n);
+        headerTypes.push(t || ArrayBufferSer.FLOAT32);
+      }
+    }
+
+    // pack the data
+    var header = new ArrayBufferSer(ArrayBufferSer.STRING, headersNames);
+    var data = [header];
+    for(i = 0; i < headersNames.length; ++i) {
+      var d = [];
+      var n = headersNames[i];
+      for(var r = 0; r < total_rows; ++r) {
+        var row = rows[r][n];
+        if(headerTypes[i] > ArrayBufferSer.BUFFER) {
+          row = new ArrayBufferSer(headerTypes[i] - ArrayBufferSer.BUFFER, row);
+        }
+        d.push(row);
+      }
+      var b = new ArrayBufferSer(headerTypes[i], d);
+      data.push(b);
+    }
+
+    // create the final buffer
+    var all = new ArrayBufferSer(ArrayBufferSer.BUFFER, data);
+
+    callback(null, all.buffer);
+
+  } catch(e) {
+    callback(e, null);
+  }
+};
+
+module.exports = BinaryFormat;

app/models/formats/pg/geojson.js

@@ -0,0 +1,120 @@
+'use strict';
+
+var _ = require('underscore');
+
+var pg  = require('./../pg');
+const errorHandlerFactory = require('../../../services/error_handler_factory');
+
+function GeoJsonFormat() {
+    this.buffer = '';
+}
+
+GeoJsonFormat.prototype = new pg('geojson');
+
+GeoJsonFormat.prototype._contentType = "application/json; charset=utf-8";
+
+GeoJsonFormat.prototype.getQuery = function(sql, options) {
+    var gn = options.gn;
+    var dp = options.dp;
+    return 'SELECT *, ST_AsGeoJSON(' + gn + ',' + dp + ') as the_geom FROM (' + sql + ') as foo';
+};
+
+GeoJsonFormat.prototype.startStreaming = function() {
+    this.total_rows = 0;
+    if (this.opts.beforeSink) {
+        this.opts.beforeSink();
+    }
+    if (this.opts.callback) {
+        this.buffer += this.opts.callback + '(';
+    }
+    this.buffer += '{"type": "FeatureCollection", "features": [';
+    this._streamingStarted = true;
+};
+
+GeoJsonFormat.prototype.handleQueryRow = function(row) {
+
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    var geojson = [
+        '{',
+        '"type":"Feature",',
+        '"geometry":' + row[this.opts.gn] + ',',
+        '"properties":'
+    ];
+    delete row[this.opts.gn];
+    delete row.the_geom_webmercator;
+    geojson.push(JSON.stringify(row));
+    geojson.push('}');
+
+    this.buffer += (this.total_rows++ ? ',' : '') + geojson.join('');
+
+    if (this.total_rows % (this.opts.bufferedRows || 1000)) {
+        this.opts.sink.write(this.buffer);
+        this.buffer = '';
+    }
+};
+
+GeoJsonFormat.prototype.handleQueryEnd = function(/*result*/) {
+    if (this.error && !this._streamingStarted) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    this.buffer += ']'; // end of features
+
+    if (this.error) {
+        this.buffer += ',"error":' + JSON.stringify(errorHandlerFactory(this.error).getResponse().error);
+    }
+
+    this.buffer += '}'; // end of root object
+
+    if (this.opts.callback) {
+        this.buffer += ')';
+    }
+
+    this.opts.sink.write(this.buffer);
+    this.opts.sink.end();
+    this.buffer = '';
+
+    this.callback();
+};
+
+function _toGeoJSON(data, gn, callback){
+  try {
+    var out = {
+      type: "FeatureCollection",
+      features: []
+    };
+
+    _.each(data.rows, function(ele){
+      var _geojson = {
+          type: "Feature",
+          properties: { },
+          geometry: { }
+      };
+      _geojson.geometry = JSON.parse(ele[gn]);
+      delete ele[gn];
+      delete ele.the_geom_webmercator; // TODO: use skipfields
+      _geojson.properties = ele;
+      out.features.push(_geojson);
+    });
+
+    // return payload
+    callback(null, out);
+  } catch (err) {
+    callback(err,null);
+  }
+}
+
+module.exports = GeoJsonFormat;
+module.exports.toGeoJSON = _toGeoJSON;

app/models/formats/pg/json.js

@@ -0,0 +1,174 @@
+'use strict';
+
+var _ = require('underscore');
+
+var pg  = require('./../pg');
+const errorHandlerFactory = require('../../../services/error_handler_factory');
+
+function JsonFormat() {
+    this.buffer = '';
+    this.lastKnownResult = {};
+}
+
+JsonFormat.prototype = new pg('json');
+
+JsonFormat.prototype._contentType = "application/json; charset=utf-8";
+
+// jshint maxcomplexity:9
+JsonFormat.prototype.formatResultFields = function(flds) {
+  flds = flds || [];
+  var nfields = {};
+  for (var i=0; i<flds.length; ++i) {
+    var f = flds[i];
+    var cname = this.client.typeName(f.dataTypeID);
+    var tname;
+    if ( ! cname ) {
+      tname = 'unknown(' + f.dataTypeID + ')';
+    } else {
+      if ( cname.match('bool') ) {
+        tname = 'boolean';
+      }
+      else if ( cname.match(/int|float|numeric/) ) {
+        tname = 'number';
+      }
+      else if ( cname.match(/text|char|unknown/) ) {
+        tname = 'string';
+      }
+      else if ( cname.match(/date|time/) ) {
+        tname = 'date';
+      }
+      else {
+        tname = cname;
+      }
+      if ( tname && cname.match(/^_/) ) {
+        tname += '[]';
+      }
+    }
+    nfields[f.name] = { type: tname };
+  }
+  return nfields;
+};
+
+JsonFormat.prototype.startStreaming = function() {
+    this.total_rows = 0;
+    if (this.opts.beforeSink) {
+        this.opts.beforeSink();
+    }
+    if (this.opts.callback) {
+        this.buffer += this.opts.callback + '(';
+    }
+    this.buffer += '{"rows":[';
+    this._streamingStarted = true;
+};
+
+JsonFormat.prototype.handleQueryRow = function(row, result) {
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    this.lastKnownResult = result;
+
+    this.buffer += (this.total_rows++ ? ',' : '') + JSON.stringify(row, function (key, value) {
+        if (value !== value) {
+            return 'NaN';
+        }
+
+        if (value === Infinity) {
+            return 'Infinity';
+        }
+
+        if (value === -Infinity) {
+            return '-Infinity';
+        }
+
+        return value;
+    });
+
+    if (this.total_rows % (this.opts.bufferedRows || 1000)) {
+        this.opts.sink.write(this.buffer);
+        this.buffer = '';
+    }
+};
+
+// jshint maxcomplexity:13
+JsonFormat.prototype.handleQueryEnd = function(result) {
+    if (this.error && !this._streamingStarted) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    this.opts.total_time = (Date.now() - this.start_time)/1000;
+
+    result = result || this.lastKnownResult || {};
+
+    // Drop field description for skipped fields
+    if (this.hasSkipFields) {
+        var newfields = [];
+        var sf = this.opts.skipfields;
+        for (var i = 0; i < result.fields.length; i++) {
+            var f = result.fields[i];
+            if ( sf.indexOf(f.name) === -1 ) {
+                newfields.push(f);
+            }
+        }
+        result.fields = newfields;
+    }
+
+    var total_time = (Date.now() - this.start_time)/1000;
+
+    var out = [
+        '],', // end of "rows" array
+        '"time":', JSON.stringify(total_time),
+        ',"fields":', JSON.stringify(this.formatResultFields(result.fields)),
+        ',"total_rows":', JSON.stringify(result.rowCount || this.total_rows)
+    ];
+
+    if (this.error) {
+        out.push(',"error":', JSON.stringify(errorHandlerFactory(this.error).getResponse().error));
+    }
+
+
+    if ( result.notices && result.notices.length > 0 ) {
+        var notices = {},
+            severities = [];
+        _.each(result.notices, function(notice) {
+            var severity = notice.severity.toLowerCase() + 's';
+            if (!notices[severity]) {
+                severities.push(severity);
+                notices[severity] = [];
+            }
+            notices[severity].push(notice.message);
+        });
+        _.each(severities, function(severity) {
+            out.push(',');
+            out.push(JSON.stringify(severity));
+            out.push(':');
+            out.push(JSON.stringify(notices[severity]));
+        });
+    }
+
+    out.push('}');
+
+
+    this.buffer += out.join('');
+
+    if (this.opts.callback) {
+        this.buffer += ')';
+    }
+
+    this.opts.sink.write(this.buffer);
+    this.opts.sink.end();
+    this.buffer = '';
+
+    this.callback();
+};
+
+module.exports = JsonFormat;

app/models/formats/pg/svg.js

@@ -0,0 +1,166 @@
+'use strict';
+
+var pg  = require('./../pg');
+
+var svg_width  = 1024.0;
+var svg_height = 768.0;
+var svg_ratio = svg_width/svg_height;
+
+var radius = 5; // in pixels (based on svg_width and svg_height)
+
+var stroke_width = 1; // in pixels (based on svg_width and svg_height)
+var stroke_color = 'black';
+// fill settings affect polygons and points (circles)
+var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque
+// unused if fill_color='none'
+var fill_color = 'none'; // affects polygons and circles
+
+function SvgFormat() {
+    this.totalRows = 0;
+
+    this.bbox = null; // will be computed during the results scan
+    this.buffer = '';
+
+    this._streamingStarted = false;
+}
+
+SvgFormat.prototype = new pg('svg');
+SvgFormat.prototype._contentType = "image/svg+xml; charset=utf-8";
+
+SvgFormat.prototype.getQuery = function(sql, options) {
+  var gn = options.gn;
+  var dp = options.dp;
+  return  'WITH source AS ( ' + sql + '), extent AS ( ' +
+        ' SELECT ST_Extent(' + gn + ') AS e FROM source ' +
+        '), extent_info AS ( SELECT e, ' +
+        'st_xmin(e) as ex0, st_ymax(e) as ey0, ' +
+        'st_xmax(e)-st_xmin(e) as ew, ' +
+        'st_ymax(e)-st_ymin(e) as eh FROM extent )' +
+        ', trans AS ( SELECT CASE WHEN ' +
+        'eh = 0 THEN ' + svg_width +
+        '/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN ' +
+        svg_ratio + ' <= (ew / eh) THEN (' +
+        svg_width  + '/ew ) ELSE (' +
+        svg_height + '/eh ) END as s ' +
+        ', ex0 as x0, ey0 as y0 FROM extent_info ) ' +
+        'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as ' +
+        gn + '_box, ST_Dimension(' + gn + ') as ' + gn +
+        '_dimension, ST_AsSVG(ST_TransScale(' + gn + ', ' +
+        '-x0, -y0, s, s), 0, ' + dp + ') as ' + gn +
+        //+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY +
+        ' FROM trans, extent_info, source' +
+        ' ORDER BY the_geom_dimension ASC';
+};
+
+SvgFormat.prototype.startStreaming = function() {
+    if (this.opts.beforeSink) {
+        this.opts.beforeSink();
+    }
+
+    var header = [
+        '<?xml version="1.0" standalone="no"?>',
+        '<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">'
+    ];
+
+    var rootTag = '<svg ';
+    if ( this.bbox ) {
+        // expand box by "radius" + "stroke-width"
+        // TODO: use a Box2d class for these ops
+        var growby = radius + stroke_width;
+        this.bbox.xmin -= growby;
+        this.bbox.ymin -= growby;
+        this.bbox.xmax += growby;
+        this.bbox.ymax += growby;
+        this.bbox.width = this.bbox.xmax - this.bbox.xmin;
+        this.bbox.height = this.bbox.ymax - this.bbox.ymin;
+        rootTag += 'viewBox="' + this.bbox.xmin + ' ' + (-this.bbox.ymax) + ' ' +
+            this.bbox.width + ' ' + this.bbox.height + '" ';
+    }
+    rootTag += 'style="fill-opacity:' + fill_opacity + '; stroke:' + stroke_color + '; ' +
+        'stroke-width:' + stroke_width + '; fill:' + fill_color + '" ';
+    rootTag += 'xmlns="http://www.w3.org/2000/svg" version="1.1">\n';
+
+    header.push(rootTag);
+
+    this.opts.sink.write(header.join('\n'));
+
+    this._streamingStarted = true;
+};
+
+// jshint maxcomplexity:11
+SvgFormat.prototype.handleQueryRow = function(row) {
+    this.totalRows++;
+
+    if ( ! row.hasOwnProperty(this.opts.gn) ) {
+        this.error = new Error('column "' + this.opts.gn + '" does not exist');
+    }
+
+    var g = row[this.opts.gn];
+    if ( ! g ) {
+        return;
+    } // null or empty
+
+    // jshint ignore:start
+    var gdims = row[this.opts.gn + '_dimension'];
+    // TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found
+    // TODO: add "class" attribute to help with styling ?
+    if ( gdims == '0' ) {
+        this.buffer += '<circle r="' + radius + '" ' + g + ' />\n';
+    } else if ( gdims == '1' ) {
+        // Avoid filling closed linestrings
+        this.buffer += '<path ' + ( fill_color !== 'none' ? 'fill="none" ' : '' ) + 'd="' + g + '" />\n';
+    } else if ( gdims == '2' ) {
+        this.buffer += '<path d="' + g + '" />\n';
+    }
+    // jshint ignore:end
+
+    if ( ! this.bbox ) {
+        // Parse layer extent: "BOX(x y, X Y)"
+        // NOTE: the name of the extent field is
+        //       determined by the same code adding the
+        //       ST_AsSVG call (in queryResult)
+        //
+        var bbox = row[this.opts.gn + '_box'];
+        bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
+        this.bbox = {
+            xmin: parseFloat(bbox[1]),
+            ymin: parseFloat(bbox[2]),
+            xmax: parseFloat(bbox[3]),
+            ymax: parseFloat(bbox[4])
+        };
+    }
+
+    if (!this._streamingStarted && this.bbox) {
+        this.startStreaming();
+    }
+
+    if (this._streamingStarted && (this.totalRows % (this.opts.bufferedRows || 1000))) {
+        this.opts.sink.write(this.buffer);
+        this.buffer = '';
+    }
+};
+
+SvgFormat.prototype.handleQueryEnd = function() {
+    if ( this.error && !this._streamingStarted) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    if (!this._streamingStarted) {
+        this.startStreaming();
+    }
+
+    // rootTag close
+    this.buffer += '</svg>\n';
+
+    this.opts.sink.write(this.buffer);
+    this.opts.sink.end();
+
+    this.callback();
+};
+
+module.exports = SvgFormat;

app/models/formats/pg/topojson.js

@@ -0,0 +1,138 @@
+'use strict';
+
+var pg  = require('./../pg');
+var _ = require('underscore');
+var geojson = require('./geojson');
+var TopoJSON = require('topojson');
+
+function TopoJsonFormat() {
+    this.features = [];
+}
+
+TopoJsonFormat.prototype = new pg('topojson');
+
+TopoJsonFormat.prototype.getQuery = function(sql, options) {
+  return geojson.prototype.getQuery(sql, options) + ' where ' + options.gn + ' is not null';
+};
+
+TopoJsonFormat.prototype.handleQueryRow = function(row) {
+    var _geojson = {
+        type: "Feature"
+    };
+    _geojson.geometry = JSON.parse(row[this.opts.gn]);
+    delete row[this.opts.gn];
+    delete row.the_geom_webmercator;
+    _geojson.properties = row;
+    this.features.push(_geojson);
+};
+
+TopoJsonFormat.prototype.handleQueryEnd = function() {
+    if (this.error) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    var topology = TopoJSON.topology(this.features, {
+        "quantization": 1e4,
+        "force-clockwise": true,
+        "property-filter": function(d) {
+            return d;
+        }
+    });
+
+    this.features = [];
+
+    var stream = this.opts.sink;
+    var jsonpCallback = this.opts.callback;
+    var bufferedRows = this.opts.bufferedRows;
+    var buffer = '';
+
+    var immediately = global.setImmediate || process.nextTick;
+
+    function streamObjectSubtree(obj, key, done) {
+        buffer += '"' + key + '":';
+
+        var isObject = _.isObject(obj[key]),
+            isArray = _.isArray(obj[key]),
+            isIterable = isArray || isObject;
+
+        if (isIterable) {
+            buffer += isArray ? '[' : '{';
+            var subtreeKeys = Object.keys(obj[key]);
+            var pos = 0;
+            function streamNext() {
+                immediately(function() {
+                    var subtreeKey = subtreeKeys.shift();
+                    if (!isArray) {
+                        buffer += '"' + subtreeKey + '":';
+                    }
+                    buffer += JSON.stringify(obj[key][subtreeKey]);
+
+                    if (pos++ % (bufferedRows || 1000)) {
+                        stream.write(buffer);
+                        buffer = '';
+                    }
+
+                    if (subtreeKeys.length > 0) {
+                        delete obj[key][subtreeKey];
+                        buffer += ',';
+                        streamNext();
+                    } else {
+                        buffer += isArray ? ']' : '}';
+                        stream.write(buffer);
+                        buffer = '';
+                        done();
+                    }
+                });
+            }
+            streamNext();
+        } else {
+            buffer += JSON.stringify(obj[key]);
+            done();
+        }
+    }
+
+    if (jsonpCallback) {
+        buffer += jsonpCallback + '(';
+    }
+    buffer += '{';
+    var keys = Object.keys(topology);
+    function sendResponse() {
+        immediately(function () {
+            var key = keys.shift();
+            function done() {
+                if (keys.length > 0) {
+                    delete topology[key];
+                    buffer += ',';
+                    sendResponse();
+                } else {
+                    buffer += '}';
+                    if (jsonpCallback) {
+                        buffer += ')';
+                    }
+                    stream.write(buffer);
+                    stream.end();
+                    topology = null;
+                }
+            }
+            streamObjectSubtree(topology, key, done);
+        });
+    }
+    sendResponse();
+
+    this.callback();
+};
+
+TopoJsonFormat.prototype.cancel = function() {
+    if (this.queryCanceller) {
+        this.queryCanceller.call();
+    }
+};
+
+
+
+module.exports = TopoJsonFormat;