first commit

app/models/formats/pg/arraybuffer.js

@@ -0,0 +1,87 @@
+'use strict';
+
+var _ = require('underscore');
+
+var pg  = require('./../pg');
+var ArrayBufferSer = require("../../bin_encoder");
+
+function BinaryFormat() {}
+
+BinaryFormat.prototype = new pg('arraybuffer');
+
+BinaryFormat.prototype._contentType = "application/octet-stream";
+
+BinaryFormat.prototype._extractTypeFromName = function(name) {
+  var g = name.match(/.*__(uintclamp|uint|int|float)(8|16|32)/i);
+  if(g && g.length === 3) {
+    var typeName = g[1] + g[2];
+    return ArrayBufferSer.typeNames[typeName];
+  }
+};
+
+// jshint maxcomplexity:12
+BinaryFormat.prototype.transform = function(result, options, callback) {
+  var total_rows = result.rowCount;
+  var rows = result.rows;
+
+  // get headers
+  if(!total_rows) {
+    callback(null, new Buffer(0));
+    return;
+  }
+
+  var headersNames = Object.keys(rows[0]);
+  var headerTypes = [];
+
+  if(_.contains(headersNames, 'the_geom')) {
+    callback(new Error("geometry types are not supported"), null);
+    return;
+  }
+
+  try {
+    var i;
+    var t;
+    // get header types (and guess from name)
+    for(i = 0; i < headersNames.length; ++i) {
+      r = rows[0];
+      n = headersNames[i];
+      if(typeof(r[n]) === 'string') {
+        headerTypes.push(ArrayBufferSer.STRING);
+      } else if(typeof(r[n]) === 'object') {
+        t = this._extractTypeFromName(n);
+        t = t || ArrayBufferSer.FLOAT32;
+        headerTypes.push(ArrayBufferSer.BUFFER + t);
+      } else {
+        t = this._extractTypeFromName(n);
+        headerTypes.push(t || ArrayBufferSer.FLOAT32);
+      }
+    }
+
+    // pack the data
+    var header = new ArrayBufferSer(ArrayBufferSer.STRING, headersNames);
+    var data = [header];
+    for(i = 0; i < headersNames.length; ++i) {
+      var d = [];
+      var n = headersNames[i];
+      for(var r = 0; r < total_rows; ++r) {
+        var row = rows[r][n];
+        if(headerTypes[i] > ArrayBufferSer.BUFFER) {
+          row = new ArrayBufferSer(headerTypes[i] - ArrayBufferSer.BUFFER, row);
+        }
+        d.push(row);
+      }
+      var b = new ArrayBufferSer(headerTypes[i], d);
+      data.push(b);
+    }
+
+    // create the final buffer
+    var all = new ArrayBufferSer(ArrayBufferSer.BUFFER, data);
+
+    callback(null, all.buffer);
+
+  } catch(e) {
+    callback(e, null);
+  }
+};
+
+module.exports = BinaryFormat;

app/models/formats/pg/geojson.js

@@ -0,0 +1,120 @@
+'use strict';
+
+var _ = require('underscore');
+
+var pg  = require('./../pg');
+const errorHandlerFactory = require('../../../services/error_handler_factory');
+
+function GeoJsonFormat() {
+    this.buffer = '';
+}
+
+GeoJsonFormat.prototype = new pg('geojson');
+
+GeoJsonFormat.prototype._contentType = "application/json; charset=utf-8";
+
+GeoJsonFormat.prototype.getQuery = function(sql, options) {
+    var gn = options.gn;
+    var dp = options.dp;
+    return 'SELECT *, ST_AsGeoJSON(' + gn + ',' + dp + ') as the_geom FROM (' + sql + ') as foo';
+};
+
+GeoJsonFormat.prototype.startStreaming = function() {
+    this.total_rows = 0;
+    if (this.opts.beforeSink) {
+        this.opts.beforeSink();
+    }
+    if (this.opts.callback) {
+        this.buffer += this.opts.callback + '(';
+    }
+    this.buffer += '{"type": "FeatureCollection", "features": [';
+    this._streamingStarted = true;
+};
+
+GeoJsonFormat.prototype.handleQueryRow = function(row) {
+
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    var geojson = [
+        '{',
+        '"type":"Feature",',
+        '"geometry":' + row[this.opts.gn] + ',',
+        '"properties":'
+    ];
+    delete row[this.opts.gn];
+    delete row.the_geom_webmercator;
+    geojson.push(JSON.stringify(row));
+    geojson.push('}');
+
+    this.buffer += (this.total_rows++ ? ',' : '') + geojson.join('');
+
+    if (this.total_rows % (this.opts.bufferedRows || 1000)) {
+        this.opts.sink.write(this.buffer);
+        this.buffer = '';
+    }
+};
+
+GeoJsonFormat.prototype.handleQueryEnd = function(/*result*/) {
+    if (this.error && !this._streamingStarted) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    this.buffer += ']'; // end of features
+
+    if (this.error) {
+        this.buffer += ',"error":' + JSON.stringify(errorHandlerFactory(this.error).getResponse().error);
+    }
+
+    this.buffer += '}'; // end of root object
+
+    if (this.opts.callback) {
+        this.buffer += ')';
+    }
+
+    this.opts.sink.write(this.buffer);
+    this.opts.sink.end();
+    this.buffer = '';
+
+    this.callback();
+};
+
+function _toGeoJSON(data, gn, callback){
+  try {
+    var out = {
+      type: "FeatureCollection",
+      features: []
+    };
+
+    _.each(data.rows, function(ele){
+      var _geojson = {
+          type: "Feature",
+          properties: { },
+          geometry: { }
+      };
+      _geojson.geometry = JSON.parse(ele[gn]);
+      delete ele[gn];
+      delete ele.the_geom_webmercator; // TODO: use skipfields
+      _geojson.properties = ele;
+      out.features.push(_geojson);
+    });
+
+    // return payload
+    callback(null, out);
+  } catch (err) {
+    callback(err,null);
+  }
+}
+
+module.exports = GeoJsonFormat;
+module.exports.toGeoJSON = _toGeoJSON;

app/models/formats/pg/json.js

@@ -0,0 +1,174 @@
+'use strict';
+
+var _ = require('underscore');
+
+var pg  = require('./../pg');
+const errorHandlerFactory = require('../../../services/error_handler_factory');
+
+function JsonFormat() {
+    this.buffer = '';
+    this.lastKnownResult = {};
+}
+
+JsonFormat.prototype = new pg('json');
+
+JsonFormat.prototype._contentType = "application/json; charset=utf-8";
+
+// jshint maxcomplexity:9
+JsonFormat.prototype.formatResultFields = function(flds) {
+  flds = flds || [];
+  var nfields = {};
+  for (var i=0; i<flds.length; ++i) {
+    var f = flds[i];
+    var cname = this.client.typeName(f.dataTypeID);
+    var tname;
+    if ( ! cname ) {
+      tname = 'unknown(' + f.dataTypeID + ')';
+    } else {
+      if ( cname.match('bool') ) {
+        tname = 'boolean';
+      }
+      else if ( cname.match(/int|float|numeric/) ) {
+        tname = 'number';
+      }
+      else if ( cname.match(/text|char|unknown/) ) {
+        tname = 'string';
+      }
+      else if ( cname.match(/date|time/) ) {
+        tname = 'date';
+      }
+      else {
+        tname = cname;
+      }
+      if ( tname && cname.match(/^_/) ) {
+        tname += '[]';
+      }
+    }
+    nfields[f.name] = { type: tname };
+  }
+  return nfields;
+};
+
+JsonFormat.prototype.startStreaming = function() {
+    this.total_rows = 0;
+    if (this.opts.beforeSink) {
+        this.opts.beforeSink();
+    }
+    if (this.opts.callback) {
+        this.buffer += this.opts.callback + '(';
+    }
+    this.buffer += '{"rows":[';
+    this._streamingStarted = true;
+};
+
+JsonFormat.prototype.handleQueryRow = function(row, result) {
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    this.lastKnownResult = result;
+
+    this.buffer += (this.total_rows++ ? ',' : '') + JSON.stringify(row, function (key, value) {
+        if (value !== value) {
+            return 'NaN';
+        }
+
+        if (value === Infinity) {
+            return 'Infinity';
+        }
+
+        if (value === -Infinity) {
+            return '-Infinity';
+        }
+
+        return value;
+    });
+
+    if (this.total_rows % (this.opts.bufferedRows || 1000)) {
+        this.opts.sink.write(this.buffer);
+        this.buffer = '';
+    }
+};
+
+// jshint maxcomplexity:13
+JsonFormat.prototype.handleQueryEnd = function(result) {
+    if (this.error && !this._streamingStarted) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    if ( ! this._streamingStarted ) {
+        this.startStreaming();
+    }
+
+    this.opts.total_time = (Date.now() - this.start_time)/1000;
+
+    result = result || this.lastKnownResult || {};
+
+    // Drop field description for skipped fields
+    if (this.hasSkipFields) {
+        var newfields = [];
+        var sf = this.opts.skipfields;
+        for (var i = 0; i < result.fields.length; i++) {
+            var f = result.fields[i];
+            if ( sf.indexOf(f.name) === -1 ) {
+                newfields.push(f);
+            }
+        }
+        result.fields = newfields;
+    }
+
+    var total_time = (Date.now() - this.start_time)/1000;
+
+    var out = [
+        '],', // end of "rows" array
+        '"time":', JSON.stringify(total_time),
+        ',"fields":', JSON.stringify(this.formatResultFields(result.fields)),
+        ',"total_rows":', JSON.stringify(result.rowCount || this.total_rows)
+    ];
+
+    if (this.error) {
+        out.push(',"error":', JSON.stringify(errorHandlerFactory(this.error).getResponse().error));
+    }
+
+
+    if ( result.notices && result.notices.length > 0 ) {
+        var notices = {},
+            severities = [];
+        _.each(result.notices, function(notice) {
+            var severity = notice.severity.toLowerCase() + 's';
+            if (!notices[severity]) {
+                severities.push(severity);
+                notices[severity] = [];
+            }
+            notices[severity].push(notice.message);
+        });
+        _.each(severities, function(severity) {
+            out.push(',');
+            out.push(JSON.stringify(severity));
+            out.push(':');
+            out.push(JSON.stringify(notices[severity]));
+        });
+    }
+
+    out.push('}');
+
+
+    this.buffer += out.join('');
+
+    if (this.opts.callback) {
+        this.buffer += ')';
+    }
+
+    this.opts.sink.write(this.buffer);
+    this.opts.sink.end();
+    this.buffer = '';
+
+    this.callback();
+};
+
+module.exports = JsonFormat;

app/models/formats/pg/svg.js

@@ -0,0 +1,166 @@
+'use strict';
+
+var pg  = require('./../pg');
+
+var svg_width  = 1024.0;
+var svg_height = 768.0;
+var svg_ratio = svg_width/svg_height;
+
+var radius = 5; // in pixels (based on svg_width and svg_height)
+
+var stroke_width = 1; // in pixels (based on svg_width and svg_height)
+var stroke_color = 'black';
+// fill settings affect polygons and points (circles)
+var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque
+// unused if fill_color='none'
+var fill_color = 'none'; // affects polygons and circles
+
+function SvgFormat() {
+    this.totalRows = 0;
+
+    this.bbox = null; // will be computed during the results scan
+    this.buffer = '';
+
+    this._streamingStarted = false;
+}
+
+SvgFormat.prototype = new pg('svg');
+SvgFormat.prototype._contentType = "image/svg+xml; charset=utf-8";
+
+SvgFormat.prototype.getQuery = function(sql, options) {
+  var gn = options.gn;
+  var dp = options.dp;
+  return  'WITH source AS ( ' + sql + '), extent AS ( ' +
+        ' SELECT ST_Extent(' + gn + ') AS e FROM source ' +
+        '), extent_info AS ( SELECT e, ' +
+        'st_xmin(e) as ex0, st_ymax(e) as ey0, ' +
+        'st_xmax(e)-st_xmin(e) as ew, ' +
+        'st_ymax(e)-st_ymin(e) as eh FROM extent )' +
+        ', trans AS ( SELECT CASE WHEN ' +
+        'eh = 0 THEN ' + svg_width +
+        '/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN ' +
+        svg_ratio + ' <= (ew / eh) THEN (' +
+        svg_width  + '/ew ) ELSE (' +
+        svg_height + '/eh ) END as s ' +
+        ', ex0 as x0, ey0 as y0 FROM extent_info ) ' +
+        'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as ' +
+        gn + '_box, ST_Dimension(' + gn + ') as ' + gn +
+        '_dimension, ST_AsSVG(ST_TransScale(' + gn + ', ' +
+        '-x0, -y0, s, s), 0, ' + dp + ') as ' + gn +
+        //+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY +
+        ' FROM trans, extent_info, source' +
+        ' ORDER BY the_geom_dimension ASC';
+};
+
+SvgFormat.prototype.startStreaming = function() {
+    if (this.opts.beforeSink) {
+        this.opts.beforeSink();
+    }
+
+    var header = [
+        '<?xml version="1.0" standalone="no"?>',
+        '<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">'
+    ];
+
+    var rootTag = '<svg ';
+    if ( this.bbox ) {
+        // expand box by "radius" + "stroke-width"
+        // TODO: use a Box2d class for these ops
+        var growby = radius + stroke_width;
+        this.bbox.xmin -= growby;
+        this.bbox.ymin -= growby;
+        this.bbox.xmax += growby;
+        this.bbox.ymax += growby;
+        this.bbox.width = this.bbox.xmax - this.bbox.xmin;
+        this.bbox.height = this.bbox.ymax - this.bbox.ymin;
+        rootTag += 'viewBox="' + this.bbox.xmin + ' ' + (-this.bbox.ymax) + ' ' +
+            this.bbox.width + ' ' + this.bbox.height + '" ';
+    }
+    rootTag += 'style="fill-opacity:' + fill_opacity + '; stroke:' + stroke_color + '; ' +
+        'stroke-width:' + stroke_width + '; fill:' + fill_color + '" ';
+    rootTag += 'xmlns="http://www.w3.org/2000/svg" version="1.1">\n';
+
+    header.push(rootTag);
+
+    this.opts.sink.write(header.join('\n'));
+
+    this._streamingStarted = true;
+};
+
+// jshint maxcomplexity:11
+SvgFormat.prototype.handleQueryRow = function(row) {
+    this.totalRows++;
+
+    if ( ! row.hasOwnProperty(this.opts.gn) ) {
+        this.error = new Error('column "' + this.opts.gn + '" does not exist');
+    }
+
+    var g = row[this.opts.gn];
+    if ( ! g ) {
+        return;
+    } // null or empty
+
+    // jshint ignore:start
+    var gdims = row[this.opts.gn + '_dimension'];
+    // TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found
+    // TODO: add "class" attribute to help with styling ?
+    if ( gdims == '0' ) {
+        this.buffer += '<circle r="' + radius + '" ' + g + ' />\n';
+    } else if ( gdims == '1' ) {
+        // Avoid filling closed linestrings
+        this.buffer += '<path ' + ( fill_color !== 'none' ? 'fill="none" ' : '' ) + 'd="' + g + '" />\n';
+    } else if ( gdims == '2' ) {
+        this.buffer += '<path d="' + g + '" />\n';
+    }
+    // jshint ignore:end
+
+    if ( ! this.bbox ) {
+        // Parse layer extent: "BOX(x y, X Y)"
+        // NOTE: the name of the extent field is
+        //       determined by the same code adding the
+        //       ST_AsSVG call (in queryResult)
+        //
+        var bbox = row[this.opts.gn + '_box'];
+        bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
+        this.bbox = {
+            xmin: parseFloat(bbox[1]),
+            ymin: parseFloat(bbox[2]),
+            xmax: parseFloat(bbox[3]),
+            ymax: parseFloat(bbox[4])
+        };
+    }
+
+    if (!this._streamingStarted && this.bbox) {
+        this.startStreaming();
+    }
+
+    if (this._streamingStarted && (this.totalRows % (this.opts.bufferedRows || 1000))) {
+        this.opts.sink.write(this.buffer);
+        this.buffer = '';
+    }
+};
+
+SvgFormat.prototype.handleQueryEnd = function() {
+    if ( this.error && !this._streamingStarted) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    if (!this._streamingStarted) {
+        this.startStreaming();
+    }
+
+    // rootTag close
+    this.buffer += '</svg>\n';
+
+    this.opts.sink.write(this.buffer);
+    this.opts.sink.end();
+
+    this.callback();
+};
+
+module.exports = SvgFormat;

app/models/formats/pg/topojson.js

@@ -0,0 +1,138 @@
+'use strict';
+
+var pg  = require('./../pg');
+var _ = require('underscore');
+var geojson = require('./geojson');
+var TopoJSON = require('topojson');
+
+function TopoJsonFormat() {
+    this.features = [];
+}
+
+TopoJsonFormat.prototype = new pg('topojson');
+
+TopoJsonFormat.prototype.getQuery = function(sql, options) {
+  return geojson.prototype.getQuery(sql, options) + ' where ' + options.gn + ' is not null';
+};
+
+TopoJsonFormat.prototype.handleQueryRow = function(row) {
+    var _geojson = {
+        type: "Feature"
+    };
+    _geojson.geometry = JSON.parse(row[this.opts.gn]);
+    delete row[this.opts.gn];
+    delete row.the_geom_webmercator;
+    _geojson.properties = row;
+    this.features.push(_geojson);
+};
+
+TopoJsonFormat.prototype.handleQueryEnd = function() {
+    if (this.error) {
+        this.callback(this.error);
+        return;
+    }
+
+    if ( this.opts.profiler ) {
+        this.opts.profiler.done('gotRows');
+    }
+
+    var topology = TopoJSON.topology(this.features, {
+        "quantization": 1e4,
+        "force-clockwise": true,
+        "property-filter": function(d) {
+            return d;
+        }
+    });
+
+    this.features = [];
+
+    var stream = this.opts.sink;
+    var jsonpCallback = this.opts.callback;
+    var bufferedRows = this.opts.bufferedRows;
+    var buffer = '';
+
+    var immediately = global.setImmediate || process.nextTick;
+
+    function streamObjectSubtree(obj, key, done) {
+        buffer += '"' + key + '":';
+
+        var isObject = _.isObject(obj[key]),
+            isArray = _.isArray(obj[key]),
+            isIterable = isArray || isObject;
+
+        if (isIterable) {
+            buffer += isArray ? '[' : '{';
+            var subtreeKeys = Object.keys(obj[key]);
+            var pos = 0;
+            function streamNext() {
+                immediately(function() {
+                    var subtreeKey = subtreeKeys.shift();
+                    if (!isArray) {
+                        buffer += '"' + subtreeKey + '":';
+                    }
+                    buffer += JSON.stringify(obj[key][subtreeKey]);
+
+                    if (pos++ % (bufferedRows || 1000)) {
+                        stream.write(buffer);
+                        buffer = '';
+                    }
+
+                    if (subtreeKeys.length > 0) {
+                        delete obj[key][subtreeKey];
+                        buffer += ',';
+                        streamNext();
+                    } else {
+                        buffer += isArray ? ']' : '}';
+                        stream.write(buffer);
+                        buffer = '';
+                        done();
+                    }
+                });
+            }
+            streamNext();
+        } else {
+            buffer += JSON.stringify(obj[key]);
+            done();
+        }
+    }
+
+    if (jsonpCallback) {
+        buffer += jsonpCallback + '(';
+    }
+    buffer += '{';
+    var keys = Object.keys(topology);
+    function sendResponse() {
+        immediately(function () {
+            var key = keys.shift();
+            function done() {
+                if (keys.length > 0) {
+                    delete topology[key];
+                    buffer += ',';
+                    sendResponse();
+                } else {
+                    buffer += '}';
+                    if (jsonpCallback) {
+                        buffer += ')';
+                    }
+                    stream.write(buffer);
+                    stream.end();
+                    topology = null;
+                }
+            }
+            streamObjectSubtree(topology, key, done);
+        });
+    }
+    sendResponse();
+
+    this.callback();
+};
+
+TopoJsonFormat.prototype.cancel = function() {
+    if (this.queryCanceller) {
+        this.queryCanceller.call();
+    }
+};
+
+
+
+module.exports = TopoJsonFormat;