Bump version

Test: `end` event should not be triggered 2 times on copy-from

.travis.yml

@@ -1,7 +1,16 @@
 language: node_js
 node_js:
-  - "0.10"
-  - "0.11"
+  - "0.12"
+  - "4"
+  - "5"
+  - "6"
+
+addons:
+  postgresql: "9.2"
+
+services:
+  - postgresql
+
 before_install:
   - npm install npm --global
 env:

copy-to.js

@@ -4,12 +4,12 @@ module.exports = function(txt, options) {
 
 var Transform = require('stream').Transform
 var util = require('util')
+var code = require('./message-formats')
 
 var CopyStreamQuery = function(text, options) {
   Transform.call(this, options)
   this.text = text
-  this._listeners = {}
-  this._copyOutResponse = null
+  this._gotCopyOutResponse = false
   this.rowCount = 0
 }
 
@@ -20,81 +20,79 @@ var eventTypes = ['close', 'data', 'end', 'error']
 CopyStreamQuery.prototype.submit = function(connection) {
   connection.query(this.text)
   this.connection = connection
-  var self = this
-  eventTypes.forEach(function(type) {
-    self._listeners[type] = connection.stream.listeners(type)
-    connection.stream.removeAllListeners(type)
-  })
+  this.connection.removeAllListeners('copyData')
   connection.stream.pipe(this)
 }
 
-var code = {
-  E: 69, //Error
-  H: 72, //CopyOutResponse
-  d: 0x64, //CopyData
-  c: 0x63 //CopyDone
-}
-
 CopyStreamQuery.prototype._detach = function() {
-  this.connection.stream.unpipe()
-  var self = this
-  eventTypes.forEach(function(type) {
-    self.connection.stream.removeAllListeners(type)
-    self._listeners[type].forEach(function(listener) {
-      self.connection.stream.on(type, listener)
-    })
-  })
+  this.connection.stream.unpipe(this)
+  // Unpipe can drop us out of flowing mode
+  this.connection.stream.resume()
 }
 
 CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
   var offset = 0
+  var Byte1Len = 1;
+  var Int32Len = 4;
   if(this._remainder && chunk) {
     chunk = Buffer.concat([this._remainder, chunk])
   }
-  if(!this._copyOutResponse) {
-    this._copyOutResponse = true
-    if(chunk[0] == code.E) {
-      this._detach()
-      this.connection.stream.unshift(chunk)
-      this.push(null)
-      return cb();
-    }
-    if(chunk[0] != code.H) {
-      this.emit('error', new Error('Expected copy out response'))
-    }
-    var length = chunk.readUInt32BE(1)
-    offset = 1
-    offset += length
-  }
-  while((chunk.length - offset) > 5) {
+
+  var length;
+  var messageCode;
+  var needPush = false;
+
+  while((chunk.length - offset) >= (Byte1Len + Int32Len)) {
     var messageCode = chunk[offset]
-    //complete or error
-    if(messageCode == code.c || messageCode == code.E) {
-      this._detach()
-      if (messageCode == code.c) {
-        this.connection.stream.unshift(chunk.slice(offset + 5))
-      } else {
-        this.connection.stream.unshift(chunk.slice(offset))
+
+    //console.log('PostgreSQL message ' + String.fromCharCode(messageCode))
+    switch(messageCode) {
+
+      // detect COPY start
+      case code.CopyOutResponse:
+        if (!this._gotCopyOutResponse) {
+          this._gotCopyOutResponse = true
+        } else {
+          this.emit('error', new Error('Unexpected CopyOutResponse message (H)'))
+        }
+        break;
+
+      // meaningful row
+      case code.CopyData:
+        needPush = true;
+        break;
+
+      // standard interspersed messages. discard
+      case code.ParameterStatus:
+      case code.NoticeResponse:
+      case code.NotificationResponse:
+        break;
+  
+      case code.ErrorResponse:
+      case code.CopyDone:
+        this._detach()
+        this.push(null)
+        return cb();
+        break;
+      default:
+        this.emit('error', new Error('Unexpected PostgreSQL message ' + String.fromCharCode(messageCode)))
+    }
+
+    length = chunk.readUInt32BE(offset+Byte1Len)
+    if(chunk.length >= (offset + Byte1Len + length)) {
+      offset += Byte1Len + Int32Len
+      if (needPush) {
+        var row = chunk.slice(offset, offset + length - Int32Len)
+        this.rowCount++
+        this.push(row)
       }
-      this.push(null)
-      return cb();
-    }
-    //something bad happened
-    if(messageCode != code.d) {
-      return this.emit('error', new Error('expected "d" (copydata message)'))
-    }
-    var length = chunk.readUInt32BE(offset + 1) - 4 //subtract length of UInt32
-    //can we read the next row?
-    if(chunk.length > (offset + length + 5)) {
-      offset += 5
-      var slice = chunk.slice(offset, offset + length)
-      offset += length
-      this.push(slice)
-      this.rowCount++
+      offset += (length - Int32Len)
     } else {
+      // we need more chunks for a complete message
       break;
     }
   }
+
   if(chunk.length - offset) {
     var slice = chunk.slice(offset)
     this._remainder = slice
@@ -108,6 +106,9 @@ CopyStreamQuery.prototype.handleError = function(e) {
   this.emit('error', e)
 }
 
+CopyStreamQuery.prototype.handleCopyData = function(chunk) {
+}
+
 CopyStreamQuery.prototype.handleCommandComplete = function() {
 }
 

index.js

@@ -1,5 +1,4 @@
 var CopyToQueryStream = require('./copy-to')
-
 module.exports = {
   to: function(txt, options) {
     return new CopyToQueryStream(txt, options)
@@ -11,6 +10,7 @@ module.exports = {
 
 var Transform = require('stream').Transform
 var util = require('util')
+var code = require('./message-formats')
 
 var CopyStreamQuery = function(text, options) {
   Transform.call(this, options)
@@ -27,28 +27,23 @@ CopyStreamQuery.prototype.submit = function(connection) {
   connection.query(this.text)
 }
 
-var code = {
-  H: 72, //CopyOutResponse
-  d: 0x64, //CopyData
-  c: 0x63 //CopyDone
-}
 
-var copyDataBuffer = Buffer([code.d])
+var copyDataBuffer = Buffer([code.CopyData])
 CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
+  var Int32Len = 4;
   this.push(copyDataBuffer)
-  var lenBuffer = Buffer(4)
-  lenBuffer.writeUInt32BE(chunk.length + 4, 0)
+  var lenBuffer = Buffer(Int32Len)
+  lenBuffer.writeUInt32BE(chunk.length + Int32Len, 0)
   this.push(lenBuffer)
   this.push(chunk)
-  this.rowCount++
   cb()
 }
 
 CopyStreamQuery.prototype._flush = function(cb) {
-  var finBuffer = Buffer([code.c, 0, 0, 0, 4])
+  var Int32Len = 4;
+  var finBuffer = Buffer([code.CopyDone, 0, 0, 0, Int32Len])
   this.push(finBuffer)
-  //never call this callback, do not close underlying stream
-  //cb()
+  this.cb_flush = cb
 }
 
 CopyStreamQuery.prototype.handleError = function(e) {
@@ -56,12 +51,24 @@ CopyStreamQuery.prototype.handleError = function(e) {
 }
 
 CopyStreamQuery.prototype.handleCopyInResponse = function(connection) {
-  this.pipe(connection.stream)
+  this.pipe(connection.stream, { end: false })
 }
 
-CopyStreamQuery.prototype.handleCommandComplete = function() {
-  this.unpipe()
-  this.emit('end')
+CopyStreamQuery.prototype.handleCommandComplete = function(msg) {
+  // Parse affected row count as in
+  // https://github.com/brianc/node-postgres/blob/35e5567f86774f808c2a8518dd312b8aa3586693/lib/result.js#L37
+  var match = /COPY (\d+)/.exec((msg || {}).text)
+  if (match) {
+    this.rowCount = parseInt(match[1], 10)
+  }
+
+  // we delay the _flush cb so that the 'end' event is
+  // triggered after CommandComplete
+  this.cb_flush()
+
+  // unpipe from connection
+  this.unpipe(this.connection)
+  this.connection = null
 }
 
 CopyStreamQuery.prototype.handleReadyForQuery = function() {

message-formats.js

@@ -0,0 +1,25 @@
+/**
+ * The COPY feature uses the following protocol codes.
+ * The codes for the most recent protocol version are documented on
+ * https://www.postgresql.org/docs/current/static/protocol-message-formats.html
+ * 
+ * The protocol flow itself is described on
+ * https://www.postgresql.org/docs/current/static/protocol-flow.html
+ */
+module.exports = {
+  ErrorResponse:        0x45, // E
+  CopyInResponse:       0x47, // G
+  CopyOutResponse:      0x48, // H
+  CopyBothResponse:     0x57, // W
+  CopyDone:             0x63, // c
+  CopyData:             0x64, // d
+  CopyFail:             0x66, // f
+
+  // It is possible for NoticeResponse and ParameterStatus messages to be interspersed between CopyData messages;
+  // frontends must handle these cases, and should be prepared for other asynchronous message types as well
+  // (see Section 50.2.6).
+  // Otherwise, any message type other than CopyData or CopyDone may be treated as terminating copy-out mode.
+  NotificationResponse: 0x41, // A
+  NoticeResponse:       0x4E, // N
+  ParameterStatus:      0x53  // S
+} 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-copy-streams",
-  "version": "0.3.0",
+  "version": "1.2.0",
   "description": "Low-Level COPY TO and COPY FROM streams for PostgreSQL in JavaScript using",
   "main": "index.js",
   "scripts": {
@@ -23,7 +23,7 @@
     "url": "https://github.com/brianc/node-pg-copy-streams/issues"
   },
   "devDependencies": {
-    "pg.js": "~2.8.1",
+    "pg": "~4.4.3",
     "concat-stream": "~1.1.0",
     "gonna": "0.0.0",
     "lodash": "~2.2.1",

test/binary.js

@@ -4,7 +4,7 @@ var gonna = require('gonna')
 var async = require('async')
 var concat = require('concat-stream')
 var _ = require('lodash')
-var pg = require('pg.js')
+var pg = require('pg')
 
 var from = require('../').from
 var to = require('../').to

test/copy-from.js

@@ -3,7 +3,7 @@ var gonna = require('gonna')
 
 var concat = require('concat-stream')
 var _ = require('lodash')
-var pg = require('pg.js')
+var pg = require('pg')
 
 var copy = require('../').from
 
@@ -30,10 +30,6 @@ var testRange = function(top) {
 
   var txt = 'COPY numbers FROM STDIN'
   var stream = fromClient.query(copy(txt))
-  var rowEmitCount = 0
-  stream.on('row', function() {
-    rowEmitCount++
-  })
   for(var i = 0; i < top; i++) {
     stream.write(Buffer('' + i + '\t' + i*10 + '\n'))
   }
@@ -43,6 +39,7 @@ var testRange = function(top) {
     fromClient.query('SELECT COUNT(*) FROM numbers', function(err, res) {
       assert.ifError(err)
       assert.equal(res.rows[0].count, top, 'expected ' + top + ' rows but got ' + res.rows[0].count)
+      assert.equal(stream.rowCount, top, 'expected ' + top + ' rows but db count is ' + stream.rowCount)
       //console.log('found ', res.rows.length, 'rows')
       countDone()
       var firstRowDone = gonna('have correct result')
@@ -58,3 +55,19 @@ var testRange = function(top) {
 }
 
 testRange(1000)
+
+var testSingleEnd = function() {
+  var fromClient = client()
+  fromClient.query('CREATE TEMP TABLE numbers(num int)')
+  var txt = 'COPY numbers FROM STDIN';
+  var stream = fromClient.query(copy(txt))
+  var count = 0;
+  stream.on('end', function() {
+    count++;
+    assert(count==1, '`end` Event was triggered ' + count + ' times');
+    if (count == 1) fromClient.end();
+  })
+  stream.end(Buffer('1\n'))
+    
+}
+testSingleEnd()

test/copy-to.js

@@ -4,9 +4,10 @@ var gonna = require('gonna')
 var _ = require('lodash')
 var async = require('async')
 var concat = require('concat-stream')
-var pg = require('pg.js')
+var pg = require('pg')
 
 var copy = require('../').to
+var code = require('../message-formats')
 
 var client = function() {
   var client = new pg.Client()
@@ -22,9 +23,23 @@ var testConstruction = function() {
 
 testConstruction()
 
+var testComparators = function() {
+  var copy1 = copy();
+  copy1.pipe(concat(function(buf) {
+    assert(copy1._gotCopyOutResponse, 'should have received CopyOutResponse')
+    assert(!copy1._remainder, 'Message with no additional data (len=Int4Len+0) should not leave a remainder')
+  }))
+  copy1.end(new Buffer([code.CopyOutResponse, 0x00, 0x00, 0x00, 0x04])); 
+
+
+}
+testComparators();
+
 var testRange = function(top) {
   var fromClient = client()
   var txt = 'COPY (SELECT * from generate_series(0, ' + (top - 1) + ')) TO STDOUT'
+  var res;
+
 
   var stream = fromClient.query(copy(txt))
   var done = gonna('finish piping out', 1000, function() {
@@ -32,41 +47,19 @@ var testRange = function(top) {
   })
 
   stream.pipe(concat(function(buf) {
-    var res = buf.toString('utf8')
+    res = buf.toString('utf8')
+  }))
+
+  stream.on('end', function() {
     var expected = _.range(0, top).join('\n') + '\n'
     assert.equal(res, expected)
     assert.equal(stream.rowCount, top, 'should have rowCount ' + top + ' but got ' + stream.rowCount)
     done()
-  }))
+  });
 }
 
 testRange(10000)
 
-var testLeak = function(rounds) {
-  var fromClient = client()
-  var txt = 'COPY (SELECT 10) TO STDOUT'
-
-  var runStream = function(num, callback) {
-    var stream = fromClient.query(copy(txt))
-    stream.on('data', function(data) {
-      // Just throw away the data.
-    })
-    stream.on('end', callback)
-    stream.on('error', callback)
-  }
-
-  async.timesSeries(rounds, runStream, function(err) {
-    assert.equal(err, null)
-    assert.equal(fromClient.connection.stream.listeners('close').length, 0)
-    assert.equal(fromClient.connection.stream.listeners('data').length, 1)
-    assert.equal(fromClient.connection.stream.listeners('end').length, 2)
-    assert.equal(fromClient.connection.stream.listeners('error').length, 1)
-    fromClient.end()
-  })
-}
-
-testLeak(5)
-
 var testInternalPostgresError = function() {
   var cancelClient = client()
   var queryClient = client()
@@ -93,5 +86,37 @@ var testInternalPostgresError = function() {
     queryClient.end()
   })
 }
-
 testInternalPostgresError()
+
+var testNoticeResponse = function() {
+  // we use a special trick to generate a warning
+  // on the copy stream.
+  var queryClient = client()
+  var set = '';
+  set += 'SET SESSION client_min_messages = WARNING;'
+  set += 'SET SESSION standard_conforming_strings = off;'
+  set += 'SET SESSION escape_string_warning = on;'
+  queryClient.query(set, function(err, res) {
+    assert.equal(err, null, 'testNoticeResponse - could not SET parameters')
+    var runStream = function(callback) {
+      var txt = "COPY (SELECT '\\\n') TO STDOUT"
+      var stream = queryClient.query(copy(txt))
+      stream.on('data', function(data) {
+      })
+      stream.on('error', callback)
+     
+      // make sure stream is pulled from 
+      stream.pipe(concat(callback.bind(null,null)))
+    }
+
+    runStream(function(err) {
+      assert.equal(err, null, err)
+      queryClient.end()
+    })
+
+  })
+}
+
+testNoticeResponse();
+
+