Drop support for node@v0.10

* Eliminate detach/reattach strategy as it isn't able to differentiate between on/once and inconsistenly loses unshifted data depending on node version.  Instead just split stream and send it to copy stream and the connection.stream at the same time.  Disconnecting copy stream just means unpiping.  Added handleCopyData to fulfill query contract but ignore the incoming data.

Add node 4.2.2 to Travis
Minimum postgres 9.2 to allow tests to complete in Travis

Remove test that is no longer needed since we no longer disconnect/reconnect listeners

* Add resume

* Remove node 0.10 and add 0.12

* Re-enable old tests

* Add more versions to the travis test matrix

.travis.yml

@@ -0,0 +1,17 @@
+language: node_js
+node_js:
+  - "0.12"
+  - "4"
+  - "5"
+  - "6"
+
+addons:
+  postgresql: "9.2"
+
+services:
+  - postgresql
+
+before_install:
+  - npm install npm --global
+env:
+  - PGUSER=postgres PGDATABASE=postgres

Makefile

@@ -0,0 +1,14 @@
+.PHONY: publish-patch test
+
+test:
+	npm test
+
+patch: test
+	npm version patch -m "Bump version"
+	git push origin master --tags
+	npm publish
+
+minor: test
+	npm version minor -m "Bump version"
+	git push origin master --tags
+	npm publish

README.md

@@ -1,5 +1,7 @@
 ## pg-copy-streams
 
+[![Build Status](https://travis-ci.org/brianc/node-pg-copy-streams.svg)](https://travis-ci.org/brianc/node-pg-copy-streams)
+
 COPY FROM / COPY TO for node-postgres.  Stream from one database to another, and stuff.
 
 ## how? what? huh?
@@ -38,10 +40,9 @@ var copyFrom = require('pg-copy-streams').from;
 
 pg.connect(function(err, client, done) {
   var stream = client.query(copyFrom('COPY my_table FROM STDIN'));
-  var fileStream = fs.createReadStream('some_file.tdv')
-  fileStream.pipe(stream);
-  fileStream.on('end', done);
+  var fileStream = fs.createReadStream('some_file.tsv')
   fileStream.on('error', done);
+  fileStream.pipe(stream).on('finish', done).on('error', done);
 });
 ```
 
@@ -53,9 +54,11 @@ $ npm install pg-copy-streams
 
 ## notice
 
+This module __only__ works with the pure JavaScript bindings.  If you're using `require('pg').native` please make sure to use normal `require('pg')` or `require('pg.js')` when you're using copy streams.
+
 Before you set out on this magical piping journey, you _really_ should read this: http://www.postgresql.org/docs/9.3/static/sql-copy.html, and you might want to take a look at the [tests](https://github.com/brianc/node-pg-copy-streams/tree/master/test) to get an idea of how things work.
 
-## contributing 
+## contributing
 
 Instead of adding a bunch more code to the already bloated [node-postgres](https://github.com/brianc/node-postgres) I am trying to make the internals extensible and work on adding edge-case features as 3rd party modules.
 This is one of those.

copy-to.js

@@ -1,30 +1,25 @@
-module.exports = function(txt) {
-  return new CopyStreamQuery(txt)
+module.exports = function(txt, options) {
+  return new CopyStreamQuery(txt, options)
 }
 
 var Transform = require('stream').Transform
 var util = require('util')
 
-var CopyStreamQuery = function(text) {
-  Transform.call(this)
+var CopyStreamQuery = function(text, options) {
+  Transform.call(this, options)
   this.text = text
-  this._listeners = {}
   this._copyOutResponse = null
   this.rowCount = 0
 }
 
 util.inherits(CopyStreamQuery, Transform)
 
-var eventTypes = ['close', 'data', 'end']
+var eventTypes = ['close', 'data', 'end', 'error']
 
 CopyStreamQuery.prototype.submit = function(connection) {
   connection.query(this.text)
   this.connection = connection
-  var self = this
-  eventTypes.forEach(function(type) {
-    self._listeners[type] = connection.stream.listeners(type)
-    connection.stream.removeAllListeners(type)
-  })
+  this.connection.removeAllListeners('copyData')
   connection.stream.pipe(this)
 }
 
@@ -36,16 +31,12 @@ var code = {
 }
 
 CopyStreamQuery.prototype._detach = function() {
-  this.connection.stream.unpipe()
-  var self = this
-  eventTypes.forEach(function(type) {
-    self.connection.stream.removeAllListeners(type)
-    self._listeners[type].forEach(function(listener) {
-      self.connection.stream.on(type, listener)
-    })
-  })
+  this.connection.stream.unpipe(this)
+  // Unpipe can drop us out of flowing mode
+  this.connection.stream.resume()
 }
 
+
 CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
   var offset = 0
   if(this._remainder && chunk) {
@@ -55,7 +46,6 @@ CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
     this._copyOutResponse = true
     if(chunk[0] == code.E) {
       this._detach()
-      this.connection.stream.unshift(chunk)
       this.push(null)
       return cb();
     }
@@ -68,10 +58,9 @@ CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
   }
   while((chunk.length - offset) > 5) {
     var messageCode = chunk[offset]
-    //complete
-    if(messageCode == code.c) {
+    //complete or error
+    if(messageCode == code.c || messageCode == code.E) {
       this._detach()
-      this.connection.stream.unshift(chunk.slice(offset + 5))
       this.push(null)
       return cb();
     }
@@ -104,6 +93,9 @@ CopyStreamQuery.prototype.handleError = function(e) {
   this.emit('error', e)
 }
 
+CopyStreamQuery.prototype.handleCopyData = function(chunk) {
+}
+
 CopyStreamQuery.prototype.handleCommandComplete = function() {
 }
 

index.js

@@ -1,19 +1,18 @@
 var CopyToQueryStream = require('./copy-to')
-
 module.exports = {
-  to: function(txt) {
-    return new CopyToQueryStream(txt)
+  to: function(txt, options) {
+    return new CopyToQueryStream(txt, options)
   },
-  from: function (txt) {
-    return new CopyStreamQuery(txt)
+  from: function (txt, options) {
+    return new CopyStreamQuery(txt, options)
   }
 }
 
 var Transform = require('stream').Transform
 var util = require('util')
 
-var CopyStreamQuery = function(text) {
-  Transform.call(this)
+var CopyStreamQuery = function(text, options) {
+  Transform.call(this, options)
   this.text = text
   this._listeners = null
   this._copyOutResponse = null

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pg-copy-streams",
-  "version": "0.2.2",
+  "version": "1.0.0",
   "description": "Low-Level COPY TO and COPY FROM streams for PostgreSQL in JavaScript using",
   "main": "index.js",
   "scripts": {
@@ -23,7 +23,7 @@
     "url": "https://github.com/brianc/node-pg-copy-streams/issues"
   },
   "devDependencies": {
-    "pg.js": "~2.8.1",
+    "pg": "~4.4.3",
     "concat-stream": "~1.1.0",
     "gonna": "0.0.0",
     "lodash": "~2.2.1",

test/binary.js

@@ -0,0 +1,67 @@
+var assert = require('assert')
+var gonna = require('gonna')
+
+var async = require('async')
+var concat = require('concat-stream')
+var _ = require('lodash')
+var pg = require('pg')
+
+var from = require('../').from
+var to = require('../').to
+
+var testBinaryCopy = function() {
+  var client = function() {
+    var client = new pg.Client()
+    client.connect()
+    return client
+  }
+
+  var fromClient = client()
+  var toClient = client()
+
+  queries = [
+    'DROP TABLE IF EXISTS data',
+    'CREATE TABLE IF NOT EXISTS data (num BIGINT, word TEXT)',
+    'INSERT INTO data (num, word) VALUES (1, \'hello\'), (2, \'other thing\'), (3, \'goodbye\')',
+    'DROP TABLE IF EXISTS data_copy',
+    'CREATE TABLE IF NOT EXISTS data_copy (LIKE data INCLUDING ALL)'
+  ]
+
+  async.eachSeries(queries, _.bind(fromClient.query, fromClient), function(err) {
+    assert.ifError(err)
+
+    var fromStream = fromClient.query(to('COPY (SELECT * FROM data) TO STDOUT BINARY'))
+    var toStream = toClient.query(from('COPY data_copy FROM STDIN BINARY'))
+
+    runStream = function(callback) {
+      fromStream.on('error', callback)
+      toStream.on('error', callback)
+      toStream.on('finish', callback)
+      fromStream.pipe(toStream)
+    }
+    runStream(function(err) {
+      assert.ifError(err)
+
+      toClient.query('SELECT * FROM data_copy ORDER BY num', function(err, res){
+        assert.equal(res.rowCount, 3, 'expected 3 rows but got ' + res.rowCount)
+        assert.equal(res.rows[0].num, 1)
+        assert.equal(res.rows[0].word, 'hello')
+        assert.equal(res.rows[1].num, 2)
+        assert.equal(res.rows[1].word, 'other thing')
+        assert.equal(res.rows[2].num, 3)
+        assert.equal(res.rows[2].word, 'goodbye')
+        queries = [
+          'DROP TABLE data',
+          'DROP TABLE data_copy'
+        ]
+        async.each(queries, _.bind(fromClient.query, fromClient), function(err) {
+          assert.ifError(err)
+          fromClient.end()
+          toClient.end()
+        })
+      })
+    })
+  })
+}
+
+testBinaryCopy()

test/copy-from.js

@@ -3,22 +3,32 @@ var gonna = require('gonna')
 
 var concat = require('concat-stream')
 var _ = require('lodash')
-var pg = require('pg.js')
+var pg = require('pg')
+
+var copy = require('../').from
+
+var client = function() {
+  var client = new pg.Client()
+  client.connect()
+  return client
+}
+
+var testConstruction = function() {
+  var highWaterMark = 10
+  var stream = copy('COPY numbers FROM STDIN', {highWaterMark: 10, objectMode: true})
+  for(var i = 0; i < highWaterMark * 1.5; i++) {
+    stream.write('1\t2\n')
+  }
+  assert(!stream.write('1\t2\n'), 'Should correctly set highWaterMark.')
+}
+
+testConstruction()
 
 var testRange = function(top) {
-  var client = function() {
-    var client = new pg.Client()
-    client.connect()
-    client.query('CREATE TEMP TABLE numbers(num int, bigger_num int)')
-    return client
-  }
-
   var fromClient = client()
-  var copy = require('../').from
-
+  fromClient.query('CREATE TEMP TABLE numbers(num int, bigger_num int)')
 
   var txt = 'COPY numbers FROM STDIN'
-
   var stream = fromClient.query(copy(txt))
   var rowEmitCount = 0
   stream.on('row', function() {
@@ -33,7 +43,7 @@ var testRange = function(top) {
     fromClient.query('SELECT COUNT(*) FROM numbers', function(err, res) {
       assert.ifError(err)
       assert.equal(res.rows[0].count, top, 'expected ' + top + ' rows but got ' + res.rows[0].count)
-      console.log('found ', res.rows.length, 'rows')
+      //console.log('found ', res.rows.length, 'rows')
       countDone()
       var firstRowDone = gonna('have correct result')
       assert.equal(stream.rowCount, top, 'should have rowCount ' + top + ' ')

test/copy-to.js

@@ -4,7 +4,7 @@ var gonna = require('gonna')
 var _ = require('lodash')
 var async = require('async')
 var concat = require('concat-stream')
-var pg = require('pg.js')
+var pg = require('pg')
 
 var copy = require('../').to
 
@@ -14,9 +14,19 @@ var client = function() {
   return client
 }
 
+var testConstruction = function() {
+  var txt = 'COPY (SELECT * FROM generate_series(0, 10)) TO STDOUT'
+  var stream = copy(txt, {highWaterMark: 10})
+  assert.equal(stream._readableState.highWaterMark, 10, 'Client should have been set with a correct highWaterMark.')
+}
+
+testConstruction()
+
 var testRange = function(top) {
   var fromClient = client()
   var txt = 'COPY (SELECT * from generate_series(0, ' + (top - 1) + ')) TO STDOUT'
+  var res;
+
 
   var stream = fromClient.query(copy(txt))
   var done = gonna('finish piping out', 1000, function() {
@@ -24,36 +34,44 @@ var testRange = function(top) {
   })
 
   stream.pipe(concat(function(buf) {
-    var res = buf.toString('utf8')
+    res = buf.toString('utf8')
+  }))
+
+  stream.on('end', function() {
     var expected = _.range(0, top).join('\n') + '\n'
     assert.equal(res, expected)
     assert.equal(stream.rowCount, top, 'should have rowCount ' + top + ' but got ' + stream.rowCount)
     done()
-  }))
+  });
 }
 
 testRange(10000)
 
-var testLeak = function(rounds) {
-  var fromClient = client()
-  var txt = 'COPY (SELECT 10) TO STDOUT'
+var testInternalPostgresError = function() {
+  var cancelClient = client()
+  var queryClient = client()
 
-  var runStream = function(num, callback) {
-    var stream = fromClient.query(copy(txt))
+  var runStream = function(callback) {
+    var txt = "COPY (SELECT pg_sleep(10)) TO STDOUT"
+    var stream = queryClient.query(copy(txt))
     stream.on('data', function(data) {
       // Just throw away the data.
     })
-    stream.on('end', callback)
     stream.on('error', callback)
+
+    setTimeout(function() {
+      var cancelQuery = "SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE query ~ 'pg_sleep' AND NOT query ~ 'pg_cancel_backend'"
+      cancelClient.query(cancelQuery)
+    }, 50)
   }
 
-  async.timesSeries(rounds, runStream, function(err) {
-    assert.equal(err, null)
-    assert.equal(fromClient.connection.stream.listeners('data').length, 1)
-    assert.equal(fromClient.connection.stream.listeners('end').length, 2)
-    assert.equal(fromClient.connection.stream.listeners('close').length, 0)
-    fromClient.end()
+  runStream(function(err) {
+    assert.notEqual(err, null)
+    var expectedMessage = 'canceling statement due to user request'
+    assert.notEqual(err.toString().indexOf(expectedMessage), -1, 'Error message should mention reason for query failure.')
+    cancelClient.end()
+    queryClient.end()
   })
 }
 
-testLeak(5)
+testInternalPostgresError()

test/index.js

@@ -1,2 +1,3 @@
 require('./copy-from')
 require('./copy-to')
+require('./binary')