Compare commits
39 Commits
PushSpring
...
v1.2-carto
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ff2cf6746 | ||
|
|
94c8d89aba | ||
|
|
c19ab0746c | ||
|
|
a27bd1fd8f | ||
|
|
01ab20fce2 | ||
|
|
d55dbe70d5 | ||
|
|
de47558628 | ||
|
|
38c78677e8 | ||
|
|
6f1d5cb4a5 | ||
|
|
99d397956c | ||
|
|
c85f7d27b8 | ||
|
|
8b355b6e72 | ||
|
|
d7e5c1383f | ||
|
|
cb2227d159 | ||
|
|
7930d1b8dd | ||
|
|
e94fefe902 | ||
|
|
9293926047 | ||
|
|
fd3cc95573 | ||
|
|
922627daaf | ||
|
|
61bc713e0c | ||
|
|
e15feb199a | ||
|
|
191a4ec16a | ||
|
|
399bff7ed7 | ||
|
|
8174e10fb5 | ||
|
|
f29aef3bba | ||
|
|
b2e108571e | ||
|
|
7003f6070f | ||
|
|
ade7ab95a0 | ||
|
|
9ccda04036 | ||
|
|
a5e532f20b | ||
|
|
2a4db2920e | ||
|
|
f155899570 | ||
|
|
18be125596 | ||
|
|
ae5b344395 | ||
|
|
ee84aba89f | ||
|
|
e0aa7db324 | ||
|
|
0f0ddf7ad4 | ||
|
|
ed57e131e9 | ||
|
|
c4a0e6dd58 |
@@ -1,9 +1,8 @@
|
|||||||
language: node_js
|
language: node_js
|
||||||
node_js:
|
node_js:
|
||||||
- "0.12"
|
|
||||||
- "4"
|
|
||||||
- "5"
|
|
||||||
- "6"
|
- "6"
|
||||||
|
- "8"
|
||||||
|
- "10"
|
||||||
|
|
||||||
addons:
|
addons:
|
||||||
postgresql: "9.2"
|
postgresql: "9.2"
|
||||||
|
|||||||
30
NEWS.carto.md
Normal file
30
NEWS.carto.md
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
# CARTO's Changelog
|
||||||
|
|
||||||
|
## v1.2.0-carto.4
|
||||||
|
Released 2018-mm-dd
|
||||||
|
|
||||||
|
## v1.2.0-carto.3
|
||||||
|
Released 2018-11-21
|
||||||
|
|
||||||
|
Features:
|
||||||
|
* Drop support for Node.js 0.12, 4 and, 5.
|
||||||
|
* Add support for Node.js 8 and 10.
|
||||||
|
* Add package-lock.json
|
||||||
|
* Do not use deprecated Buffer constructors.
|
||||||
|
|
||||||
|
## v1.2.0-carto.2
|
||||||
|
Released 2018-10-26
|
||||||
|
|
||||||
|
Bug fixes:
|
||||||
|
* Make all modules to use strict mode semantics.
|
||||||
|
|
||||||
|
## v1.2.0-carto.1
|
||||||
|
Released 2018-06-11
|
||||||
|
|
||||||
|
Bug fixes:
|
||||||
|
* Improves performance of COPY TO by sending bigger chunks through low level `push()`. See https://github.com/CartoDB/node-pg-copy-streams/pull/1
|
||||||
|
|
||||||
|
## v1.2.0
|
||||||
|
Released 2016-08-22
|
||||||
|
|
||||||
|
Vanilla version v1.2.0 from upstream repository. See https://github.com/CartoDB/node-pg-copy-streams/releases/tag/v1.2.0
|
||||||
16
README.md
16
README.md
@@ -1,3 +1,7 @@
|
|||||||
|
## Note
|
||||||
|
|
||||||
|
This is forked repository from [brianc/node-pg-copy-streams](https://github.com/brianc/node-pg-query-stream)
|
||||||
|
|
||||||
## pg-copy-streams
|
## pg-copy-streams
|
||||||
|
|
||||||
[](https://travis-ci.org/brianc/node-pg-copy-streams)
|
[](https://travis-ci.org/brianc/node-pg-copy-streams)
|
||||||
@@ -42,10 +46,15 @@ pg.connect(function(err, client, done) {
|
|||||||
var stream = client.query(copyFrom('COPY my_table FROM STDIN'));
|
var stream = client.query(copyFrom('COPY my_table FROM STDIN'));
|
||||||
var fileStream = fs.createReadStream('some_file.tsv')
|
var fileStream = fs.createReadStream('some_file.tsv')
|
||||||
fileStream.on('error', done);
|
fileStream.on('error', done);
|
||||||
fileStream.pipe(stream).on('finish', done).on('error', done);
|
stream.on('error', done);
|
||||||
|
stream.on('end', done);
|
||||||
|
fileStream.pipe(stream);
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
|
*Important*: Even if `pg-copy-streams.from` is used as a Writable (via `pipe`), you should not listen for the 'finish' event and expect that the COPY command has already been correctly acknowledged by the database. Internally, a duplex stream is used to pipe the data into the database connection and the COPY command should be considered complete only when the 'end' event is triggered.
|
||||||
|
|
||||||
|
|
||||||
## install
|
## install
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -56,7 +65,10 @@ $ npm install pg-copy-streams
|
|||||||
|
|
||||||
This module __only__ works with the pure JavaScript bindings. If you're using `require('pg').native` please make sure to use normal `require('pg')` or `require('pg.js')` when you're using copy streams.
|
This module __only__ works with the pure JavaScript bindings. If you're using `require('pg').native` please make sure to use normal `require('pg')` or `require('pg.js')` when you're using copy streams.
|
||||||
|
|
||||||
Before you set out on this magical piping journey, you _really_ should read this: http://www.postgresql.org/docs/9.3/static/sql-copy.html, and you might want to take a look at the [tests](https://github.com/brianc/node-pg-copy-streams/tree/master/test) to get an idea of how things work.
|
Before you set out on this magical piping journey, you _really_ should read this: http://www.postgresql.org/docs/current/static/sql-copy.html, and you might want to take a look at the [tests](https://github.com/brianc/node-pg-copy-streams/tree/master/test) to get an idea of how things work.
|
||||||
|
|
||||||
|
Take note of the following warning in the PostgreSQL documentation:
|
||||||
|
> COPY stops operation at the first error. This should not lead to problems in the event of a COPY TO, but the target table will already have received earlier rows in a COPY FROM. These rows will not be visible or accessible, but they still occupy disk space. This might amount to a considerable amount of wasted disk space if the failure happened well into a large copy operation. You might wish to invoke VACUUM to recover the wasted space.
|
||||||
|
|
||||||
## contributing
|
## contributing
|
||||||
|
|
||||||
|
|||||||
107
copy-to.js
107
copy-to.js
@@ -1,14 +1,17 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
module.exports = function(txt, options) {
|
module.exports = function(txt, options) {
|
||||||
return new CopyStreamQuery(txt, options)
|
return new CopyStreamQuery(txt, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
var Transform = require('stream').Transform
|
var Transform = require('stream').Transform
|
||||||
var util = require('util')
|
var util = require('util')
|
||||||
|
var code = require('./message-formats')
|
||||||
|
|
||||||
var CopyStreamQuery = function(text, options) {
|
var CopyStreamQuery = function(text, options) {
|
||||||
Transform.call(this, options)
|
Transform.call(this, options)
|
||||||
this.text = text
|
this.text = text
|
||||||
this._copyOutResponse = null
|
this._gotCopyOutResponse = false
|
||||||
this.rowCount = 0
|
this.rowCount = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -23,63 +26,89 @@ CopyStreamQuery.prototype.submit = function(connection) {
|
|||||||
connection.stream.pipe(this)
|
connection.stream.pipe(this)
|
||||||
}
|
}
|
||||||
|
|
||||||
var code = {
|
|
||||||
E: 69, //Error
|
|
||||||
H: 72, //CopyOutResponse
|
|
||||||
d: 0x64, //CopyData
|
|
||||||
c: 0x63 //CopyDone
|
|
||||||
}
|
|
||||||
|
|
||||||
CopyStreamQuery.prototype._detach = function() {
|
CopyStreamQuery.prototype._detach = function() {
|
||||||
this.connection.stream.unpipe(this)
|
this.connection.stream.unpipe(this)
|
||||||
// Unpipe can drop us out of flowing mode
|
// Unpipe can drop us out of flowing mode
|
||||||
this.connection.stream.resume()
|
this.connection.stream.resume()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
||||||
var offset = 0
|
var offset = 0
|
||||||
|
var Byte1Len = 1;
|
||||||
|
var Int32Len = 4;
|
||||||
if(this._remainder && chunk) {
|
if(this._remainder && chunk) {
|
||||||
chunk = Buffer.concat([this._remainder, chunk])
|
chunk = Buffer.concat([this._remainder, chunk])
|
||||||
}
|
}
|
||||||
if(!this._copyOutResponse) {
|
|
||||||
this._copyOutResponse = true
|
var length;
|
||||||
if(chunk[0] == code.E) {
|
var messageCode;
|
||||||
this._detach()
|
var needPush = false;
|
||||||
this.push(null)
|
|
||||||
return cb();
|
var buffer = Buffer.alloc(chunk.length);
|
||||||
|
var buffer_offset = 0;
|
||||||
|
|
||||||
|
this.pushBufferIfneeded = function() {
|
||||||
|
if (needPush && buffer_offset > 0) {
|
||||||
|
this.push(buffer.slice(0, buffer_offset))
|
||||||
|
buffer_offset = 0;
|
||||||
}
|
}
|
||||||
if(chunk[0] != code.H) {
|
|
||||||
this.emit('error', new Error('Expected copy out response'))
|
|
||||||
}
|
|
||||||
var length = chunk.readUInt32BE(1)
|
|
||||||
offset = 1
|
|
||||||
offset += length
|
|
||||||
}
|
}
|
||||||
while((chunk.length - offset) > 5) {
|
|
||||||
|
while((chunk.length - offset) >= (Byte1Len + Int32Len)) {
|
||||||
var messageCode = chunk[offset]
|
var messageCode = chunk[offset]
|
||||||
//complete or error
|
|
||||||
if(messageCode == code.c || messageCode == code.E) {
|
//console.log('PostgreSQL message ' + String.fromCharCode(messageCode))
|
||||||
this._detach()
|
switch(messageCode) {
|
||||||
this.push(null)
|
|
||||||
return cb();
|
// detect COPY start
|
||||||
|
case code.CopyOutResponse:
|
||||||
|
if (!this._gotCopyOutResponse) {
|
||||||
|
this._gotCopyOutResponse = true
|
||||||
|
} else {
|
||||||
|
this.emit('error', new Error('Unexpected CopyOutResponse message (H)'))
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
// meaningful row
|
||||||
|
case code.CopyData:
|
||||||
|
needPush = true;
|
||||||
|
break;
|
||||||
|
|
||||||
|
// standard interspersed messages. discard
|
||||||
|
case code.ParameterStatus:
|
||||||
|
case code.NoticeResponse:
|
||||||
|
case code.NotificationResponse:
|
||||||
|
break;
|
||||||
|
|
||||||
|
case code.ErrorResponse:
|
||||||
|
case code.CopyDone:
|
||||||
|
this.pushBufferIfneeded();
|
||||||
|
this._detach()
|
||||||
|
this.push(null)
|
||||||
|
return cb();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
this.emit('error', new Error('Unexpected PostgreSQL message ' + String.fromCharCode(messageCode)))
|
||||||
}
|
}
|
||||||
//something bad happened
|
|
||||||
if(messageCode != code.d) {
|
length = chunk.readUInt32BE(offset+Byte1Len)
|
||||||
return this.emit('error', new Error('expected "d" (copydata message)'))
|
if(chunk.length >= (offset + Byte1Len + length)) {
|
||||||
}
|
offset += Byte1Len + Int32Len
|
||||||
var length = chunk.readUInt32BE(offset + 1) - 4 //subtract length of UInt32
|
if (needPush) {
|
||||||
//can we read the next row?
|
var row = chunk.slice(offset, offset + length - Int32Len)
|
||||||
if(chunk.length > (offset + length + 5)) {
|
this.rowCount++
|
||||||
offset += 5
|
row.copy(buffer, buffer_offset);
|
||||||
var slice = chunk.slice(offset, offset + length)
|
buffer_offset += row.length;
|
||||||
offset += length
|
}
|
||||||
this.push(slice)
|
offset += (length - Int32Len)
|
||||||
this.rowCount++
|
|
||||||
} else {
|
} else {
|
||||||
|
// we need more chunks for a complete message
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.pushBufferIfneeded();
|
||||||
|
|
||||||
if(chunk.length - offset) {
|
if(chunk.length - offset) {
|
||||||
var slice = chunk.slice(offset)
|
var slice = chunk.slice(offset)
|
||||||
this._remainder = slice
|
this._remainder = slice
|
||||||
|
|||||||
42
index.js
42
index.js
@@ -1,3 +1,5 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
var CopyToQueryStream = require('./copy-to')
|
var CopyToQueryStream = require('./copy-to')
|
||||||
module.exports = {
|
module.exports = {
|
||||||
to: function(txt, options) {
|
to: function(txt, options) {
|
||||||
@@ -10,6 +12,7 @@ module.exports = {
|
|||||||
|
|
||||||
var Transform = require('stream').Transform
|
var Transform = require('stream').Transform
|
||||||
var util = require('util')
|
var util = require('util')
|
||||||
|
var code = require('./message-formats')
|
||||||
|
|
||||||
var CopyStreamQuery = function(text, options) {
|
var CopyStreamQuery = function(text, options) {
|
||||||
Transform.call(this, options)
|
Transform.call(this, options)
|
||||||
@@ -26,28 +29,23 @@ CopyStreamQuery.prototype.submit = function(connection) {
|
|||||||
connection.query(this.text)
|
connection.query(this.text)
|
||||||
}
|
}
|
||||||
|
|
||||||
var code = {
|
|
||||||
H: 72, //CopyOutResponse
|
|
||||||
d: 0x64, //CopyData
|
|
||||||
c: 0x63 //CopyDone
|
|
||||||
}
|
|
||||||
|
|
||||||
var copyDataBuffer = Buffer([code.d])
|
var copyDataBuffer = Buffer.from([code.CopyData])
|
||||||
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
||||||
|
var Int32Len = 4;
|
||||||
this.push(copyDataBuffer)
|
this.push(copyDataBuffer)
|
||||||
var lenBuffer = Buffer(4)
|
var lenBuffer = Buffer.alloc(Int32Len)
|
||||||
lenBuffer.writeUInt32BE(chunk.length + 4, 0)
|
lenBuffer.writeUInt32BE(chunk.length + Int32Len, 0)
|
||||||
this.push(lenBuffer)
|
this.push(lenBuffer)
|
||||||
this.push(chunk)
|
this.push(chunk)
|
||||||
this.rowCount++
|
|
||||||
cb()
|
cb()
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype._flush = function(cb) {
|
CopyStreamQuery.prototype._flush = function(cb) {
|
||||||
var finBuffer = Buffer([code.c, 0, 0, 0, 4])
|
var Int32Len = 4;
|
||||||
|
var finBuffer = Buffer.from([code.CopyDone, 0, 0, 0, Int32Len])
|
||||||
this.push(finBuffer)
|
this.push(finBuffer)
|
||||||
//never call this callback, do not close underlying stream
|
this.cb_flush = cb
|
||||||
//cb()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleError = function(e) {
|
CopyStreamQuery.prototype.handleError = function(e) {
|
||||||
@@ -55,12 +53,24 @@ CopyStreamQuery.prototype.handleError = function(e) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleCopyInResponse = function(connection) {
|
CopyStreamQuery.prototype.handleCopyInResponse = function(connection) {
|
||||||
this.pipe(connection.stream)
|
this.pipe(connection.stream, { end: false })
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleCommandComplete = function() {
|
CopyStreamQuery.prototype.handleCommandComplete = function(msg) {
|
||||||
this.unpipe()
|
// Parse affected row count as in
|
||||||
this.emit('end')
|
// https://github.com/brianc/node-postgres/blob/35e5567f86774f808c2a8518dd312b8aa3586693/lib/result.js#L37
|
||||||
|
var match = /COPY (\d+)/.exec((msg || {}).text)
|
||||||
|
if (match) {
|
||||||
|
this.rowCount = parseInt(match[1], 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// we delay the _flush cb so that the 'end' event is
|
||||||
|
// triggered after CommandComplete
|
||||||
|
this.cb_flush()
|
||||||
|
|
||||||
|
// unpipe from connection
|
||||||
|
this.unpipe(this.connection)
|
||||||
|
this.connection = null
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleReadyForQuery = function() {
|
CopyStreamQuery.prototype.handleReadyForQuery = function() {
|
||||||
|
|||||||
25
message-formats.js
Normal file
25
message-formats.js
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
/**
|
||||||
|
* The COPY feature uses the following protocol codes.
|
||||||
|
* The codes for the most recent protocol version are documented on
|
||||||
|
* https://www.postgresql.org/docs/current/static/protocol-message-formats.html
|
||||||
|
*
|
||||||
|
* The protocol flow itself is described on
|
||||||
|
* https://www.postgresql.org/docs/current/static/protocol-flow.html
|
||||||
|
*/
|
||||||
|
module.exports = {
|
||||||
|
ErrorResponse: 0x45, // E
|
||||||
|
CopyInResponse: 0x47, // G
|
||||||
|
CopyOutResponse: 0x48, // H
|
||||||
|
CopyBothResponse: 0x57, // W
|
||||||
|
CopyDone: 0x63, // c
|
||||||
|
CopyData: 0x64, // d
|
||||||
|
CopyFail: 0x66, // f
|
||||||
|
|
||||||
|
// It is possible for NoticeResponse and ParameterStatus messages to be interspersed between CopyData messages;
|
||||||
|
// frontends must handle these cases, and should be prepared for other asynchronous message types as well
|
||||||
|
// (see Section 50.2.6).
|
||||||
|
// Otherwise, any message type other than CopyData or CopyDone may be treated as terminating copy-out mode.
|
||||||
|
NotificationResponse: 0x41, // A
|
||||||
|
NoticeResponse: 0x4E, // N
|
||||||
|
ParameterStatus: 0x53 // S
|
||||||
|
}
|
||||||
193
package-lock.json
generated
Normal file
193
package-lock.json
generated
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
{
|
||||||
|
"name": "pg-copy-streams",
|
||||||
|
"version": "1.2.0-carto.4",
|
||||||
|
"lockfileVersion": 1,
|
||||||
|
"requires": true,
|
||||||
|
"dependencies": {
|
||||||
|
"async": {
|
||||||
|
"version": "0.2.10",
|
||||||
|
"resolved": "http://registry.npmjs.org/async/-/async-0.2.10.tgz",
|
||||||
|
"integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"base64-js": {
|
||||||
|
"version": "0.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-0.0.2.tgz",
|
||||||
|
"integrity": "sha1-Ak8Pcq+iW3X5wO5zzU9V7Bvtl4Q=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"bops": {
|
||||||
|
"version": "0.0.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/bops/-/bops-0.0.6.tgz",
|
||||||
|
"integrity": "sha1-CC0dVfoB5g29wuvC26N/ZZVUzzo=",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"base64-js": "0.0.2",
|
||||||
|
"to-utf8": "0.0.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"buffer-writer": {
|
||||||
|
"version": "1.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
|
||||||
|
"integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"concat-stream": {
|
||||||
|
"version": "1.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.1.0.tgz",
|
||||||
|
"integrity": "sha1-hCae/YzGUCdeMi8wnfRIZ7xRxfM=",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"bops": "0.0.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"generic-pool": {
|
||||||
|
"version": "2.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz",
|
||||||
|
"integrity": "sha1-rwTcLDJc/Ll1Aj+lK/zpYXp0Nf0=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"gonna": {
|
||||||
|
"version": "0.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/gonna/-/gonna-0.0.0.tgz",
|
||||||
|
"integrity": "sha1-6k4ZsVJ6F4LhJQVeMCSabUvHmlk=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"heroku-env": {
|
||||||
|
"version": "0.1.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/heroku-env/-/heroku-env-0.1.1.tgz",
|
||||||
|
"integrity": "sha1-wGeRyUTpuHSOMXf1S/cBQyZ+Yxc=",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"parse-database-url": "~0.2.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"lodash": {
|
||||||
|
"version": "2.2.1",
|
||||||
|
"resolved": "http://registry.npmjs.org/lodash/-/lodash-2.2.1.tgz",
|
||||||
|
"integrity": "sha1-ypNf0UqzwMhyq6zxmLnNpQFECGc=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"packet-reader": {
|
||||||
|
"version": "0.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.2.0.tgz",
|
||||||
|
"integrity": "sha1-gZ300BC4LV6lZx+KGjrPA5vNdwA=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"parse-database-url": {
|
||||||
|
"version": "0.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/parse-database-url/-/parse-database-url-0.2.2.tgz",
|
||||||
|
"integrity": "sha1-SGFa56fA/HfjKU0jVCpqUnPDVws=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"pg": {
|
||||||
|
"version": "4.4.6",
|
||||||
|
"resolved": "http://registry.npmjs.org/pg/-/pg-4.4.6.tgz",
|
||||||
|
"integrity": "sha1-EZgiP7rva6QRqm9Q4X9OtaGTFVk=",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"buffer-writer": "1.0.1",
|
||||||
|
"generic-pool": "2.1.1",
|
||||||
|
"packet-reader": "0.2.0",
|
||||||
|
"pg-connection-string": "0.1.3",
|
||||||
|
"pg-types": "1.*",
|
||||||
|
"pgpass": "0.0.3",
|
||||||
|
"semver": "^4.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg-connection-string": {
|
||||||
|
"version": "0.1.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
|
||||||
|
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"pg-int8": {
|
||||||
|
"version": "1.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
||||||
|
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"pg-types": {
|
||||||
|
"version": "1.13.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz",
|
||||||
|
"integrity": "sha512-lfKli0Gkl/+za/+b6lzENajczwZHc7D5kiUCZfgm914jipD2kIOIvEkAhZ8GrW3/TUoP9w8FHjwpPObBye5KQQ==",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"pg-int8": "1.0.1",
|
||||||
|
"postgres-array": "~1.0.0",
|
||||||
|
"postgres-bytea": "~1.0.0",
|
||||||
|
"postgres-date": "~1.0.0",
|
||||||
|
"postgres-interval": "^1.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pgpass": {
|
||||||
|
"version": "0.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-0.0.3.tgz",
|
||||||
|
"integrity": "sha1-EuZ+NDsxicLzEgbrycwL7//PkUA=",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"split": "~0.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"postgres-array": {
|
||||||
|
"version": "1.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz",
|
||||||
|
"integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ==",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"postgres-bytea": {
|
||||||
|
"version": "1.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
|
||||||
|
"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"postgres-date": {
|
||||||
|
"version": "1.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz",
|
||||||
|
"integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"postgres-interval": {
|
||||||
|
"version": "1.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz",
|
||||||
|
"integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"xtend": "^4.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"semver": {
|
||||||
|
"version": "4.3.6",
|
||||||
|
"resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
|
||||||
|
"integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"split": {
|
||||||
|
"version": "0.3.3",
|
||||||
|
"resolved": "http://registry.npmjs.org/split/-/split-0.3.3.tgz",
|
||||||
|
"integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"through": "2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"through": {
|
||||||
|
"version": "2.3.8",
|
||||||
|
"resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||||
|
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"to-utf8": {
|
||||||
|
"version": "0.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/to-utf8/-/to-utf8-0.0.1.tgz",
|
||||||
|
"integrity": "sha1-0Xrqcv8vujm55DYBvns/9y4ImFI=",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
|
"xtend": {
|
||||||
|
"version": "4.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
|
||||||
|
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
|
||||||
|
"dev": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "pg-copy-streams",
|
"name": "pg-copy-streams",
|
||||||
"version": "0.3.0",
|
"version": "1.2.0-carto.4",
|
||||||
"description": "Low-Level COPY TO and COPY FROM streams for PostgreSQL in JavaScript using",
|
"description": "Low-Level COPY TO and COPY FROM streams for PostgreSQL in JavaScript using",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
var assert = require('assert')
|
var assert = require('assert')
|
||||||
var gonna = require('gonna')
|
var gonna = require('gonna')
|
||||||
|
|
||||||
@@ -19,7 +21,7 @@ var testBinaryCopy = function() {
|
|||||||
var fromClient = client()
|
var fromClient = client()
|
||||||
var toClient = client()
|
var toClient = client()
|
||||||
|
|
||||||
queries = [
|
var queries = [
|
||||||
'DROP TABLE IF EXISTS data',
|
'DROP TABLE IF EXISTS data',
|
||||||
'CREATE TABLE IF NOT EXISTS data (num BIGINT, word TEXT)',
|
'CREATE TABLE IF NOT EXISTS data (num BIGINT, word TEXT)',
|
||||||
'INSERT INTO data (num, word) VALUES (1, \'hello\'), (2, \'other thing\'), (3, \'goodbye\')',
|
'INSERT INTO data (num, word) VALUES (1, \'hello\'), (2, \'other thing\'), (3, \'goodbye\')',
|
||||||
@@ -33,7 +35,7 @@ var testBinaryCopy = function() {
|
|||||||
var fromStream = fromClient.query(to('COPY (SELECT * FROM data) TO STDOUT BINARY'))
|
var fromStream = fromClient.query(to('COPY (SELECT * FROM data) TO STDOUT BINARY'))
|
||||||
var toStream = toClient.query(from('COPY data_copy FROM STDIN BINARY'))
|
var toStream = toClient.query(from('COPY data_copy FROM STDIN BINARY'))
|
||||||
|
|
||||||
runStream = function(callback) {
|
var runStream = function(callback) {
|
||||||
fromStream.on('error', callback)
|
fromStream.on('error', callback)
|
||||||
toStream.on('error', callback)
|
toStream.on('error', callback)
|
||||||
toStream.on('finish', callback)
|
toStream.on('finish', callback)
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
var assert = require('assert')
|
var assert = require('assert')
|
||||||
var gonna = require('gonna')
|
var gonna = require('gonna')
|
||||||
|
|
||||||
@@ -30,12 +32,8 @@ var testRange = function(top) {
|
|||||||
|
|
||||||
var txt = 'COPY numbers FROM STDIN'
|
var txt = 'COPY numbers FROM STDIN'
|
||||||
var stream = fromClient.query(copy(txt))
|
var stream = fromClient.query(copy(txt))
|
||||||
var rowEmitCount = 0
|
|
||||||
stream.on('row', function() {
|
|
||||||
rowEmitCount++
|
|
||||||
})
|
|
||||||
for(var i = 0; i < top; i++) {
|
for(var i = 0; i < top; i++) {
|
||||||
stream.write(Buffer('' + i + '\t' + i*10 + '\n'))
|
stream.write(Buffer.from('' + i + '\t' + i*10 + '\n'))
|
||||||
}
|
}
|
||||||
stream.end()
|
stream.end()
|
||||||
var countDone = gonna('have correct count')
|
var countDone = gonna('have correct count')
|
||||||
@@ -43,6 +41,7 @@ var testRange = function(top) {
|
|||||||
fromClient.query('SELECT COUNT(*) FROM numbers', function(err, res) {
|
fromClient.query('SELECT COUNT(*) FROM numbers', function(err, res) {
|
||||||
assert.ifError(err)
|
assert.ifError(err)
|
||||||
assert.equal(res.rows[0].count, top, 'expected ' + top + ' rows but got ' + res.rows[0].count)
|
assert.equal(res.rows[0].count, top, 'expected ' + top + ' rows but got ' + res.rows[0].count)
|
||||||
|
assert.equal(stream.rowCount, top, 'expected ' + top + ' rows but db count is ' + stream.rowCount)
|
||||||
//console.log('found ', res.rows.length, 'rows')
|
//console.log('found ', res.rows.length, 'rows')
|
||||||
countDone()
|
countDone()
|
||||||
var firstRowDone = gonna('have correct result')
|
var firstRowDone = gonna('have correct result')
|
||||||
@@ -58,3 +57,19 @@ var testRange = function(top) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
testRange(1000)
|
testRange(1000)
|
||||||
|
|
||||||
|
var testSingleEnd = function() {
|
||||||
|
var fromClient = client()
|
||||||
|
fromClient.query('CREATE TEMP TABLE numbers(num int)')
|
||||||
|
var txt = 'COPY numbers FROM STDIN';
|
||||||
|
var stream = fromClient.query(copy(txt))
|
||||||
|
var count = 0;
|
||||||
|
stream.on('end', function() {
|
||||||
|
count++;
|
||||||
|
assert(count==1, '`end` Event was triggered ' + count + ' times');
|
||||||
|
if (count == 1) fromClient.end();
|
||||||
|
})
|
||||||
|
stream.end(Buffer.from('1\n'))
|
||||||
|
|
||||||
|
}
|
||||||
|
testSingleEnd()
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
var assert = require('assert')
|
var assert = require('assert')
|
||||||
var gonna = require('gonna')
|
var gonna = require('gonna')
|
||||||
|
|
||||||
@@ -7,6 +9,7 @@ var concat = require('concat-stream')
|
|||||||
var pg = require('pg')
|
var pg = require('pg')
|
||||||
|
|
||||||
var copy = require('../').to
|
var copy = require('../').to
|
||||||
|
var code = require('../message-formats')
|
||||||
|
|
||||||
var client = function() {
|
var client = function() {
|
||||||
var client = new pg.Client()
|
var client = new pg.Client()
|
||||||
@@ -22,6 +25,18 @@ var testConstruction = function() {
|
|||||||
|
|
||||||
testConstruction()
|
testConstruction()
|
||||||
|
|
||||||
|
var testComparators = function() {
|
||||||
|
var copy1 = copy();
|
||||||
|
copy1.pipe(concat(function(buf) {
|
||||||
|
assert(copy1._gotCopyOutResponse, 'should have received CopyOutResponse')
|
||||||
|
assert(!copy1._remainder, 'Message with no additional data (len=Int4Len+0) should not leave a remainder')
|
||||||
|
}))
|
||||||
|
copy1.end(Buffer.from([code.CopyOutResponse, 0x00, 0x00, 0x00, 0x04]));
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
testComparators();
|
||||||
|
|
||||||
var testRange = function(top) {
|
var testRange = function(top) {
|
||||||
var fromClient = client()
|
var fromClient = client()
|
||||||
var txt = 'COPY (SELECT * from generate_series(0, ' + (top - 1) + ')) TO STDOUT'
|
var txt = 'COPY (SELECT * from generate_series(0, ' + (top - 1) + ')) TO STDOUT'
|
||||||
@@ -73,5 +88,35 @@ var testInternalPostgresError = function() {
|
|||||||
queryClient.end()
|
queryClient.end()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
testInternalPostgresError()
|
testInternalPostgresError()
|
||||||
|
|
||||||
|
var testNoticeResponse = function() {
|
||||||
|
// we use a special trick to generate a warning
|
||||||
|
// on the copy stream.
|
||||||
|
var queryClient = client()
|
||||||
|
var set = '';
|
||||||
|
set += 'SET SESSION client_min_messages = WARNING;'
|
||||||
|
set += 'SET SESSION standard_conforming_strings = off;'
|
||||||
|
set += 'SET SESSION escape_string_warning = on;'
|
||||||
|
queryClient.query(set, function(err, res) {
|
||||||
|
assert.equal(err, null, 'testNoticeResponse - could not SET parameters')
|
||||||
|
var runStream = function(callback) {
|
||||||
|
var txt = "COPY (SELECT '\\\n') TO STDOUT"
|
||||||
|
var stream = queryClient.query(copy(txt))
|
||||||
|
stream.on('data', function(data) {
|
||||||
|
})
|
||||||
|
stream.on('error', callback)
|
||||||
|
|
||||||
|
// make sure stream is pulled from
|
||||||
|
stream.pipe(concat(callback.bind(null,null)))
|
||||||
|
}
|
||||||
|
|
||||||
|
runStream(function(err) {
|
||||||
|
assert.equal(err, null, err)
|
||||||
|
queryClient.end()
|
||||||
|
})
|
||||||
|
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
testNoticeResponse();
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
require('./copy-from')
|
require('./copy-from')
|
||||||
require('./copy-to')
|
require('./copy-to')
|
||||||
require('./binary')
|
require('./binary')
|
||||||
|
|||||||
Reference in New Issue
Block a user