Compare commits
5 Commits
v1.2-carto
...
PushSpring
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be7af371d8 | ||
|
|
17697e98d7 | ||
|
|
bd4a87d3a0 | ||
|
|
9d197a91e1 | ||
|
|
e1ce9a3948 |
@@ -1,8 +1,9 @@
|
|||||||
language: node_js
|
language: node_js
|
||||||
node_js:
|
node_js:
|
||||||
|
- "0.12"
|
||||||
|
- "4"
|
||||||
|
- "5"
|
||||||
- "6"
|
- "6"
|
||||||
- "8"
|
|
||||||
- "10"
|
|
||||||
|
|
||||||
addons:
|
addons:
|
||||||
postgresql: "9.2"
|
postgresql: "9.2"
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
# CARTO's Changelog
|
|
||||||
|
|
||||||
## v1.2.0-carto.4
|
|
||||||
Released 2018-mm-dd
|
|
||||||
|
|
||||||
## v1.2.0-carto.3
|
|
||||||
Released 2018-11-21
|
|
||||||
|
|
||||||
Features:
|
|
||||||
* Drop support for Node.js 0.12, 4 and, 5.
|
|
||||||
* Add support for Node.js 8 and 10.
|
|
||||||
* Add package-lock.json
|
|
||||||
* Do not use deprecated Buffer constructors.
|
|
||||||
|
|
||||||
## v1.2.0-carto.2
|
|
||||||
Released 2018-10-26
|
|
||||||
|
|
||||||
Bug fixes:
|
|
||||||
* Make all modules to use strict mode semantics.
|
|
||||||
|
|
||||||
## v1.2.0-carto.1
|
|
||||||
Released 2018-06-11
|
|
||||||
|
|
||||||
Bug fixes:
|
|
||||||
* Improves performance of COPY TO by sending bigger chunks through low level `push()`. See https://github.com/CartoDB/node-pg-copy-streams/pull/1
|
|
||||||
|
|
||||||
## v1.2.0
|
|
||||||
Released 2016-08-22
|
|
||||||
|
|
||||||
Vanilla version v1.2.0 from upstream repository. See https://github.com/CartoDB/node-pg-copy-streams/releases/tag/v1.2.0
|
|
||||||
16
README.md
16
README.md
@@ -1,7 +1,3 @@
|
|||||||
## Note
|
|
||||||
|
|
||||||
This is forked repository from [brianc/node-pg-copy-streams](https://github.com/brianc/node-pg-query-stream)
|
|
||||||
|
|
||||||
## pg-copy-streams
|
## pg-copy-streams
|
||||||
|
|
||||||
[](https://travis-ci.org/brianc/node-pg-copy-streams)
|
[](https://travis-ci.org/brianc/node-pg-copy-streams)
|
||||||
@@ -46,15 +42,10 @@ pg.connect(function(err, client, done) {
|
|||||||
var stream = client.query(copyFrom('COPY my_table FROM STDIN'));
|
var stream = client.query(copyFrom('COPY my_table FROM STDIN'));
|
||||||
var fileStream = fs.createReadStream('some_file.tsv')
|
var fileStream = fs.createReadStream('some_file.tsv')
|
||||||
fileStream.on('error', done);
|
fileStream.on('error', done);
|
||||||
stream.on('error', done);
|
fileStream.pipe(stream).on('finish', done).on('error', done);
|
||||||
stream.on('end', done);
|
|
||||||
fileStream.pipe(stream);
|
|
||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
*Important*: Even if `pg-copy-streams.from` is used as a Writable (via `pipe`), you should not listen for the 'finish' event and expect that the COPY command has already been correctly acknowledged by the database. Internally, a duplex stream is used to pipe the data into the database connection and the COPY command should be considered complete only when the 'end' event is triggered.
|
|
||||||
|
|
||||||
|
|
||||||
## install
|
## install
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -65,10 +56,7 @@ $ npm install pg-copy-streams
|
|||||||
|
|
||||||
This module __only__ works with the pure JavaScript bindings. If you're using `require('pg').native` please make sure to use normal `require('pg')` or `require('pg.js')` when you're using copy streams.
|
This module __only__ works with the pure JavaScript bindings. If you're using `require('pg').native` please make sure to use normal `require('pg')` or `require('pg.js')` when you're using copy streams.
|
||||||
|
|
||||||
Before you set out on this magical piping journey, you _really_ should read this: http://www.postgresql.org/docs/current/static/sql-copy.html, and you might want to take a look at the [tests](https://github.com/brianc/node-pg-copy-streams/tree/master/test) to get an idea of how things work.
|
Before you set out on this magical piping journey, you _really_ should read this: http://www.postgresql.org/docs/9.3/static/sql-copy.html, and you might want to take a look at the [tests](https://github.com/brianc/node-pg-copy-streams/tree/master/test) to get an idea of how things work.
|
||||||
|
|
||||||
Take note of the following warning in the PostgreSQL documentation:
|
|
||||||
> COPY stops operation at the first error. This should not lead to problems in the event of a COPY TO, but the target table will already have received earlier rows in a COPY FROM. These rows will not be visible or accessible, but they still occupy disk space. This might amount to a considerable amount of wasted disk space if the failure happened well into a large copy operation. You might wish to invoke VACUUM to recover the wasted space.
|
|
||||||
|
|
||||||
## contributing
|
## contributing
|
||||||
|
|
||||||
|
|||||||
107
copy-to.js
107
copy-to.js
@@ -1,17 +1,14 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
module.exports = function(txt, options) {
|
module.exports = function(txt, options) {
|
||||||
return new CopyStreamQuery(txt, options)
|
return new CopyStreamQuery(txt, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
var Transform = require('stream').Transform
|
var Transform = require('stream').Transform
|
||||||
var util = require('util')
|
var util = require('util')
|
||||||
var code = require('./message-formats')
|
|
||||||
|
|
||||||
var CopyStreamQuery = function(text, options) {
|
var CopyStreamQuery = function(text, options) {
|
||||||
Transform.call(this, options)
|
Transform.call(this, options)
|
||||||
this.text = text
|
this.text = text
|
||||||
this._gotCopyOutResponse = false
|
this._copyOutResponse = null
|
||||||
this.rowCount = 0
|
this.rowCount = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -26,89 +23,63 @@ CopyStreamQuery.prototype.submit = function(connection) {
|
|||||||
connection.stream.pipe(this)
|
connection.stream.pipe(this)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var code = {
|
||||||
|
E: 69, //Error
|
||||||
|
H: 72, //CopyOutResponse
|
||||||
|
d: 0x64, //CopyData
|
||||||
|
c: 0x63 //CopyDone
|
||||||
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype._detach = function() {
|
CopyStreamQuery.prototype._detach = function() {
|
||||||
this.connection.stream.unpipe(this)
|
this.connection.stream.unpipe(this)
|
||||||
// Unpipe can drop us out of flowing mode
|
// Unpipe can drop us out of flowing mode
|
||||||
this.connection.stream.resume()
|
this.connection.stream.resume()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
||||||
var offset = 0
|
var offset = 0
|
||||||
var Byte1Len = 1;
|
|
||||||
var Int32Len = 4;
|
|
||||||
if(this._remainder && chunk) {
|
if(this._remainder && chunk) {
|
||||||
chunk = Buffer.concat([this._remainder, chunk])
|
chunk = Buffer.concat([this._remainder, chunk])
|
||||||
}
|
}
|
||||||
|
if(!this._copyOutResponse) {
|
||||||
var length;
|
this._copyOutResponse = true
|
||||||
var messageCode;
|
if(chunk[0] == code.E) {
|
||||||
var needPush = false;
|
this._detach()
|
||||||
|
this.push(null)
|
||||||
var buffer = Buffer.alloc(chunk.length);
|
return cb();
|
||||||
var buffer_offset = 0;
|
|
||||||
|
|
||||||
this.pushBufferIfneeded = function() {
|
|
||||||
if (needPush && buffer_offset > 0) {
|
|
||||||
this.push(buffer.slice(0, buffer_offset))
|
|
||||||
buffer_offset = 0;
|
|
||||||
}
|
}
|
||||||
|
if(chunk[0] != code.H) {
|
||||||
|
this.emit('error', new Error('Expected copy out response'))
|
||||||
|
}
|
||||||
|
var length = chunk.readUInt32BE(1)
|
||||||
|
offset = 1
|
||||||
|
offset += length
|
||||||
}
|
}
|
||||||
|
while((chunk.length - offset) > 5) {
|
||||||
while((chunk.length - offset) >= (Byte1Len + Int32Len)) {
|
|
||||||
var messageCode = chunk[offset]
|
var messageCode = chunk[offset]
|
||||||
|
//complete or error
|
||||||
//console.log('PostgreSQL message ' + String.fromCharCode(messageCode))
|
if(messageCode == code.c || messageCode == code.E) {
|
||||||
switch(messageCode) {
|
this._detach()
|
||||||
|
this.push(null)
|
||||||
// detect COPY start
|
return cb();
|
||||||
case code.CopyOutResponse:
|
|
||||||
if (!this._gotCopyOutResponse) {
|
|
||||||
this._gotCopyOutResponse = true
|
|
||||||
} else {
|
|
||||||
this.emit('error', new Error('Unexpected CopyOutResponse message (H)'))
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
|
|
||||||
// meaningful row
|
|
||||||
case code.CopyData:
|
|
||||||
needPush = true;
|
|
||||||
break;
|
|
||||||
|
|
||||||
// standard interspersed messages. discard
|
|
||||||
case code.ParameterStatus:
|
|
||||||
case code.NoticeResponse:
|
|
||||||
case code.NotificationResponse:
|
|
||||||
break;
|
|
||||||
|
|
||||||
case code.ErrorResponse:
|
|
||||||
case code.CopyDone:
|
|
||||||
this.pushBufferIfneeded();
|
|
||||||
this._detach()
|
|
||||||
this.push(null)
|
|
||||||
return cb();
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
this.emit('error', new Error('Unexpected PostgreSQL message ' + String.fromCharCode(messageCode)))
|
|
||||||
}
|
}
|
||||||
|
//something bad happened
|
||||||
length = chunk.readUInt32BE(offset+Byte1Len)
|
if(messageCode != code.d) {
|
||||||
if(chunk.length >= (offset + Byte1Len + length)) {
|
return this.emit('error', new Error('expected "d" (copydata message)'))
|
||||||
offset += Byte1Len + Int32Len
|
}
|
||||||
if (needPush) {
|
var length = chunk.readUInt32BE(offset + 1) - 4 //subtract length of UInt32
|
||||||
var row = chunk.slice(offset, offset + length - Int32Len)
|
//can we read the next row?
|
||||||
this.rowCount++
|
if(chunk.length > (offset + length + 5)) {
|
||||||
row.copy(buffer, buffer_offset);
|
offset += 5
|
||||||
buffer_offset += row.length;
|
var slice = chunk.slice(offset, offset + length)
|
||||||
}
|
offset += length
|
||||||
offset += (length - Int32Len)
|
this.push(slice)
|
||||||
|
this.rowCount++
|
||||||
} else {
|
} else {
|
||||||
// we need more chunks for a complete message
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.pushBufferIfneeded();
|
|
||||||
|
|
||||||
if(chunk.length - offset) {
|
if(chunk.length - offset) {
|
||||||
var slice = chunk.slice(offset)
|
var slice = chunk.slice(offset)
|
||||||
this._remainder = slice
|
this._remainder = slice
|
||||||
|
|||||||
42
index.js
42
index.js
@@ -1,5 +1,3 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var CopyToQueryStream = require('./copy-to')
|
var CopyToQueryStream = require('./copy-to')
|
||||||
module.exports = {
|
module.exports = {
|
||||||
to: function(txt, options) {
|
to: function(txt, options) {
|
||||||
@@ -12,7 +10,6 @@ module.exports = {
|
|||||||
|
|
||||||
var Transform = require('stream').Transform
|
var Transform = require('stream').Transform
|
||||||
var util = require('util')
|
var util = require('util')
|
||||||
var code = require('./message-formats')
|
|
||||||
|
|
||||||
var CopyStreamQuery = function(text, options) {
|
var CopyStreamQuery = function(text, options) {
|
||||||
Transform.call(this, options)
|
Transform.call(this, options)
|
||||||
@@ -29,23 +26,28 @@ CopyStreamQuery.prototype.submit = function(connection) {
|
|||||||
connection.query(this.text)
|
connection.query(this.text)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var code = {
|
||||||
|
H: 72, //CopyOutResponse
|
||||||
|
d: 0x64, //CopyData
|
||||||
|
c: 0x63 //CopyDone
|
||||||
|
}
|
||||||
|
|
||||||
var copyDataBuffer = Buffer.from([code.CopyData])
|
var copyDataBuffer = Buffer([code.d])
|
||||||
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
CopyStreamQuery.prototype._transform = function(chunk, enc, cb) {
|
||||||
var Int32Len = 4;
|
|
||||||
this.push(copyDataBuffer)
|
this.push(copyDataBuffer)
|
||||||
var lenBuffer = Buffer.alloc(Int32Len)
|
var lenBuffer = Buffer(4)
|
||||||
lenBuffer.writeUInt32BE(chunk.length + Int32Len, 0)
|
lenBuffer.writeUInt32BE(chunk.length + 4, 0)
|
||||||
this.push(lenBuffer)
|
this.push(lenBuffer)
|
||||||
this.push(chunk)
|
this.push(chunk)
|
||||||
|
this.rowCount++
|
||||||
cb()
|
cb()
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype._flush = function(cb) {
|
CopyStreamQuery.prototype._flush = function(cb) {
|
||||||
var Int32Len = 4;
|
var finBuffer = Buffer([code.c, 0, 0, 0, 4])
|
||||||
var finBuffer = Buffer.from([code.CopyDone, 0, 0, 0, Int32Len])
|
|
||||||
this.push(finBuffer)
|
this.push(finBuffer)
|
||||||
this.cb_flush = cb
|
//never call this callback, do not close underlying stream
|
||||||
|
//cb()
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleError = function(e) {
|
CopyStreamQuery.prototype.handleError = function(e) {
|
||||||
@@ -53,24 +55,12 @@ CopyStreamQuery.prototype.handleError = function(e) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleCopyInResponse = function(connection) {
|
CopyStreamQuery.prototype.handleCopyInResponse = function(connection) {
|
||||||
this.pipe(connection.stream, { end: false })
|
this.pipe(connection.stream)
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleCommandComplete = function(msg) {
|
CopyStreamQuery.prototype.handleCommandComplete = function() {
|
||||||
// Parse affected row count as in
|
this.unpipe()
|
||||||
// https://github.com/brianc/node-postgres/blob/35e5567f86774f808c2a8518dd312b8aa3586693/lib/result.js#L37
|
this.emit('end')
|
||||||
var match = /COPY (\d+)/.exec((msg || {}).text)
|
|
||||||
if (match) {
|
|
||||||
this.rowCount = parseInt(match[1], 10)
|
|
||||||
}
|
|
||||||
|
|
||||||
// we delay the _flush cb so that the 'end' event is
|
|
||||||
// triggered after CommandComplete
|
|
||||||
this.cb_flush()
|
|
||||||
|
|
||||||
// unpipe from connection
|
|
||||||
this.unpipe(this.connection)
|
|
||||||
this.connection = null
|
|
||||||
}
|
}
|
||||||
|
|
||||||
CopyStreamQuery.prototype.handleReadyForQuery = function() {
|
CopyStreamQuery.prototype.handleReadyForQuery = function() {
|
||||||
|
|||||||
@@ -1,25 +0,0 @@
|
|||||||
/**
|
|
||||||
* The COPY feature uses the following protocol codes.
|
|
||||||
* The codes for the most recent protocol version are documented on
|
|
||||||
* https://www.postgresql.org/docs/current/static/protocol-message-formats.html
|
|
||||||
*
|
|
||||||
* The protocol flow itself is described on
|
|
||||||
* https://www.postgresql.org/docs/current/static/protocol-flow.html
|
|
||||||
*/
|
|
||||||
module.exports = {
|
|
||||||
ErrorResponse: 0x45, // E
|
|
||||||
CopyInResponse: 0x47, // G
|
|
||||||
CopyOutResponse: 0x48, // H
|
|
||||||
CopyBothResponse: 0x57, // W
|
|
||||||
CopyDone: 0x63, // c
|
|
||||||
CopyData: 0x64, // d
|
|
||||||
CopyFail: 0x66, // f
|
|
||||||
|
|
||||||
// It is possible for NoticeResponse and ParameterStatus messages to be interspersed between CopyData messages;
|
|
||||||
// frontends must handle these cases, and should be prepared for other asynchronous message types as well
|
|
||||||
// (see Section 50.2.6).
|
|
||||||
// Otherwise, any message type other than CopyData or CopyDone may be treated as terminating copy-out mode.
|
|
||||||
NotificationResponse: 0x41, // A
|
|
||||||
NoticeResponse: 0x4E, // N
|
|
||||||
ParameterStatus: 0x53 // S
|
|
||||||
}
|
|
||||||
193
package-lock.json
generated
193
package-lock.json
generated
@@ -1,193 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "pg-copy-streams",
|
|
||||||
"version": "1.2.0-carto.4",
|
|
||||||
"lockfileVersion": 1,
|
|
||||||
"requires": true,
|
|
||||||
"dependencies": {
|
|
||||||
"async": {
|
|
||||||
"version": "0.2.10",
|
|
||||||
"resolved": "http://registry.npmjs.org/async/-/async-0.2.10.tgz",
|
|
||||||
"integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"base64-js": {
|
|
||||||
"version": "0.0.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-0.0.2.tgz",
|
|
||||||
"integrity": "sha1-Ak8Pcq+iW3X5wO5zzU9V7Bvtl4Q=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"bops": {
|
|
||||||
"version": "0.0.6",
|
|
||||||
"resolved": "https://registry.npmjs.org/bops/-/bops-0.0.6.tgz",
|
|
||||||
"integrity": "sha1-CC0dVfoB5g29wuvC26N/ZZVUzzo=",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"base64-js": "0.0.2",
|
|
||||||
"to-utf8": "0.0.1"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"buffer-writer": {
|
|
||||||
"version": "1.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
|
|
||||||
"integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"concat-stream": {
|
|
||||||
"version": "1.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.1.0.tgz",
|
|
||||||
"integrity": "sha1-hCae/YzGUCdeMi8wnfRIZ7xRxfM=",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"bops": "0.0.6"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"generic-pool": {
|
|
||||||
"version": "2.1.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz",
|
|
||||||
"integrity": "sha1-rwTcLDJc/Ll1Aj+lK/zpYXp0Nf0=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"gonna": {
|
|
||||||
"version": "0.0.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/gonna/-/gonna-0.0.0.tgz",
|
|
||||||
"integrity": "sha1-6k4ZsVJ6F4LhJQVeMCSabUvHmlk=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"heroku-env": {
|
|
||||||
"version": "0.1.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/heroku-env/-/heroku-env-0.1.1.tgz",
|
|
||||||
"integrity": "sha1-wGeRyUTpuHSOMXf1S/cBQyZ+Yxc=",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"parse-database-url": "~0.2.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"lodash": {
|
|
||||||
"version": "2.2.1",
|
|
||||||
"resolved": "http://registry.npmjs.org/lodash/-/lodash-2.2.1.tgz",
|
|
||||||
"integrity": "sha1-ypNf0UqzwMhyq6zxmLnNpQFECGc=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"packet-reader": {
|
|
||||||
"version": "0.2.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.2.0.tgz",
|
|
||||||
"integrity": "sha1-gZ300BC4LV6lZx+KGjrPA5vNdwA=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"parse-database-url": {
|
|
||||||
"version": "0.2.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/parse-database-url/-/parse-database-url-0.2.2.tgz",
|
|
||||||
"integrity": "sha1-SGFa56fA/HfjKU0jVCpqUnPDVws=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"pg": {
|
|
||||||
"version": "4.4.6",
|
|
||||||
"resolved": "http://registry.npmjs.org/pg/-/pg-4.4.6.tgz",
|
|
||||||
"integrity": "sha1-EZgiP7rva6QRqm9Q4X9OtaGTFVk=",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"buffer-writer": "1.0.1",
|
|
||||||
"generic-pool": "2.1.1",
|
|
||||||
"packet-reader": "0.2.0",
|
|
||||||
"pg-connection-string": "0.1.3",
|
|
||||||
"pg-types": "1.*",
|
|
||||||
"pgpass": "0.0.3",
|
|
||||||
"semver": "^4.1.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"pg-connection-string": {
|
|
||||||
"version": "0.1.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
|
|
||||||
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"pg-int8": {
|
|
||||||
"version": "1.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
|
|
||||||
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"pg-types": {
|
|
||||||
"version": "1.13.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.13.0.tgz",
|
|
||||||
"integrity": "sha512-lfKli0Gkl/+za/+b6lzENajczwZHc7D5kiUCZfgm914jipD2kIOIvEkAhZ8GrW3/TUoP9w8FHjwpPObBye5KQQ==",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"pg-int8": "1.0.1",
|
|
||||||
"postgres-array": "~1.0.0",
|
|
||||||
"postgres-bytea": "~1.0.0",
|
|
||||||
"postgres-date": "~1.0.0",
|
|
||||||
"postgres-interval": "^1.1.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"pgpass": {
|
|
||||||
"version": "0.0.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-0.0.3.tgz",
|
|
||||||
"integrity": "sha1-EuZ+NDsxicLzEgbrycwL7//PkUA=",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"split": "~0.3"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"postgres-array": {
|
|
||||||
"version": "1.0.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz",
|
|
||||||
"integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ==",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"postgres-bytea": {
|
|
||||||
"version": "1.0.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
|
|
||||||
"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"postgres-date": {
|
|
||||||
"version": "1.0.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz",
|
|
||||||
"integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"postgres-interval": {
|
|
||||||
"version": "1.1.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz",
|
|
||||||
"integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"xtend": "^4.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"semver": {
|
|
||||||
"version": "4.3.6",
|
|
||||||
"resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
|
|
||||||
"integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"split": {
|
|
||||||
"version": "0.3.3",
|
|
||||||
"resolved": "http://registry.npmjs.org/split/-/split-0.3.3.tgz",
|
|
||||||
"integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=",
|
|
||||||
"dev": true,
|
|
||||||
"requires": {
|
|
||||||
"through": "2"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"through": {
|
|
||||||
"version": "2.3.8",
|
|
||||||
"resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
|
||||||
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"to-utf8": {
|
|
||||||
"version": "0.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/to-utf8/-/to-utf8-0.0.1.tgz",
|
|
||||||
"integrity": "sha1-0Xrqcv8vujm55DYBvns/9y4ImFI=",
|
|
||||||
"dev": true
|
|
||||||
},
|
|
||||||
"xtend": {
|
|
||||||
"version": "4.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
|
|
||||||
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
|
|
||||||
"dev": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "pg-copy-streams",
|
"name": "pg-copy-streams",
|
||||||
"version": "1.2.0-carto.4",
|
"version": "0.3.0",
|
||||||
"description": "Low-Level COPY TO and COPY FROM streams for PostgreSQL in JavaScript using",
|
"description": "Low-Level COPY TO and COPY FROM streams for PostgreSQL in JavaScript using",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var assert = require('assert')
|
var assert = require('assert')
|
||||||
var gonna = require('gonna')
|
var gonna = require('gonna')
|
||||||
|
|
||||||
@@ -21,7 +19,7 @@ var testBinaryCopy = function() {
|
|||||||
var fromClient = client()
|
var fromClient = client()
|
||||||
var toClient = client()
|
var toClient = client()
|
||||||
|
|
||||||
var queries = [
|
queries = [
|
||||||
'DROP TABLE IF EXISTS data',
|
'DROP TABLE IF EXISTS data',
|
||||||
'CREATE TABLE IF NOT EXISTS data (num BIGINT, word TEXT)',
|
'CREATE TABLE IF NOT EXISTS data (num BIGINT, word TEXT)',
|
||||||
'INSERT INTO data (num, word) VALUES (1, \'hello\'), (2, \'other thing\'), (3, \'goodbye\')',
|
'INSERT INTO data (num, word) VALUES (1, \'hello\'), (2, \'other thing\'), (3, \'goodbye\')',
|
||||||
@@ -35,7 +33,7 @@ var testBinaryCopy = function() {
|
|||||||
var fromStream = fromClient.query(to('COPY (SELECT * FROM data) TO STDOUT BINARY'))
|
var fromStream = fromClient.query(to('COPY (SELECT * FROM data) TO STDOUT BINARY'))
|
||||||
var toStream = toClient.query(from('COPY data_copy FROM STDIN BINARY'))
|
var toStream = toClient.query(from('COPY data_copy FROM STDIN BINARY'))
|
||||||
|
|
||||||
var runStream = function(callback) {
|
runStream = function(callback) {
|
||||||
fromStream.on('error', callback)
|
fromStream.on('error', callback)
|
||||||
toStream.on('error', callback)
|
toStream.on('error', callback)
|
||||||
toStream.on('finish', callback)
|
toStream.on('finish', callback)
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var assert = require('assert')
|
var assert = require('assert')
|
||||||
var gonna = require('gonna')
|
var gonna = require('gonna')
|
||||||
|
|
||||||
@@ -32,8 +30,12 @@ var testRange = function(top) {
|
|||||||
|
|
||||||
var txt = 'COPY numbers FROM STDIN'
|
var txt = 'COPY numbers FROM STDIN'
|
||||||
var stream = fromClient.query(copy(txt))
|
var stream = fromClient.query(copy(txt))
|
||||||
|
var rowEmitCount = 0
|
||||||
|
stream.on('row', function() {
|
||||||
|
rowEmitCount++
|
||||||
|
})
|
||||||
for(var i = 0; i < top; i++) {
|
for(var i = 0; i < top; i++) {
|
||||||
stream.write(Buffer.from('' + i + '\t' + i*10 + '\n'))
|
stream.write(Buffer('' + i + '\t' + i*10 + '\n'))
|
||||||
}
|
}
|
||||||
stream.end()
|
stream.end()
|
||||||
var countDone = gonna('have correct count')
|
var countDone = gonna('have correct count')
|
||||||
@@ -41,7 +43,6 @@ var testRange = function(top) {
|
|||||||
fromClient.query('SELECT COUNT(*) FROM numbers', function(err, res) {
|
fromClient.query('SELECT COUNT(*) FROM numbers', function(err, res) {
|
||||||
assert.ifError(err)
|
assert.ifError(err)
|
||||||
assert.equal(res.rows[0].count, top, 'expected ' + top + ' rows but got ' + res.rows[0].count)
|
assert.equal(res.rows[0].count, top, 'expected ' + top + ' rows but got ' + res.rows[0].count)
|
||||||
assert.equal(stream.rowCount, top, 'expected ' + top + ' rows but db count is ' + stream.rowCount)
|
|
||||||
//console.log('found ', res.rows.length, 'rows')
|
//console.log('found ', res.rows.length, 'rows')
|
||||||
countDone()
|
countDone()
|
||||||
var firstRowDone = gonna('have correct result')
|
var firstRowDone = gonna('have correct result')
|
||||||
@@ -57,19 +58,3 @@ var testRange = function(top) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
testRange(1000)
|
testRange(1000)
|
||||||
|
|
||||||
var testSingleEnd = function() {
|
|
||||||
var fromClient = client()
|
|
||||||
fromClient.query('CREATE TEMP TABLE numbers(num int)')
|
|
||||||
var txt = 'COPY numbers FROM STDIN';
|
|
||||||
var stream = fromClient.query(copy(txt))
|
|
||||||
var count = 0;
|
|
||||||
stream.on('end', function() {
|
|
||||||
count++;
|
|
||||||
assert(count==1, '`end` Event was triggered ' + count + ' times');
|
|
||||||
if (count == 1) fromClient.end();
|
|
||||||
})
|
|
||||||
stream.end(Buffer.from('1\n'))
|
|
||||||
|
|
||||||
}
|
|
||||||
testSingleEnd()
|
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
var assert = require('assert')
|
var assert = require('assert')
|
||||||
var gonna = require('gonna')
|
var gonna = require('gonna')
|
||||||
|
|
||||||
@@ -9,7 +7,6 @@ var concat = require('concat-stream')
|
|||||||
var pg = require('pg')
|
var pg = require('pg')
|
||||||
|
|
||||||
var copy = require('../').to
|
var copy = require('../').to
|
||||||
var code = require('../message-formats')
|
|
||||||
|
|
||||||
var client = function() {
|
var client = function() {
|
||||||
var client = new pg.Client()
|
var client = new pg.Client()
|
||||||
@@ -25,18 +22,6 @@ var testConstruction = function() {
|
|||||||
|
|
||||||
testConstruction()
|
testConstruction()
|
||||||
|
|
||||||
var testComparators = function() {
|
|
||||||
var copy1 = copy();
|
|
||||||
copy1.pipe(concat(function(buf) {
|
|
||||||
assert(copy1._gotCopyOutResponse, 'should have received CopyOutResponse')
|
|
||||||
assert(!copy1._remainder, 'Message with no additional data (len=Int4Len+0) should not leave a remainder')
|
|
||||||
}))
|
|
||||||
copy1.end(Buffer.from([code.CopyOutResponse, 0x00, 0x00, 0x00, 0x04]));
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
testComparators();
|
|
||||||
|
|
||||||
var testRange = function(top) {
|
var testRange = function(top) {
|
||||||
var fromClient = client()
|
var fromClient = client()
|
||||||
var txt = 'COPY (SELECT * from generate_series(0, ' + (top - 1) + ')) TO STDOUT'
|
var txt = 'COPY (SELECT * from generate_series(0, ' + (top - 1) + ')) TO STDOUT'
|
||||||
@@ -88,35 +73,5 @@ var testInternalPostgresError = function() {
|
|||||||
queryClient.end()
|
queryClient.end()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
testInternalPostgresError()
|
testInternalPostgresError()
|
||||||
|
|
||||||
var testNoticeResponse = function() {
|
|
||||||
// we use a special trick to generate a warning
|
|
||||||
// on the copy stream.
|
|
||||||
var queryClient = client()
|
|
||||||
var set = '';
|
|
||||||
set += 'SET SESSION client_min_messages = WARNING;'
|
|
||||||
set += 'SET SESSION standard_conforming_strings = off;'
|
|
||||||
set += 'SET SESSION escape_string_warning = on;'
|
|
||||||
queryClient.query(set, function(err, res) {
|
|
||||||
assert.equal(err, null, 'testNoticeResponse - could not SET parameters')
|
|
||||||
var runStream = function(callback) {
|
|
||||||
var txt = "COPY (SELECT '\\\n') TO STDOUT"
|
|
||||||
var stream = queryClient.query(copy(txt))
|
|
||||||
stream.on('data', function(data) {
|
|
||||||
})
|
|
||||||
stream.on('error', callback)
|
|
||||||
|
|
||||||
// make sure stream is pulled from
|
|
||||||
stream.pipe(concat(callback.bind(null,null)))
|
|
||||||
}
|
|
||||||
|
|
||||||
runStream(function(err) {
|
|
||||||
assert.equal(err, null, err)
|
|
||||||
queryClient.end()
|
|
||||||
})
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
testNoticeResponse();
|
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
'use strict';
|
|
||||||
|
|
||||||
require('./copy-from')
|
require('./copy-from')
|
||||||
require('./copy-to')
|
require('./copy-to')
|
||||||
require('./binary')
|
require('./binary')
|
||||||
|
|||||||
Reference in New Issue
Block a user