Large object support for PostgreSQL clients (node-postgres)
npm install pg-large-objectnode-pg-large-object
====================
Large object support for PostgreSQL clients using the node-postgres library.
The API of this library resembles the JDBC library for PostgreSQL.
Installation
------------
```
npm install --save pg-large-object
You will also need to install either the pg library, or the pg-promise library:
``
npm install --save pgor
npm install --save pg-promise
Some of the methods in this library require PostgreSQL 9.3 (server) and up:
* LargeObject.seek()
* LargeObject.tell()
* LargeObject.size()
* LargeObject.truncate()
All other methods should work on PostgreSQL 8.4 and up.
Large Objects
-------------
Large Objects in PostgreSQL lets you store files/objects up to 4 TiB in size. The main benefit
of using Large Objects instead of a simple column is that the data can be read and written in
chunks (e.g. as a stream), instead of having to load the entire column into memory.
Examples
--------
The easiest way to use this library is in combination with promises and pg-promise. This
library exposes a callback style interface (for backwards compatibility) and a promise style
interface (see API Documentation). All functions that end with "Async" will return a promise
javascript
const pgp = require('pg-promise')();
const {LargeObjectManager} = require('pg-large-object');
const {createWriteStream} = require('fs');const db = pgp('postgres://postgres:1234@localhost/postgres');
// When working with Large Objects, always use a transaction
db.tx(tx => {
const man = new LargeObjectManager({pgPromise: tx});
// A LargeObject oid, probably stored somewhere in one of your own tables.
const oid = 123;
// If you are on a high latency connection and working with
// large LargeObjects, you should increase the buffer size.
// The buffer should be divisible by 2048 for best performance
// (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
const bufferSize = 16384;
return man.openAndReadableStreamAsync(oid, bufferSize)
.then(([size, stream]) => {
console.log('Streaming a large object with a total size of', size);
// Store it as an image
const fileStream = createWriteStream('my-file.png');
stream.pipe(fileStream);
return new Promise((resolve, reject) => {
stream.on('end', resolve);
stream.on('error', reject);
});
});
})
.then(() => {
console.log('Done!');
})
.catch(error => {
console.log('Something went horribly wrong!', error);
});
`$3
`javascript
const pgp = require('pg-promise')();
const {LargeObjectManager} = require('pg-large-object');
const {createReadStream} = require('fs');const db = pgp('postgres://postgres:1234@localhost/postgres');
// When working with Large Objects, always use a transaction
db.tx(tx => {
const man = new LargeObjectManager({pgPromise: tx});
// If you are on a high latency connection and working with
// large LargeObjects, you should increase the buffer size.
// The buffer should be divisible by 2048 for best performance
// (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
const bufferSize = 16384;
return man.createAndWritableStreamAsync(bufferSize)
.then(([oid, stream]) => {
// The server has generated an oid
console.log('Creating a large object with the oid', oid);
const fileStream = createReadStream('upload-my-file.png');
fileStream.pipe(stream);
return new Promise((resolve, reject) => {
stream.on('finish', resolve);
stream.on('error', reject);
});
});
})
.then(() => {
console.log('Done!');
})
.catch(error => {
console.log('Something went horribly wrong!', error);
});
`
$3
`javascript
var pg = require('pg');
var LargeObjectManager = require('pg-large-object').LargeObjectManager;
var conString = "postgres://postgres:1234@localhost/postgres";pg.connect(conString, function(err, client, done)
{
if (err)
{
return console.error('could not connect to postgres', err);
}
var man = new LargeObjectManager({pg: client});
// When working with Large Objects, always use a transaction
client.query('BEGIN', function(err, result)
{
if (err)
{
done(err);
return client.emit('error', err);
}
// A LargeObject oid, probably stored somewhere in one of your own tables.
var oid = 123;
// If you are on a high latency connection and working with
// large LargeObjects, you should increase the buffer size.
// The buffer should be divisible by 2048 for best performance
// (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
var bufferSize = 16384;
man.openAndReadableStream(oid, bufferSize, function(err, size, stream)
{
if (err)
{
done(err);
return console.error('Unable to read the given large object', err);
}
console.log('Streaming a large object with a total size of', size);
stream.on('end', function()
{
client.query('COMMIT', done);
});
// Store it as an image
var fileStream = require('fs').createWriteStream('my-file.png');
stream.pipe(fileStream);
});
});
});
`
$3
`javascript
var pg = require('pg');
var LargeObjectManager = require('pg-large-object').LargeObjectManager;
var conString = "postgres://postgres:1234@localhost/postgres";pg.connect(conString, function(err, client, done)
{
if (err)
{
return console.error('could not connect to postgres', err);
}
var man = new LargeObjectManager({pg: client});
// When working with Large Objects, always use a transaction
client.query('BEGIN', function(err, result)
{
if (err)
{
done(err);
return client.emit('error', err);
}
// If you are on a high latency connection and working with
// large LargeObjects, you should increase the buffer size.
// The buffer should be divisible by 2048 for best performance
// (2048 is the default page size in PostgreSQL, see LOBLKSIZE)
var bufferSize = 16384;
man.createAndWritableStream(bufferSize, function(err, oid, stream)
{
if (err)
{
done(err);
return console.error('Unable to create a new large object', err);
}
// The server has generated an oid
console.log('Creating a large object with the oid', oid);
stream.on('finish', function()
{
// Actual writing of the large object in DB may
// take some time, so one should provide a
// callback to client.query.
client.query('COMMIT', done);
});
// Upload an image
var fileStream = require('fs').createReadStream('upload-my-file.png');
fileStream.pipe(stream);
});
});
});
`$3
`javascript
var pg = require('pg');
var LargeObjectManager = require('pg-large-object').LargeObjectManager;
var LargeObject = require('pg-large-object').LargeObject;
var conString = "postgres://postgres:1234@localhost/postgres";pg.connect(conString, function(err, client, done)
{
if (err)
{
return console.error('could not connect to postgres', err);
}
var man = new LargeObjectManager({pg: client});
// When working with Large Objects, always use a transaction
client.query('BEGIN', function(err, result)
{
if (err)
{
done(err);
return client.emit('error');
}
// A LargeObject oid, probably stored somewhere in one of your own tables.
var oid = 123;
// Open with READWRITE if you would like to use
// write() and truncate()
man.open(oid, LargeObjectManager.READ, function(err, obj)
{
if (err)
{
done(err);
return console.error(
'Unable to open the given large object',
oid,
err);
}
// Read the first 50 bytes
obj.read(50, function(err, buf)
{
// buf is a standard node.js Buffer
console.log(buf.toString('hex'));
});
// pg uses a query queue, this guarantees the LargeObject
// will be executed in the order you call them, even if you do not
// wait on the callbacks.
// In this library the callback for methods that only return an error
// is optional (such as for seek below). If you do not give a callback
// and an error occurs, this error will me emit()ted on the client object.
// Set the position to byte 5000
obj.seek(5000, LargeObject.SEEK_SET);
obj.tell(function(err, position)
{
console.log(err, position); // 5000
});
obj.size(function(err, size)
{
console.log(err, size); // The size of the entire LargeObject
});
// Done with the object, close it
obj.close();
client.query('COMMIT', done);
});
});
});
`Testing
-------
You can test this library by running:
`
npm install pg-large-object
npm test
`The test assumes that postgres://nodetest:nodetest@localhost/nodetest is a valid database.
You also need to place a large file named "test.jpg" in the test folder.
API Documentation
-----------------
Modules
- pg-large-object
- pg-large-object/lib/LargeObject
Represents an opened large object.
- pg-large-object/lib/LargeObjectManager
This class lets you use the Large Object functionality of PostgreSQL.
All usage of Large Object should take place within a transaction block!
(BEGIN ... COMMIT)
- pg-large-object/lib/promiseFromCallback ⇒
Promise
- pg-large-object/lib/ReadStream ⇐
stream.Readable
- pg-large-object/lib/WriteStream ⇐
stream.Writable
pg-large-object
* pg-large-object
* .LargeObjectManager : function
* .LargeObject : function
* .ReadStream : function
* .WriteStream : function
$3
pg-large-object/lib/LargeObjectManagerKind: static constant of pg-large-object
$3
pg-large-object/lib/LargeObjectKind: static constant of pg-large-object
$3
pg-large-object/lib/ReadStreamKind: static constant of pg-large-object
$3
pg-large-object/lib/WriteStreamKind: static constant of pg-large-object
pg-large-object/lib/LargeObject
Represents an opened large object.
* pg-large-object/lib/LargeObject
* _instance_
* [.close([callback])](#module_pg-large-object/lib/LargeObject+close)
* .closeAsync() ⇒ Promise
* .read(length, callback)
* .readAsync(length) ⇒ Promise.<Buffer>
* [.write(buffer, [callback])](#module_pg-large-object/lib/LargeObject+write)
* .writeAsync(buffer) ⇒ Promise
* [.seek(position, ref, [callback])](#module_pg-large-object/lib/LargeObject+seek)
* .seekAsync(position, ref) ⇒ Promise.<number>
* .tell(callback)
* .tellAsync() ⇒ Promise.<number>
* .size(callback)
* .sizeAsync() ⇒ Promise.<number>
* [.truncate(length, [callback])](#module_pg-large-object/lib/LargeObject+truncate)
* .truncateAsync(length) ⇒ Promise
* [.getReadableStream([bufferSize])](#module_pg-large-object/lib/LargeObject+getReadableStream) ⇒ pg-large-object/lib/ReadStream
* [.getWritableStream([bufferSize])](#module_pg-large-object/lib/LargeObject+getWritableStream) ⇒ pg-large-object/lib/WriteStream
* _static_
* .SEEK_SET : Number
* .SEEK_CUR : Number
* .SEEK_END : Number
* _inner_
* ~closeCallback : function
* ~readCallback : function
* ~writeCallback : function
* ~seekCallback : function
* ~tellCallback : function
* ~sizeCallback : function
* ~truncateCallback : function
$3
Closes this large object.
You should no longer call any methods on this object.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type |
| --- | --- |
| [callback] | closeCallback |
$3
Closes this large object.
You should no longer call any methods on this object.Kind: instance method of pg-large-object/lib/LargeObject
$3
Reads some data from the large object.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type | Description |
| --- | --- | --- |
| length | Number | How many bytes to read |
| callback | readCallback | |
$3
Reads some data from the large object.Kind: instance method of pg-large-object/lib/LargeObject
Returns: Promise.<Buffer> - The binary data that was read.
If the length of this buffer is less than the supplied
length param, there is no more data to be read.
| Param | Type | Description |
| --- | --- | --- |
| length | Number | How many bytes to read |
$3
Writes some data to the large object.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type | Description |
| --- | --- | --- |
| buffer | Buffer | data to write |
| [callback] | writeCallback | |
$3
Writes some data to the large object.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type | Description |
| --- | --- | --- |
| buffer | Buffer | data to write |
$3
Sets the position within the large object.
Beware floating point rounding with values greater than 2^53 (8192 TiB)Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type | Description |
| --- | --- | --- |
| position | Number | |
| ref | Number | One of SEEK_SET, SEEK_CUR, SEEK_END |
| [callback] | seekCallback | |
$3
Sets the position within the large object.
Beware floating point rounding with values greater than 2^53 (8192 TiB)Kind: instance method of pg-large-object/lib/LargeObject
Returns: Promise.<number> - The new position
| Param | Type | Description |
| --- | --- | --- |
| position | Number | |
| ref | Number | One of SEEK_SET, SEEK_CUR, SEEK_END |
$3
Retrieves the current position within the large object.
Beware floating point rounding with values greater than 2^53 (8192 TiB)Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type |
| --- | --- |
| callback | tellCallback |
$3
Retrieves the current position within the large object.
Beware floating point rounding with values greater than 2^53 (8192 TiB)Kind: instance method of pg-large-object/lib/LargeObject
$3
Find the total size of the large object.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type |
| --- | --- |
| callback | sizeCallback |
$3
Find the total size of the large object.Kind: instance method of pg-large-object/lib/LargeObject
$3
Truncates the large object to the given length in bytes.
If the number of bytes is larger than the current large
object length, the large object will be filled with zero
bytes. This method does not modify the current file offset.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type |
| --- | --- |
| length | Number |
| [callback] | truncateCallback |
$3
Truncates the large object to the given length in bytes.
If the number of bytes is larger than the current large
object length, the large object will be filled with zero
bytes. This method does not modify the current file offset.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type |
| --- | --- |
| length | Number |
$3
Return a stream to read this large object.
Call this within a transaction block.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type | Default | Description |
| --- | --- | --- | --- |
| [bufferSize] | Number | 16384 | A larger buffer size will require more memory on both the server and client, however it will make transfers faster because there is less overhead (less read calls to the server). his overhead is most noticeable on high latency connections because each ransfered chunk will incur at least RTT of additional transfer time. |
$3
Return a stream to write to this large object.
Call this within a transaction block.Kind: instance method of pg-large-object/lib/LargeObject
| Param | Type | Default | Description |
| --- | --- | --- | --- |
| [bufferSize] | Number | 16384 | A larger buffer size will require more memory on both the server and client, however it will make transfers faster because there is less overhead (less read calls to the server). his overhead is most noticeable on high latency connections because each ransfered chunk will incur at least RTT of additional transfer time. |
$3
A seek from the beginning of a objectKind: static constant of pg-large-object/lib/LargeObject
$3
A seek from the current positionKind: static constant of pg-large-object/lib/LargeObject
$3
A seek from the end of a objectKind: static constant of pg-large-object/lib/LargeObject
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| data | Buffer | The binary data that was read. If the length of this buffer is less than the supplied length param, there is no more data to be read. |
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| position | Number | The new position |
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| position | Number | The position |
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| size | Number | Object size in bytes |
$3
Kind: inner typedef of pg-large-object/lib/LargeObject | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
pg-large-object/lib/LargeObjectManager
This class lets you use the Large Object functionality of PostgreSQL.
All usage of Large Object should take place within a transaction block!
(BEGIN ... COMMIT)
| Param | Type | Description |
| --- | --- | --- |
| options | object | Either
pg or pgPromise must be given |
| options.pg | module:pg/Client | A pg (https://www.npmjs.com/package/pg) Client object |
| options.pgPromise | module:pg-promise/Task | A pg-promise (https://www.npmjs.com/package/pg-promise) transaction object as given by db.tx() |Example
`js
new LargeObjectManager(client)
``* pg-large-object/lib/LargeObjectManager
* _instance_
* .open(oid, mode, callback)
* .openAsync(oid, mode) ⇒ Promise.<pg-large-object/lib/LargeObject>
* .create(callback)
* .createAsync() ⇒ Promise.<number>
* [.unlink(oid, [callback])](#module_pg-large-object/lib/LargeObjectManager+unlink)
* .unlinkAsync(oid) ⇒ Promise
* [.openAndReadableStream(oid, [bufferSize], callback)](#module_pg-large-object/lib/LargeObjectManager+openAndReadableStream)
* [.openAndReadableStreamAsync(oid, [bufferSize])](#module_pg-large-object/lib/LargeObjectManager+openAndReadableStreamAsync) ⇒ Promise.<Array>
* [.createAndWritableStream([bufferSize], [callback])](#module_pg-large-object/lib/LargeObjectManager+createAndWritableStream)
* [.createAndWritableStreamAsync([bufferSize])](#module_pg-large-object/lib/LargeObjectManager+createAndWritableStreamAsync) ⇒ promise.<Array>
* _static_
* .WRITE : Number
* .READ : Number
* .READWRITE : Number
* _inner_
* ~openCallback : function
* ~createCallback : function
* ~unlinkCallback : function
* ~openAndReadableStreamCallback : function
* ~createAndWritableStreamCallback : function
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type | Description |
| --- | --- | --- |
| oid | Number | |
| mode | Number | One of WRITE, READ, or READWRITE |
| callback | openCallback | |
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type | Description |
| --- | --- | --- |
| oid | Number | |
| mode | Number | One of WRITE, READ, or READWRITE |
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type |
| --- | --- |
| callback | createCallback |
Kind: instance method of pg-large-object/lib/LargeObjectManager
Returns: Promise.<number> - oid
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type |
| --- | --- |
| oid | number |
| [callback] | unlinkCallback |
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type |
| --- | --- |
| oid | number |
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type | Default |
| --- | --- | --- |
| oid | Number | |
| [bufferSize] | Number | 16384 |
| callback | openAndReadableStreamCallback | |
Kind: instance method of pg-large-object/lib/LargeObjectManager
Returns: Promise.<Array> - The total size and a ReadStream
| Param | Type | Default |
| --- | --- | --- |
| oid | Number | |
| [bufferSize] | Number | 16384 |
Kind: instance method of pg-large-object/lib/LargeObjectManager
| Param | Type | Default |
| --- | --- | --- |
| [bufferSize] | Number | 16384 |
| [callback] | createAndWritableStreamCallback | |
Kind: instance method of pg-large-object/lib/LargeObjectManager
Returns: promise.<Array> - The oid and a WriteStream
| Param | Type | Default |
| --- | --- | --- |
| [bufferSize] | Number | 16384 |
pg-large-object/lib/LargeObjectManager pg-large-object/lib/LargeObjectManager pg-large-object/lib/LargeObjectManager pg-large-object/lib/LargeObjectManager | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| result | pg-large-object/lib/LargeObject | |
pg-large-object/lib/LargeObjectManager | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| oid | Number | |
pg-large-object/lib/LargeObjectManager | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
pg-large-object/lib/LargeObjectManager | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| size | Number | The total size of the large object |
| stream | pg-large-object/lib/ReadStream | |
pg-large-object/lib/LargeObjectManager | Param | Type | Description |
| --- | --- | --- |
| error | Error | If set, an error occurred. |
| oid | Number | |
| stream | pg-large-object/lib/WriteStream | |
Promise| Param | Type |
| --- | --- |
| fn | function |
| self | object |
| [options] | object |
stream.Readablestream.Readable stream.Writablestream.Writable