Node JS - How to write stream big json data into json array file?

Question

I have difficult to write a json data into json file using stream module. I learn about this from several blog tutorial, one of them is this page

Let say i am working with big json data on a json file. I think it is not possible to store all json object inside my memory. So i decided to do it using stream module.

Here the codes i have done:

writeStream.js

var Writable = require('stream').Writable,
util = require('util');

var WriteStream = function() {
    Writable.call(this, {
        objectMode: true
    });
};

util.inherits(WriteStream, Writable);

WriteStream.prototype._write = function(chunk, encoding, callback) {
    console.log('write            : ' + JSON.stringify(chunk));
    callback();
};

module.exports = WriteStream;

readStream.js

var data = require('./test_data.json'),
Readable = require('stream').Readable,
util = require('util');

var ReadStream = function() {
    Readable.call(this, {
        objectMode: true
    });

    this.data = data;
    this.curIndex = 0;
};

util.inherits(ReadStream, Readable);

ReadStream.prototype._read = function() {
    if (this.curIndex === this.data.length) {
        return this.push(null);
    }

    var data = this.data[this.curIndex++];
    console.log('read             : ' + JSON.stringify(data));
    this.push(data);
};

module.exports = ReadStream;

Called with this code:

var ReadStream = require('./readStream.js'),
WriteStream = require('./writeStream.js');

var rs = new ReadStream();
var ws = new WriteStream();
rs.pipe(ws);

Problem: I want to write it into different file, how is it possible? Can you please help me?

Answer 1

If you are looking for a solution to just write the data from your ReadStream into a different file, you can try fs.createWriteStream. It will return you a writeable stream which can be piped directly to your ReadStream.

You will have to make a minor change in your readStream.js. You are currently pushing an object thus making it an object stream while a write stream expects either String or Buffer unless started in the ObjectMode. So you can do one of the following:

• Start the write stream in the object mode. More info here.
• Push String or Buffer in your read stream as writable stream internally calls writable.write which expects either String or Buffer. More info here.

If we follow the second option as an example, then your readStream.js should look like this:

var data = require('./test_data.json'),
Readable = require('stream').Readable,
util = require('util');

var ReadStream = function() {
    Readable.call(this, {
        objectMode: true
    });

    this.data = data;
    this.curIndex = 0;
};

util.inherits(ReadStream, Readable);

ReadStream.prototype._read = function() {
    if (this.curIndex === this.data.length) {
        return this.push(null);
    }

    var data = this.data[this.curIndex++];
    console.log('read             : ' + JSON.stringify(data));
    this.push(JSON.stringify(data));
};

module.exports = ReadStream;

You can call the above by using the following code

var ReadStream = require('./readStream.js');
const fs = require('fs');

var rs = new ReadStream();
const file = fs.createWriteStream('/path/to/output/file');
rs.pipe(file);

This will write the data from test_data.json to the output file.

Also as a good practice and to reliably detect write errors, add a listener for the 'error' event. For the above code, you can add the following:

file.on('error',function(err){
  console.log("err:", err);
});

Hope this helps.