Getting chunks by newline in Node.js data stream

Question

At one point I thought you could tell Node.js child process to chunk the data by newline character. As is below, the stderr data event from the child process is firing for characters and words, not lines. Ideally I could pass a flag to tell the stream to only fire the data event when a line of data is ready. Isn't there a way to do this?

I have this:

const sh = spawn('sh', [ b ], {
  cwd: cwd,
});

sh.stdout.pipe(fs.createWriteStream('/dev/null'));

var stderr = '';
var line = '';

sh.stderr.setEncoding('utf8');

sh.stderr.on('data', function (d) {

  //trying to split by newlines, but this is hairy logic
  const lines = String(d).split('\n');

  line += lines.shift();

  if(lines.length > 0){

    if (!String(d).match(/npm/ig) && !String(d).match(/npm/ig)) {
      stderr += d;
      process.stderr.write.apply(process.stderr, arguments);
    }

  }

});

and the data come back in this handler is not whole lines

sh.stderr.on('data', function (d) {
   // d is chunks of data but not whole lines
});

isn't there a way to tell stderr to wait for newline chars before firing the 'data' event?

Answer 1

You can use a Transform stream for that.

The implementation is not so trivial, so I would recommend using a library like split2.

Here is the basic idea :

const Transform = require('stream').Transform;
const StringDecoder = require('string_decoder').StringDecoder;

const decoder = new StringDecoder('utf8');

const myTransform = new Transform({
   transform(chunk, encoding, cb) {
      if ( this._last === undefined ) { this._last = "" }
      this._last += decoder.write(chunk);
      var list = this._last.split(/\n/);          
      this._last = list.pop();
      for (var i = 0; i < list.length; i++) {
        this.push( list[i] );
      }
      cb();
  },

  flush(cb) {
      this._last += decoder.end()
      if (this._last) { this.push(this._last) }
      cb()
  }
});

sh.stderr.pipe( myTransform )
         .on('data', function (line) {
              console.log("[" + line + "]");
         });