what should I use instead of readableStream.push('')

Question

I am trying to implement the ._read function of a readable stream, a problem happens when ._read is called and there isn't data, the documentation says that I can push('') until more data comes, and I should only return false when the stream will never have more data. https://nodejs.org/api/stream.html#stream_readable_read_size_1

But it also says that if I need to do that then something is terribly wrong with my design. https://nodejs.org/api/stream.html#stream_stream_push

But I can't find an alternative to that.

code:

    var http = require('http');
    var https = require('https');
    var Readable = require('stream').Readable;
    var router = require('express').Router();
    var buffer = [];

    router.post('/', function(clientRequest, clientResponse) {
        var delayedMSStream = new Readable;
        delayedMSStream._read = function() {
            var a=buffer.shift();
            if(typeof a === 'undefined'){
                this.push('');
                return true;
            }
            else {
                this.push(a);
                if(a===null) {
                    return false;
                }
                return true;
            }
        };
        //I need to get a url from example.com
        https.request({hostname:'example.com'}, function(exampleResponse){
            data='';
            exampleResponse.on('data',function(chunk){data+=chunk});
            exampleResponse.on('end',function(){
                var MSRequestOptions = {hostname: data, method: 'POST'};
                var MSRequest = https.request(MSRequestOptions, function(MSResponse){
                    MSResponse.on('end', function () {
                            console.log("MSResponse.on(end)");//>>>
                    });//end MSResponse.on(end)
                }); //end MSRequest
                delayedMSStream.pipe(MSRequest);
            });
        });

        clientRequest.on('data', function (chunk) {
            buffer.push(chunk);
        });

        clientRequest.on('end', function () {//when done streaming audio
            buffer.push(null);
        }); 
    });//end router.post('/')

explanation: client sends a POST request streaming audio to my server, my server requests a url from example.com, when example.com responds with the url, my server streams the audio to it.

What's a smarter way to do it?

Answer 1

So if I undertstand the code correctly, you:

• receive a request,
• make your own request to a remote endpoint and fetch a URL
• make a new request to that URL and pipe that to original response.

There are ways to do this other then yours, and even your way would look cleaner to me if you just improve the naming a bit. Also, splitting the huge request into a few functions with smaller responsibility scopes might help.

I would make the endpoint this way:

let http = require('http');
let https = require('https');
let Readable = require('stream').Readable;
let router = require('express').Router();
let buffer = [];

/**
 * Gets some data from a remote host. Calls back when done.
 * We cannot pipe this directly into your stream chain as we need the complete data to get the end result.
 */
function getHostname(cb) {

  https.request({

    hostname: 'example.com'
  }, function(response) {

    let data = '';
    response.on('error', err => cb(err)); // shortened for brewity
    response.on('data', function(chunk) {

        data = data + chunk;
    });
    response.on('end', function() {

      // we're done here.
      cb(null, data.toString());
    });
  });
}

router.post('/', function(request, response) {

  // first let's get that url.
  getHostname(function(err, hostname) {

    if (err) { return response.status(500).end(); } 

    // now make that other request which we can stream.
    https.request({

      hostname: hostname,
      method: 'POST'
    }, function(dataStream) {

        dataStream.pipe(response);
    });
  });
});

Now, as said in the comments, with streams2, you don't have to manage your streams. With node versions pre 0.10 you have had to listen to 'read', 'data' etc events, with newer node versions, it's handled. Furthermore, you don't even need it here, streams are smart enough to handle backpressure on their own.