node.js stream pipe asynchronous or synchronous

Question

Problem: This is the problem as the one in learnyounode module JugglingAsync of node.js. "This problem is the same as the previous problem (HTTP COLLECT) in that you need to use http.get(). However, this time you will be provided with three URLs as the first three command-line arguments.

You must collect the complete content provided to you by each of the URLs and print it to the console (stdout). You don't need to print out the length, just the data as a String; one line per URL. The catch is that you must print them out in the same order as the URLs are provided to you as command line arguments."

I tried to use node.js stream.readable class to pipe response from first URL to the second and response from that to the third one. I was expecting that this would run synchronously i.e. when the first request is completed then the second request would be piped. I am using package bl (https://www.npmjs.org/package/bl) to collect all the response data for a get request. Code Snippet below:

var https = require('http');
var bl = require('bl');
var finalString = '';

https.get( process.argv[2], function(response)
    {
        response.setEncoding('utf8');
        response.pipe(bl(function (err, data)
        {
            console.log("First request called");
            if (err) return console.error(err);
            console.log(data.toString());

        })).pipe(bl(function(err, data)
        {

            console.log("Second Request called");
            https.get (process.argv[3], function( response)
            {
                response.setEncoding('utf8');
                response.pipe(bl( function (err, data)
                {
                    if (err) return console.error(err);                    
                    console.log(data.toString());

                }))
            }).on('error', function(err)
                {
                    console.log(err);
                })
        })).pipe( bl(function(err,data)
            {
                console.log("Third request called");
                https.get (process.argv[4], function( response)
                {

                    response.setEncoding('utf8');
                    response.pipe(bl( function (err, data)
                    {
                        if (err) return console.error(err);                        
                        console.log(data.toString());

                    }))

                }).on('error', function(err)
                    {
                        console.log(err);
                    })
            })
            )
    }).on('error', function (err)
    {
            console.log(err);
    }
);

The output does not follow the sequence of request. What am I doing wrong?

Answer 1

As far as I know pipe in Streams {in nodejs} are asynchronous, but I have read somewhere that they can act as Synchronous as well. Your Solution above is running in async manner and thus prints the data in random order.

Your solution can be made workable as follows :-

var https = require('http');
var bl = require('bl');
var finalData = [];
var count = 0;

https.get( process.argv[2], function(response)
    {
        response.setEncoding('utf8');
        response.pipe(bl(function (err, data)
        {
            // console.log("First request called");
            if (err) return console.error(err);
            finalData[0] = data.toString();
            count++;
            printThemOut(count);

        })).pipe(bl(function(err, data)
        {

            // console.log("Second Request called");
            https.get (process.argv[3], function( response)
            {
                response.setEncoding('utf8');
                response.pipe(bl( function (err, data)
                {
                    if (err) return console.error(err);
                    finalData[1] = data.toString();
                    count++;
                    printThemOut(count);

                }))
            }).on('error', function(err)
                {
                    console.log(err);
                })
        })).pipe( bl(function(err,data)
            {
                // console.log("Third request called");
                https.get (process.argv[4], function( response)
                {

                    response.setEncoding('utf8');
                    response.pipe(bl( function (err, data)
                    {
                        if (err) return console.error(err);
                        finalData[2] = data.toString();
                        count++;
                        printThemOut(count);
                    }))

                }).on('error', function(err)
                    {
                        console.log(err);
                    })
            })
            )
    }).on('error', function (err)
    {
            console.log(err);
    }



);

function printThemOut(count) {
  if (count == 3) {
    for (var i = 0; i < finalData.length; i++) {
      console.log(finalData[i]);
    }
  }
}

Above solution works, but the problem is (in my case), that the first data on first URL call, the first word gets lost.

{I think it's becoz of using multiple pipes on the same data for first URL}

Explaination

We need a counter for indexing the data fetched by http.get(), so that first URL data gets stored at index 0 and second in 1 index and so on. This helps in syncing all async data...

Proper Solution

Proper solution would be to loop the call and using single pipe on each data...

var http  = require("http");
var bl    = require("bl");

var completeData = [];
var count = 0;
for (var i = 2; i < process.argv.length; i++) {
  (function(index){
    http.get(process.argv[index], function(res) {
      res.pipe(bl(function(err, data) {
        if (err) {
          return console.log(err);
        }
        // console.log(index);
        completeData[index-2] = data.toString();
        count++;
        // console.log("Data: " + data.toString());
        if (count === process.argv.length-2) {
          printThemOut(completeData);
        }
      }))
    })
  })(i);
}

function printThemOut(data) {
  for (var i = 0; i < data.length; i++) {
    console.log(data[i]);
  }
}