Node.js wait for callback of REST service that makes HTTP request

Question

I am using the express module to build a RESTful API within Node.js. In my service I am making additional HTTP requests to external endpoints (server side). I need to return the data from those HTTP requests to the request body of my Web service.

On all the actions the Web service is conducting, by using console.log() I have confirmed that I get the data I need. However, when I try to return those values to the service they come back with value null. I know this is because a request is asynchronous and the callback is not waiting for the HTTP request to finish.

Is there a way to make this work?

Answer 1

A common practice is to use the async module.

npm install async

The async module has primitives to handle various forms of asynchronous events.

In your case, the async#parallel call will allow you to make requests to all external APIs at the same time and then combine the results for return to the requester.

Since you're making external http requests, you will probably find the request module helpful as well.

npm install request

Using request and async#parallel your route handler would look something like this...

var request = require('request');
var async = require('async');

exports.handler = function(req, res) {
  async.parallel([
    /*
     * First external endpoint
     */
    function(callback) {
      var url = "http://external1.com/api/some_endpoint";
      request(url, function(err, response, body) {
        // JSON body
        if(err) { console.log(err); callback(true); return; }
        obj = JSON.parse(body);
        callback(false, obj);
      });
    },
    /*
     * Second external endpoint
     */
    function(callback) {
      var url = "http://external2.com/api/some_endpoint";
      request(url, function(err, response, body) {
        // JSON body
        if(err) { console.log(err); callback(true); return; }
        obj = JSON.parse(body);
        callback(false, obj);
      });
    },
  ],
  /*
   * Collate results
   */
  function(err, results) {
    if(err) { console.log(err); res.send(500,"Server Error"); return; }
    res.send({api1:results[0], api2:results[1]});
  }
  );
};

You can also read about other callback sequencing methods here.

Answer 2

Node.js is all about callbacks. Unless the API call is synchronous (rare and shouldn't be done) you never return values from those calls, but callback with the result from within the callback method, or call the express method res.send

A great library for invoking web requests is request.js

Let's take the really simple example of calling google. Using res.send, your express.js code could look like:

var request = require('request');
app.get('/callGoogle', function(req, res){
  request('http://www.google.com', function (error, response, body) {
    if (!error && response.statusCode == 200) {
      // from within the callback, write data to response, essentially returning it.
      res.send(body);
    }
  })
});

Alternatively, you can pass a callback to the method that invokes the web request, and invoke that callback from within that method:

app.get('/callGoogle', function(req, res){
  invokeAndProcessGoogleResponse(function(err, result){
    if(err){
      res.send(500, { error: 'something blew up' });
    } else {
      res.send(result);
    }
  });
});

var invokeAndProcessGoogleResponse = function(callback){
  request('http://www.google.com', function (error, response, body) {

    if (!error && response.statusCode == 200) {
      status = "succeeded";
      callback(null, {status : status});
    } else {
      callback(error);
    }
  })
}

Answer 3

Wait.for https://github.com/luciotato/waitfor

Other answer's examples using wait.for:

Example from from Daniel's Answer (async), but using Wait.for

var request = require('request');
var wait = require('wait.for');

exports.handler = function(req, res) {
try {  
    //execute parallel, 2 endpoints, wait for results
    var result = wait.parallel.map(["http://external1.com/api/some_endpoint"
                 ,"http://external2.com/api/some_endpoint"]
                 , request.standardGetJSON); 
    //return result
    res.send(result);
}
catch(err){
    console.log(err); 
    res.end(500,"Server Error")
}
};

//wait.for requires standard callbacks(err,data)
//standardized request.get: 
request.standardGetJSON = function ( options, callback) {
    request.get(options,
            function (error, response, body) {
                //standardized callback
                var data;
                if (!error) data={ response: response, obj:JSON.parse(body)};
                callback(error,data);
            });
}