Nodejs http retry on timeout or error

Question

I'm trying to automatically retry HTTP requests on timeout or error.
Currently my code looks like this:

var req = http.get(url, doStuff)
              .on('error', retry)
              .setTimeout(10000, retry);

However, a single request can sometimes trigger both "on error" and "timeout" events. What is a better way of implementing retry?

Answer 1

I was looking for same thing and found interesting module requestretry, well suited for such requirement.

Here is usage:

var request = require('requestretry')

request({
  url: myURL,
  json: true,
  maxAttempts: 5,  // (default) try 5 times 
  retryDelay: 5000, // (default) wait for 5s before trying again
  retrySrategy: request.RetryStrategies.HTTPOrNetworkError // (default) retry on 5xx or network errors
}, function(err, response, body){
  // this callback will only be called when the request succeeded or after maxAttempts or on error 
  if (response) {
    console.log('The number of request attempts: ' + response.attempts);
  }
})

Answer 2

You could try something like this:

function doRequest(url, callback) {
  var timer,
      req,
      sawResponse = false;
  req = http.get(url, callback)
            .on('error', function(err) {
              clearTimeout(timer);
              req.abort();
              // prevent multiple execution of `callback` if error after
              // response
              if (!sawResponse)
                doRequest(url, callback);
            }).on('socket', function(sock) {
              timer = setTimeout(function() {
                req.abort();
                doRequest(url, callback);
              }, 10000);
            }).once('response', function(res) {
              sawResponse = true;
              clearTimeout(timer);
            });
}

UPDATE: In recent/modern versions of node, you can now specify a timeout option (measured in milliseconds) which sets the socket timeout (before the socket is connected). For example:

http.get({
 host: 'example.org',
 path: '/foo',
 timeout: 5000
}, (res) => {
  // ...
});

Answer 3

Using Request-promise, Why not just use a while loop inside an async anonymous function:

(async () => {
        var download_success = false;
        while (!download_success) {             
            await requestpromise(options)
            .then(function (response) {
               download_success = true;
               console.log(`Download SUCCESS`);                                          
            })
            .catch(function (err) {
               console.log(`Error downloading : ${err.message}`);                      
        });
    }
})();

Also note that a requestretry module exists.

Answer 4

This was the code that worked for me. The key was to destroy the socket after timeout as well as to check that the response is complete.

function httpGet(url, callback) {
    var retry = function(e) {
        console.log("Got error: " + e.message);
        httpGet(url, callback); //retry
    }

    var req = http.get(url, function(res) {
        var body = new Buffer(0);
        res.on('data', function (chunk) {
            body = Buffer.concat([body, chunk]);
        });
        res.on('end', function () {
            if(this.complete) callback(body);
            else retry({message: "Incomplete response"});
        });
    }).on('error', retry)
    .setTimeout(20000, function(thing){
        this.socket.destroy();
    });
}