How to do repeated requests until one succeeds without blocking in node?

Question

I have a function that takes a parameter and a callback. It's supposed to do a request to a remote API and get some info based on the parameter. When it gets the info, it needs to send it to the callback. Now, the remote API sometimes fails to provide. I need my function to keep trying until it manages to do it and then call the callback with the correct data.

Currently, I have the below code inside the function but I think that stuff like while (!done); isn't proper node code.

var history = {};
while (true) {
    var done = false;
    var retry = true;
    var req = https.request(options, function(res) {
        var acc = "";
        res.on("data", function(msg) {
            acc += msg.toString("utf-8");
        });
        res.on("end", function() {
            done = true;
            history = JSON.parse(acc);
            if (history.success) {
                retry = false;
            }
        });
    });
    req.end();
    while (!done);
    if (!retry) break;
}
callback(history);

How do I do it the right way?

Answer 1

There is no need to re-invent the wheel... you can use a popular async utility library, 'retry' method in this case.

// try calling apiMethod 3 times
async.retry(3, apiMethod, function(err, result) {
    // do something with the result
});

// try calling apiMethod 3 times, waiting 200 ms between each retry
async.retry({times: 3, interval: 200}, apiMethod, function(err, result) {
    // do something with the result
});

async GitHub page

async.retry docs

Answer 2

Definitely not the way to go - while(!done); will go into a hard loop and take up all of your cpu.

Instead you could do something like this (untested and you may want to implement a back-off of some sort):

function tryUntilSuccess(options, callback) {
    var req = https.request(options, function(res) {
        var acc = "";
        res.on("data", function(msg) {
            acc += msg.toString("utf-8");
        });
        res.on("end", function() {
            var history = JSON.parse(acc);  //<== Protect this if you may not get JSON back
            if (history.success) {
                callback(null, history);
            } else {
                tryUntilSuccess(options, callback);
            }
        });
    });
    req.end();

    req.on('error', function(e) {
        // Decide what to do here
        // if error is recoverable
        //     tryUntilSuccess(options, callback);
        // else
        //     callback(e);
    });
}

// Use the standard callback pattern of err in first param, success in second
tryUntilSuccess(options, function(err, resp) {
    // Your code here...
});

Answer 3

I found Dmitry's answer using the async utility library very useful and the best answer.

This answer expands his example to a working version that defines the apiMethod function and passes it a parameter. I was going to add the code as a comment but a separate answer is clearer.

const async = require('async');

const apiMethod = function(uri, callback) {
  try {
    // Call your api here (or whatever thing you want to do) and assign to result.
    const result = ...
    callback(null, result);
  } catch (err) {
    callback(err);
  }
};

const uri = 'http://www.test.com/api';

async.retry(
    { times: 5, interval: 200 },
    function (callback) { return apiMethod(uri, callback) },
    function(err, result) {
      if (err) {
        throw err; // Error still thrown after retrying N times, so rethrow.
      }
  });

Retry documentation: https://caolan.github.io/async/docs.html#retry

Note, an alternative to calling apiMethod(uri, callback) in the task is to use async.apply:

async.retry(
        {times: 5, interval: 200},
        async.apply(task, dir),
        function(err, result) {
          if (err) {
            throw err; // Error still thrown after retrying N times, so rethrow.
          }
      });

I hope this provides a good copy/paste boiler plate solution for someone.