setTimeout inside iteration of iteration

Question

I have the code below and i would like to put a setTimeout between each iteration of Myurl. There are a number of classes and each of them contains a number of elements.

//Some calculations before...
var i = 0;
async.whilst(
function () {
    return i <= thefooz.length - 1;
},
function (innerCallback) {

    //Some calculations where I get classes array.

    async.forEachOfSeries(classes, function (Myurl, m, eachDone) {
        // Here I want a delay
        async.waterfall([
            function (next) {
                connection.query(
                    'SELECT * FROM mydata WHERE UrlLink=? LIMIT 1', [Myurl],
                    next
                );
            },
            function (results, fields, next) {
                if (results.length !== 0) {
                    console.log("Already Present");
                    return next();
                }
                console.log("New Thing!");
                request(options2, function (err, resp, body) {
                    if (!err && resp.statusCode == 200) {
                        var $ = cheerio.load(body);
                        //Some calculations, where I get AllLinks.
                        var post = {
                            ThisUrl: AllLinks[0],
                            Time: AllLinks[1],
                        };
                        var query = connection.query('Insert INTO mydata Set ?', post, next);
                    };
                });
            }
        ], eachDone);

    }, function (err) {
        if (err) throw err;
    });
    setTimeout(function () {
        i++;
        innerCallback();
        console.log("Done");
    }, 20000);

    //Some calculations after...

So how could I set a delay between each Myurl in async.waterfall? Say I want a 5 seconds delay. I managed to set setTimeout between each async.whilstiteration but not between each async.forEachOfSeries iteration. It simply does not wait, instead it continues to loop until each async.forEachOfSeries is done and then calls the async.whilst setTimeout.

EDIT: Queue solution does not work. That solution seems to just go to next page, and next page and so on, without outputting to my database. Of course I could apply it in the wrong way, but I really tried to do exactly as the example said.

Answer 1

First we must implement a simple queue

Queue

function Queue() {
  var obj = {};  
  var queue = [];
  var _delay;
  
  function next() {
    // If queue is empty stops execution
    if(queue.length == 0) return;
    
    // Prepare next call to next
    setTimeout(next, _delay);
    // Take out an element from the queue and execute it.
    (queue.shift())();          
  }
  
  // Add a new function to the queue
  obj.add = function (myFunc) {
    queue.push(myFunc);
  };
  
  // Start the queue execution passing the delay between each call
  obj.run = function(delay) {
    _delay = delay;
    
    // call next function
    next();
  }
  
  return obj;
  
}

Then we use it inside the code

// create the queue
var myQueue = Queue();

async.forEachOfSeries(classes, function (Myurl, m, eachDone) {
  // Add the function to the queue
  myQueue.add(executeWaterfall.bind(this));
}, function (err) {
  if (err) throw err;
});

// Start the queue with 5 second delay
myQueue.run(5000);

function executeWaterfall() {
  async.waterfall([
    function (next) {
      connection.query(
        'SELECT * FROM mydata WHERE UrlLink=? LIMIT 1', [Myurl],
        next
      );
    },
    function (results, fields, next) {
      if (results.length !== 0) {
        console.log("Already Present");
        return next();
      }
      console.log("New Thing!");
      request(options2, function (err, resp, body) {
        if (!err && resp.statusCode == 200) {
          var $ = cheerio.load(body);
          //Some calculations, where I get AllLinks.
          var post = {
            ThisUrl: AllLinks[0],
            Time: AllLinks[1],
          };
          var query = connection.query('Insert INTO mydata Set ?', post, next);
        };
      });
    }
  ], eachDone);
}

This is far from optimal because anyway you are falling in what is called the Pyramid of doom

Bonus

Pyramid of doom

When handling asynchronous operations in succession with normal callbacks, you'll end up nesting calls within each other; with this nesting comes more indentation, creating a pyramid (pointing to the right), hence the name "Pyramid of Doom".

Solution

In this case is better to use some promise pattern to save your code from the pyramid of doom and facilitate the solution of this kind of issues.

• More Info
• PromiseJS

Answer 2

I think you don't fully understand how setTimeout works:

(function () {
    var seconds=0;
[1,2,3].forEach(function(value) {
    setTimeout(function() {
        console.log('Showing value '+value+ 'at '+Date());
    },1000*seconds++);
})
})()

This code, for each element, creates a callback function to be executed after a second. Please note that JS is single threaded, so what the code really does is adding "executions" to a queue. So if the current execution does not stop, the callbacks are not called. So the time (in millis) that you pass to the setTimeout function as second parameter is just the minimum time to have that code executed.

Then, the execution of those callbacks is made in FIFO order.

UPDATE: Here is an example of what I'm explaining:

function myFunction() { 
    var test=0;
    setTimeout(function(){
        console.log("This is the current value of test: "+test); 
    }, 0);
    console.log("This is run first");
    for (var i=0;i<50;i++) {
        test++;
    }
    console.log("Main thread ending, not the callbacks will be executed");
} 

The setTimeout will wait 0 (zero) before executing, but as the main thread has not finished, it cannot be executed. Then, when the loop ends, the callback is executed, finding that test is 50, not 0.