delay in a for loop for http request

Question

I am just getting started with JS and Node.js. I am trying to build a simple scraper as first project, using Node.js and some modules such as request and cheerio. I would like to add a 5 secs delay between each http request for each domain contained into the array. Can you explain me how to do it?

Here is my code:

var request = require('request');

var arr = [ "http://allrecipes.com/", "http://www.gossip.fr/" ];

for(var i=0; i < arr.length; i++) {
    request(arr[i], function (error, response, body){
        console.log('error:', error);
        console.log('statusCode:', response && response.statusCode);
        console.log('body:', body);
    });
}

Answer 1

Anybody looking for a flashy ES6+ Promises and Async/Await answer can use this.

We are using request-native-promises here.

const rp = require("request-promise-native");
const productID = [0,1,2,3,4,5,6]

//here we make our timeout synchronous using Promises
function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

//run your code in an async block
async function demo() {
  for (let i = 0; i < productID.length; i++) {
    const options = {
      method: "GET",
      url: `https://link_to_your_api/productID[i]`,
      json: true,
    };
    const body = await rp(options);
    console.log(`Awaiting 1s ...`);
    //waiting before sleep function executes using it synchronously
    await sleep(1000);
  }
}
demo();

//since I haven't done any error handling
process.on("unhandledRejection", err => {
  console.log("Unhandled rejection:", err.message);
});

Answer 2

Use setTimeout:

var request = require('request');
var arr = ["http://allrecipes.com/", "http://www.gossip.fr/" ];

for (var i = 0; i < arr.length; i++) {
    setTimeout(request, 5000 * i, arr[i], function (error, response, body){
        console.log('error:', error);
        console.log('statusCode:', response && response.statusCode);
        console.log('body:', body);
    });
}

You basically make the loop and say that the request method should be called with a delay equal to 5000 * i, which is 5000ms multiplied by i, so it will increase with 5 seconds for each loop iteration. Then you provide all the arguments that will be passed to the function.