Logo Questions Linux Laravel Mysql Ubuntu Git Menu
 

How to slow down this Node looping http request?

I'm trying to call this API with the request module about 200-300 times with a Lambda function. I need to add second between each call so I don't get a 429 response. I've tried a few different ways to make this happen, but it seems to ignore the code to slow it down.

How do people normally slow down these requests in AWS lambda? It would be great if I could insert something like utilities.sleep(1000) in the loop to make it wait a second before continuing. I'm sure there is a simple solution to this issue, but all the examples I've seen seem to make it complex.

function findProjects(items){

    var toggleData = [];

    for( var i = 0; i < items.length; i++ ){
        setTimeout( callToggle( items[i] ), 1000 );
    }

    function callToggle( data ){
        request({
            'url': 'https://www.toggl.com/api/v8/projects/' + data.toggle.data.id,
            'method': 'GET',
            'headers': {
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            },
            'auth': {
                'user': 'xxxxxxx',
                'pass': 'api_token'
        }}, function( error, response, body ){
            if( error ) {

                console.log( error );
                context.done( null, error );
            } else {

                console.log(response.statusCode, "toggle projects were listed");
                var info = JSON.parse(body);
                toggleData.push(info);
            }
        });
    }

    findDocument( toggleData );   
}
like image 996
Brit Gwaltney Avatar asked Aug 31 '16 18:08

Brit Gwaltney


2 Answers

You can do something like this:

for(var i = 0; i<items.length; i++){
    setTimeout(callToggl, 1000 + (( i * X ) % Y), items[i]);
}

Where Y is the max delay (1000 + Y) then you want (es 5 sec) and X is the timing for each call (es X=10 : 1000,1010,1020,1030,...

if you want 1s each call:

for(var i = 0; i<items.length; i++){
    setTimeout(callToggl(items[i]), 1000 + ( i * 1000 ));
}

EDIT

for(var i = 0; i<items.length; i++){
    setTimeout(callToggl, 1000 + ( i * 1000 ), items[i]);
}
like image 192
gianlucatursi Avatar answered Oct 24 '22 01:10

gianlucatursi


You can chain the requests together:

function findProjects(items){

    var toggleData = [];

    // chain requests with delay
    items.reduce(function (requestChain, item) {
      return requestChain
        .then(callToggle.bind(null, item))
        .then(wait.bind(null, 1000));
    }, Promise.resolve());

    function wait (ms) {
      return new Promise(function (resolve, reject) {
        setTimeout(resolve, ms);
      });
    }

    function callToggle(data) {
        request({
            'url': 'https://www.toggl.com/api/v8/projects/' + data.toggle.data.id,
            'method': 'GET',
            'headers': {
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            },
            'auth': {
                'user': 'xxxxxxx',
                'pass': 'api_token'
        }}, function( error, response, body ){
            if( error ) {

                console.log( error );
                context.done( null, error );
            } else {

                console.log(response.statusCode, "toggle projects were listed");
                var info = JSON.parse(body);
                toggleData.push(info);
            }
        });
    }

    findDocument( toggleData );   
}
like image 21
William B Avatar answered Oct 24 '22 00:10

William B