setTimeout inside a loop, stops script from working

Question

My script is receiving data from API and store in MongoDB automatically. I was needed to create a at least 2 second delay before receiving one data after another. The problem is that my script is stop working on second time. Let's say my script working every hour, I enable the script at 14.00 - it works and at 15.00 - it stops. I start researching the problem and come to the point that is problem with setTimeout() inside a loop.

This is an article what I found Watch Out When Using SetTimeout() in For Loop #JS

This line is a Node-Schedule package It basically calls the script every 15th minutes (if someone is wondering that it is)

const j = schedule.scheduleJob('*/15 * * * *', callIt)

My goal: How can I change my code to still having a 2 second's delay and working loop. Is there any alternatives to use instead of setTimeout(), maybe I just need to put a setTimeout() in another place in a code or maybe there is even some type of packages that I can add additional.

Problematic code area:

    var symbols = ["ZRXBTC", "ETHBTC", "ETCBTC", "KAVABTC", "AEBTC"]
];
let cnt = 0;

const callIt = () => {

    fetch(`https://api.binance.com/api/v3/klines?symbol=${symbols[cnt]}&interval=30m&limit=1`)
        .then(res => res.json())
        .then(data => {
            const btcusdtdata = data.map(d => {
                return {
                    Open: parseFloat(d[1]),
                    High: parseFloat(d[2]),
                    Low: parseFloat(d[3]),
                    Close: parseFloat(d[4]),
                    Volume: parseFloat(d[5]),
                    Timespan: 30,
                }
            });
            console.log(btcusdtdata);
            saveToDatebase(btcusdtdata);
            cnt++;
            if (cnt < symbols.length) setTimeout(callIt, 2000)
        })
        .catch((err) => {
            console.log(err);
        })
};

FULL CODE

var requestPromise = require('request-promise');
const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
const fetch = require("node-fetch");

var symbols = ["ZRXBTC", "ETHBTC", "ETCBTC", "KAVABTC", "AEBTC"]
    ];
    let cnt = 0;

    const callIt = () => {

        fetch(`https://api.binance.com/api/v3/klines?symbol=${symbols[cnt]}&interval=30m&limit=1`)
            .then(res => res.json())
            .then(data => {
                const btcusdtdata = data.map(d => {
                    return {
                        Open: parseFloat(d[1]),
                        High: parseFloat(d[2]),
                        Low: parseFloat(d[3]),
                        Close: parseFloat(d[4]),
                        Volume: parseFloat(d[5]),
                        Timespan: 30,
                    }
                });
                console.log(btcusdtdata);
                saveToDatebase(btcusdtdata);
                cnt++;
                if (cnt < symbols.length) setTimeout(callIt, 2000)
            })
            .catch((err) => {
                console.log(err);
            })
    };

const j = schedule.scheduleJob('*/15 * * * *', callIt)

const saveToDatebase = function(BTCdata) {

    const url = 'mongodb+srv://username:[email protected]/<dbname>?retryWrites=true&w=majority';

    var today = new Date();
    var date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
    var time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
    var dateTime = date + ' ' + time;

    MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true }, (err, db) => {
        if (err) throw err;
        const dbo = db.db('CryptoCurrencies');
        const myobj = { Name: symbols[cnt - 1], Array: BTCdata, Date: dateTime };
        dbo.collection(`${symbols[cnt - 1]}`).insertOne(myobj, (error, res) => {
            if (error) throw error;
            console.log('1 document inserted');
            db.close();
        });
    });

};

EDIT1: To be more correct I'll specify the task more accurate. My node-schedule package suppose to call the script every 15 minutes and I want to call properties in array every 2 seconds. I need to call properties from array every 2 seconds otherwise I will receive IP ban from Binance API for calling the API to much/fast.

EDIT2 Alright. setInterval() is not a solution for me. Since I need to call the script every 15 minutes it should go through array and when it call all properties from array it suppose to stop. In setInterval() after calling the all properties in array in starts it again, this is doesn't what I need, unfortunately.

EDIT3: I tested few options from answers below, all of them get me to the same problem that the script cannot start the second time or the script is starts working immediately or repeats even after array properties are ended. Still thank you for answers, but my problem is still one.

Currently I trying to use async/await methods. But I receive an error that await is only valid in async function

EDIT4: So this is a fullcode solution from @yoavmatchulsky. Script starts working, but I don't receive any data or something. It's just like working - but nothing happening. No errors, nothing.

var requestPromise = require('request-promise');
const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
const fetch = require("node-fetch");

const symbols = ["ZRXBTC", "ETHBTC", "ETCBTC", "KAVABTC", "AEBTC"];

const sleep = async(timeout) => {
    return new Promise(resolve => {
        setTimeout(resolve, timeout);
    });
}

const callIt = async(symbol) => {
    return fetch(`https://api.binance.com/api/v3/klines?symbol=${symbol}&interval=30m&limit=1`)
        .then(res => res.json())
        .then(data => async() => {
            const btcusdtdata = data.map(d => {
                return {
                    Open: parseFloat(d[1]),
                    High: parseFloat(d[2]),
                    Low: parseFloat(d[3]),
                    Close: parseFloat(d[4]),
                    Volume: parseFloat(d[5]),
                    Timespan: 30,
                }
            });
            console.log(btcusdtdata);
            await saveToDatebase(btcusdtdata);
        })
        .catch((err) => {
            console.log(err);
        })
};

const saveToDatebase = async function(BTCdata) {
    return new Promise((resolve, reject) => {

        const url = 'mongodb+srv://username:[email protected]/<dbname>?retryWrites=true&w=majority';

        var today = new Date();
        var date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
        var time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
        var dateTime = date + ' ' + time;

        MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true }, (err, db) => {
            if (err) {
                return reject(err);
            }
            const dbo = db.db('CryptoCurrencies');
            const myobj = { Name: symbols[cnt - 1], Array: BTCdata, Date: dateTime };
            dbo.collection(`${symbols[cnt - 1]}`).insertOne(myobj, (error, res) => {
                if (error) {
                    return reject(error);
                }
                console.log('1 document inserted');
                db.close();
                resolve();
            });
        });
    });
};

const run = async() => {
    let cnt = 0;
    while (cnt < symbols.length) {
        await callIt(symbols[cnt]);
        await sleep(2000);
        cnt++;
    }
}

const j = schedule.scheduleJob('*/2 * * * *', run);

Answer 1

I think this problem has been made much more complex than it needs to be. The core problem is simple: You never reset cnt to 0 after the first loop. So when the second loop starts, cnt is still greater than the array size, and it exits early! Let's look at fixing this problem first.

The easiest way is to change your schedule.scheduleJob callback to an anonymous function which resets cnt to 0 and then calls callIt() to do the recursive loop again. From your original code, this is one small change to the scheduleJob callback:

const j = schedule.scheduleJob('*/15 * * * *', () => {
  cnt = 0;
  callIt();
});

With this, cnt will be reset to 0 and your code will work repeatedly correctly.

Others have pointed out that async/await is a good way to make this code simpler and I agree. I'll also note that you're using the callback form of your mongodb functions, but all mongodb functions also return promises. Try the above first to confirm it works, then if you'd like, consider the improvements below.

const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
const fetch = require("node-fetch");

const symbols = ["ZRXBTC", "ETHBTC", "ETCBTC", "KAVABTC", "AEBTC"];

//a descriptive name helps your future self and others understand code easier
const getBTCData = async symbol => {  //make this function accept the current symbol
    //async/await lets us write this much nicer and with less nested indents
    let data = await fetch(`https://api.binance.com/api/v3/klines?symbol=${symbol}&interval=30m&limit=1`).then(res => res.json());
    const btcusdtdata = data.map(d => {
        return {
            Open: parseFloat(d[1]),
            High: parseFloat(d[2]),
            Low: parseFloat(d[3]),
            Close: parseFloat(d[4]),
            Volume: parseFloat(d[5]),
            Timespan: 30,
        }
    });
    console.log(btcusdtdata);
    saveToDatebase(symbol, btcusdtdata);
    //recursive functions are complicated, we can get rid of it here
    //by moving the responsibility to the caller
};

//helper function for an awaitable timeout
const sleep = ms => new Promise(res => setTimeout(res,ms));

const j = schedule.scheduleJob('*/15 * * * *', async () => {
    //expand this function to be responsible for looping the data
    for(let symbol of symbols) {
        //we can pass symbol to getBTCData instead of making it
        //responsible for figuring out which symbol it should get
        await getBTCData(symbol); 
        await sleep(2000);
    }
});

//make this a helper function so `saveToDatabase()` isn't also responsible for it
const getDateTime = () => {
    let today = new Date();
    let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
    let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
    return date + ' ' + time;
};

const saveToDatebase = async (symbol, BTCdata) => {
    const url = 'mongodb+srv://username:[email protected]/<dbname>?retryWrites=true&w=majority';

    let dateTime = getDateTime();
    
    //use await here and below to vastly simplify this function
    let db = await MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true });
    const dbo = db.db('CryptoCurrencies');
    const myobj = { Name: symbol, Array: BTCdata, Date: dateTime };
    await dbo.collection(symbol).insertOne(myobj);
    console.log('1 document inserted');
    db.close();
};

I haven't tested this code - let me know if there are any errors.