How to scrape multi-level links using puppeteer js?

Question

I am scraping table rows of site page using Puppeteer. I have the code to scrape content and assign them to an object for each in the table. In each table row there is a link that I need to open in a new page (puppeteer) and then scrape for a particular element then assign it to the same object and return the whole object with the new keys to puppeteer. How is that possible with Puppeteer?

async function run() {
    const browser = await puppeteer.launch({
        headless: false
    })
    const page = await browser.newPage()

    await page.goto('https://tokenmarket.net/blockchain/', {waitUntil: 'networkidle0'})
    await page.waitFor(5000)
    var onlink = ''
    var result = await page.$$eval('table > tbody tr .col-actions a:first-child', (els) => Array.from(els).map(function(el) {

        //running ajax requests to load the inner page links.
     $.get(el.children[0].href, function(response) {
            onlink = $(response).find('#page-wrapper > main > div.container > div > table > tbody > tr > td:nth-child(2)').text()
        })



        return {
            icoImgUrl: el.children[0].children[0].children[0].currentSrc,
            icoDate: el.children[2].innerText.split('\n').shift() === 'To be announced' ? null : new Date( el.children[2].innerText.split('\n').shift() ).toISOString(),
            icoName:el.children[1].children[0].innerText,
            link:el.children[1].children[0].children[0].href,
            description:el.children[3].innerText,
            assets :onlink
        }

    }))

    console.log(result)

    UpcomingIco.insertMany(result, function(error, docs) {})


    browser.close()
}

run()

Answer 1

If you try opening a new tab for each ICO page in parallel you might end up with 100+ pages loading at the same time.

So the best thing you could do is to first collect the URLs and then visit them one by one in a loop.

This also allows keeping the code simple and readable.

For example (please, see my comments):

const browser = await puppeteer.launch({ headless: false });

const page = await browser.newPage();

await page.goto('https://tokenmarket.net/blockchain/');

// Gather assets page urls for all the blockchains
const assetUrls = await page.$$eval(
  '.table-assets > tbody > tr .col-actions a:first-child',
  assetLinks => assetLinks.map(link => link.href)
);

const results = [];

// Visit each assets page one by one
for (let assetsUrl of assetUrls) {
  await page.goto(assetsUrl);

  // Now collect all the ICO urls.
  const icoUrls = await page.$$eval(
    '#page-wrapper > main > div.container > div > table > tbody > tr > td:nth-child(2) a',
    links => links.map(link => link.href)
  );

  // Visit each ICO one by one and collect the data.
  for (let icoUrl of icoUrls) {
    await page.goto(icoUrl);

    const icoImgUrl = await page.$eval('#asset-logo-wrapper img', img => img.src);
    const icoName = await page.$eval('h1', h1 => h1.innerText.trim());
    // TODO: Gather all the needed info like description etc here.

    results.push([{
      icoName,
      icoUrl,
      icoImgUrl
    }]);
  }
}

// Results are ready
console.log(results);

browser.close();