Python: Pyppeteer clicking on pop up window

Question

I'm trying to accept the cookies consent on a pop up window that is generated on this page. I tried to use waitForSelector but none of the selectors that I used seems to be visible to the headless browser. I would like to actually switch to "YES" and then submit the form. I guess it's displayed on window.onload so perhaps this will need to be done in JavaScript?

import asyncio
import time

from pyppeteer import launch
from pyppeteer.errors import TimeoutError
from urllib.parse import urlparse

URLS = [
    'https://www.trustarc.com/'
]

start = time.time()

async def fetch(url, browser):
    page = await browser.newPage()
    try:
        #await page.setRequestInterception(True)
        page.on('request', callback)
        await page.goto(url, {'waitUntil': 'networkidle0'})
        await page.screenshot({'path': f'img/{urlparse(url)[1]}.png', 'fullPage': True})
    except TimeoutError as e:
        print(f'Timeout for: {url}')
    finally:
        await page.close()


async def callback(req): 
    print(f'Request: {req.url}')

async def run():
    browser = await launch(headless=True, args=['--no-sandbox'])
    tasks = []

    for url in URLS:
        task = asyncio.ensure_future(fetch(url, browser))
        tasks.append(task)

    ret = await asyncio.gather(*tasks)
    await browser.close()

loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run())
loop.run_until_complete(future)

print(f'It took {time.time()-start} seconds.')

Answer 1

In case someone will find this useful, this is my Python implementation based on the accepted answer:

import asyncio
import time

from pyppeteer import launch
from pyppeteer.errors import TimeoutError
from urllib.parse import urlparse

URLS = [
    'https://www.trustarc.com/'
]

start = time.time()

async def fetch(url, browser):
    page = await browser.newPage()
    try:
        #await page.setRequestInterception(True)
        #page.on('request', callback)
        await page.goto(url, {'waitUntil': 'networkidle0'})
        if not await page.J('.truste_overlay'):
            await page.click('#teconsent > a')
        cookies_frame = page.frames[1]
        await cookies_frame.waitForSelector( '.active', {'visible': True})
        await cookies_frame.evaluate('''() =>
            {
               const yes_buttons   = document.getElementsByClassName( 'off' );
               const submit_button = document.getElementsByClassName( 'submit' )[0];

               yes_buttons[0].click();
               yes_buttons[1].click();

               submit_button.click();
            }''')
        close_button = await cookies_frame.waitForSelector( '#gwt-debug-close_id' )
        await close_button.click()
        await page.screenshot({'path': f'img/{urlparse(url)[1]}.png', 'fullPage': True})
    except TimeoutError as e:
        print(f'Timeout for: {url}')
    finally:
        await page.close()


async def callback(req): 
    print(f'Request: {req.url}')

async def run():
    browser = await launch(headless=True, args=['--no-sandbox'])
    tasks = []

    for url in URLS:
        task = asyncio.ensure_future(fetch(url, browser))
        tasks.append(task)

    ret = await asyncio.gather(*tasks)
    await browser.close()

loop = asyncio.get_event_loop()
future = asyncio.ensure_future(run())
loop.run_until_complete(future)

print(f'It took {time.time()-start} seconds.')

Answer 2

If the Cookie Preferences pop-up frame does not open up automatically, you can manually open the pop-up by clicking on the button in the bottom-right corner of the web page.

The cookie options are located in an iframe, so you will have to wait until the frame content has loaded before selecting "YES" for Functional Cookies and Advertising Cookies.

After submitting the preferences, you will need to wait for and close the confirmation message to continue using the website.

Full Example:

// Navigate to the website

await page.goto( 'https://www.trustarc.com/', { 'waitUntil' : 'networkidle0' } );

// Open the Cookie Preferences pop-up (if necessary)

if ( await page.$( '.truste_overlay' ) === null )
{
    await page.click( '#teconsent > a' );
}

// Wait for the Cookie Preferences frame and content to load

const cookies_frame = page.frames()[1];

await cookies_frame.waitForSelector( '.active', { 'visible' : true } );

// Fill out and submit form

await cookies_frame.evaluate( () =>
{
    const yes_buttons   = document.getElementsByClassName( 'off' );
    const submit_button = document.getElementsByClassName( 'submit' )[0];

    yes_buttons[0].click();
    yes_buttons[1].click();

    submit_button.click();
});

// Wait for and close confirmation

const close_button = await cookies_frame.waitForSelector( '#gwt-debug-close_id' );

await close_button.click();