Not clicking all tabs and not looping once issues

Question

I am trying to click the tabs on the webpage as seen below. Unfortunately, it only seems to click some of the tabs despite correct correct xpath in inspect Chrome. I can only assume it’s not clicking all the tabs because the full xpath is not being used.

However.. I have tried changing the xpath:

//div[@class="KambiBC-collapsible-container KambiBC-mod-event-group-container"] To:

//div[@class='KambiBC-event-groups-list']//div[@class="KambiBC-collapsible-container KambiBC-mod-event-group-container"] FOR:

clickMe = wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH,'(//div[@class="KambiBC-collapsible-container KambiBC-mod-event-group-container"])[%s]' % str(index + 1))))    

However the issue persists. I have also tried using CSS:

#KambiBC-contentWrapper__bottom > div > div > div > div > div.KambiBC-quick-browse-container.KambiBC-quick-browse-container--list-only-mode > div.KambiBC-quick-browse__list.KambiBC-delay-scroll--disabled > div > div.KambiBC-time-ordered-list-container > div.KambiBC-time-ordered-list-content > div > div > div.KambiBC-collapsible-container.KambiBC-mod-event-group-container > header

However this keeps giving me errors… For:

clickMe = wait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR,'("#KambiBC-contentWrapper__bottom > div > div > div > div > div.KambiBC-quick-browse-container.KambiBC-quick-browse-container--list-only-mode > div.KambiBC-quick-browse__list.KambiBC-delay-scroll > div > div.KambiBC-time-ordered-list-container > div.KambiBC-time-ordered-list-content > div > div > div > header")[%s]' % str(index + 1))))

It should be noted that I want to click all the unopened tabs and I cannot seem to use CSS Selectors to find a specific enough element as I believe it does not allow you to narrow down the class element in this case.

Is there a way to get around this issue of not clicking everything?

It should be noted that I am using...

for index in indexes:

indexes = [index for index in range(len(options))]
shuffle(indexes)
for index in indexes:

Is there a more elegant way of using for 1 loop?

[import sys
sys.exit()][1]

Full code

Answer 1

This cycles through all the matches from each league 1 by 1, collecting all the relevant data as needed. You can collect further data within each match by prefixing each query with . and selecting a match via match.find_element_by_xpath('.//your-query-here'). Let me know if this did the trick!

import sys, io, os, csv, requests, time
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium import webdriver

driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()

try:
    os.remove('vtg121.csv')
except OSError:
    pass

driver.get('https://www.unibet.com.au/betting#filter/football')
time.sleep(1)

clickMe = wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, 
    ('//div[@class="KambiBC-collapsible-container '\
    'KambiBC-mod-event-group-container"]'))))
time.sleep(0)

xp_opened = '//div[contains(@class, "KambiBC-expanded")]'
xp_unopened = '//div[@class="KambiBC-collapsible-container ' \
    'KambiBC-mod-event-group-container" ' \
    'and not(contains(@class, "KambiBC-expanded"))]'
opened = driver.find_elements_by_xpath(xp_opened)
unopened = driver.find_elements_by_xpath(xp_unopened)

data = []
for league in opened:
    xp_matches = './/li[contains(@class,"KambiBC-event-item")]'
    matches = league.find_elements_by_xpath(xp_matches)

    try:
        # League Name
        xp_ln = './/span[@class="KambiBC-mod-event-group-header__main-title"]'
        ln = league.find_element_by_xpath(xp_ln).text.strip()
    except:
        ln = None
    print(ln)

    for match in matches:
        # get all the data per 'match group'
        xp_team1_name = './/button[@class="KambiBC-mod-outcome"][1]//' \
            'span[@class="KambiBC-mod-outcome__label"]'
        xp_team1_odds = './/button[@class="KambiBC-mod-outcome"][1]//' \
            'span[@class="KambiBC-mod-outcome__odds"]'
        xp_team2_name = './/button[@class="KambiBC-mod-outcome"][3]//' \
            'span[@class="KambiBC-mod-outcome__label"]'
        xp_team2_odds = './/button[@class="KambiBC-mod-outcome"][3]//' \
            'span[@class="KambiBC-mod-outcome__odds"]'

        try:
            team1_name = match.find_element_by_xpath(xp_team1_name).text
        except:
            team1_name = None

        try:
            team1_odds = match.find_element_by_xpath(xp_team1_odds).text
        except:
            team1_odds = None

        try:
            team2_name = match.find_element_by_xpath(xp_team2_name).text
        except:
            team2_name = None

        try:
            team2_odds = match.find_element_by_xpath(xp_team2_odds).text
        except:
            team2_odds = None

        data.append([ln, team1_name, team1_odds, team2_name, team2_odds])

for league in unopened:
    league.click()
    time.sleep(0.5)
    matches = league.find_elements_by_xpath(xp_matches)

    try:
        ln = league.find_element_by_xpath(xp_ln).text.strip()
    except:
        ln = None
    print(ln)

    for match in matches:
        try:
            team1_name = match.find_element_by_xpath(xp_team1_name).text
        except:
            team1_name = None

        try:
            team1_odds = match.find_element_by_xpath(xp_team1_odds).text
        except:
            team1_odds = None

        try:
            team2_name = match.find_element_by_xpath(xp_team2_name).text
        except:
            team2_name = None

        try:
            team2_odds = match.find_element_by_xpath(xp_team2_odds).text
        except:
            team2_odds = None

        data.append([ln, team1_name, team1_odds, team2_name, team2_odds])

with open('vtg121.csv', 'a', newline='', encoding="utf-8") as outfile:
    writer = csv.writer(outfile)
    for row in data:
        writer.writerow(row)
        print(row)

Answer 2

OP's code without extra imports

The error happens because the site's XPaths to tabs OP wants are not contiguous. It has a gap. For example, now I cannot find

//*[@id="KambiBC-contentWrapper__bottom"]/div/div/div/div/div[3]/div1/div/div[3]/div[2]/div/div/div[2]/header

A while ago before a game goes live, I cannot find

//*[@id="KambiBC-contentWrapper__bottom"]/div/div/div/div/div[3]/div1/div/div[3]/div[2]/div/div/div[1]/header

When I talk about index, I mean the bold part above.

When a game goes live, the tab suddenly turns the index from 2 to 1. (The bold part changes.) In both cases, there are gaps: either 1 cannot be found or 2 cannot be found.

The reason for having the gap, I guess, is because there is another element in between that is not clickable. See the figure below.

The league is the cause of the gap. Thus, whenever the code hits the index league occupies, it times out. Because League button and other tabs switch the position of League and a live game, the indices are swapped when the positions changes. (I think that's why I cannot find Xpath with bold part being 1 first and later on cannot find that being 2.)

Below is part of the code from OP. You can see that at the end, str(index + 1).

indexes = [index for index in range(len(options))] # 
shuffle(indexes) # the OP use shuffle from random. Still 0 and 1 is contained.
path = '(//div[@class="KambiBC-collapsible-container KambiBC-mod-event-group-container"])'
for index in indexes:
    # Because there are some indexes are missing because of League button,
    # nothing can be found at the index and it times out.
    clickMe = wait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, path + '[%s]' % str(index + 1))))

Solution

Try to catch the timeout exception to skip that index occupied by League. You can also keep a counter to allow only one timeout exception caught on one page. If there is a second timeout, you know there is something wrong other than the League button and should stop.

from selenium import webdriver 
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
driver = webdriver.Firefox()
driver.set_window_size(1024, 600)
driver.maximize_window()
wait = WebDriverWait 

driver.get('https://www.unibet.com.au/betting#filter/football')
time.sleep(5)

options = driver.find_elements_by_xpath("""//*[@id="KambiBC-contentWrapper__bottom"]/div/div/div/div/div[3]/div[1]/div/div[3]/div[2]/div/div/div""")

print("Total tabs that we want to open is {}".format(len(options)))
indexes = [index for index in range(len(options))]

for index in indexes:
    print(index)
    try:
        clickMe = wait(driver, 5).until(EC.presence_of_element_located((By.XPATH,
            """//*[@id="KambiBC-contentWrapper__bottom"]/div/div/div/div/div[3]/div[1]/div/div[3]/div[2]/div/div/div[{}]/header""".format(str(index+1)))))
        clickMe.click()
    except TimeoutException as ex:
        print("catch you! {}".format(index))
        pass