Scrape table from each option in drop down menu python

Question

I'm trying to scrape all the data from this website:
http://www.dartsdatabase.co.uk/PlayerStats.aspx?statKey=1&pg=7

However, I do not know how to iterate through the 'stat' drop-down menu. Each of these options contains a table I need to scrape.

So far I have the following code which lists the options and values associated with each element in the drop-down list:

url = 'http://www.dartsdatabase.co.uk/PlayerStats.aspx'

response = requests.get(url).text

soup = BeautifulSoup(response,"lxml")

drop = soup.find('select',{'name':'stat'}).findAll("option")

options = []

val = []

for i in range(0,len(drop)):

    options.append(drop[i].text)

    val.append(drop[i]['value'])

Any help would be greatly appreciated!

Answer 1

Make POST requests altering the stat param. You can gather the appropriate values from the page value attribute of the options

import requests
import pandas as pd
from bs4 import BeautifulSoup as bs

data = {
  'nameSearch': '',
  'dateFrom': '02/10/2017',
  'dateTo': '02/10/2019',
  'organStat': 'All',
  'stat': '1',
  'tourns': 'All',
  'pg': '7'
}

def get_soup():
    r = s.post('http://www.dartsdatabase.co.uk/PlayerStats.aspx?statKey=1&pg=7', data=data)
    soup = bs(r.content, 'lxml')  
    return soup

with requests.Session() as s:
    soup = get_soup()
    table = pd.read_html(str(soup.select_one('br + table')))[0]
    stats = [i['value'] for i in soup.select('[name="stat"] option')][1:]
    print(table)

    for i in stats:
        data['stat']=i
        soup = get_soup()
        table = pd.read_html(str(soup.select_one('br + table')))[0]
        print(table)