Search code examples
pythonhtmlseleniumweb-scraping

I'm trying to scrape data from the at the races website but the scraper is not returning any results

from selenium import webdriver

driver = webdriver.Chrome()
login_url = 'http://www.attheraces.com/racecard/Wolverhampton/6-October-2018/1715'
driver.get(login_url)

html = driver.execute_script("return document.documentElement.outerHTML")

sel_soup = BeautifulSoup(html, 'html.parser')
print(sel_soup.findAll("sectionals-time"))

When I run the last line of the script it just returns

[]

It is a dynamic website as far as I am aware, so when you go to this site and scroll down to results, you click the sectional times tab, then right click the first sectional time for the first listed horse and inspect. this then shows me the class attribute as "sectionals-time" so I'm struggling to understand why it's not producing the sectional times for the horses.

Any advice and help much appreciated.

Solution

  • This will work. Leave a comment if you need the output to be different.

    from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait

url = 'http://www.attheraces.com/racecard/Wolverhampton/6-October-2018/1715'

driver = webdriver.Chrome()
driver.get(url)
driver.implicitly_wait(2)
driver.find_element_by_xpath('//*[@id="racecard-tabs-1061960"]/div[1]/div/div[1]/ul/li[2]/a').click()

WebDriverWait(driver, 5).until(expected_conditions.presence_of_element_located((By.XPATH, '//*[@id="tab-racecard-sectional-times"]/div/div[1]/div[1]/div[2]/div/button')))

# method 1
for horse in driver.find_elements_by_class_name('card-item'):
    horseName = horse.find_element_by_class_name('form-link').text
    times = horse.find_elements_by_class_name('sectionals-time')
    times = [time.text for time in times]
    print('{}: {}'.format(horseName, times))

print()

# method 2
for horse in driver.find_elements_by_class_name('card-item'):
    for time in horse.find_elements_by_class_name('sectionals-time'):
        print(time.text)
    print()

driver.close()