Python - selenium - Message: element click intercepted: Element is not clickable at point

Question

The code below, when the headless option is commented out, works perfectly, it manages to click on the VIEW ALL button on the site, but when we include the headless option, the error is displayed: Message: element click intercepted: Element is not clickable at point. I've researched and tried everything imaginable, nothing works. Anyone with any good ideas? thanks

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager

custom_user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"  # noqa

options = Options()
options.add_argument('--headless')
options.add_argument("start-maximized")
options.add_argument(f'user-agent={custom_user_agent}')

url = 'https://www.etf.com/SPY'

driver = webdriver.Chrome(options=options, service=Service(ChromeDriverManager().install()))  # noqa
driver.get(url)
time.sleep(3)

body = driver.find_element(By.XPATH, '/html/body')
body.send_keys(Keys.PAGE_DOWN)
body.send_keys(Keys.PAGE_DOWN)
body.send_keys(Keys.PAGE_DOWN)
body.send_keys(Keys.PAGE_DOWN)
time.sleep(3)

driver.find_element(By.XPATH, '//*[@id="holdings"]/div[2]/div[1]/div[1]/div[2]/div/div/div[1]/div[2]/div[3]/button').click()  # noqa
input('wait..')

Answer 1

With below user agent was able to successfully click on View All button.

Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.138 Safari/537.36

You can get user agent with below line.

user_agent = driver.execute_script("return navigator.userAgent;")

No need to write body.send_keys(Keys.PAGE_DOWN) 4 times to scroll to the element. Instead locate the chart element and use scrollIntoView option. Can use get_screenshot... to track the process.

driver.get_screenshot_as_file("filname.png")

Code:

from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options

options = Options()
options.add_argument('--headless')
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.138 Safari/537.36'
options.add_argument(f'user-agent={user_agent}')

driver = webdriver.Chrome(options=options,service=Service(ChromeDriverManager().install()))

driver.get("https://www.etf.com/SPY")

wait = WebDriverWait(driver,30)

# Scroll to the chart where View All button exist
chart = wait.until(EC.visibility_of_element_located((By.XPATH,"//div[@data-prop-label='Top 10 Holdings']")))
driver.execute_script("arguments[0].scrollIntoView(true)",chart)

# Locate View All button and using Javascript to click on that element to avoid "element click intercepted" exception. 
element = wait.until(EC.element_to_be_clickable((By.XPATH,"//span[text()='View All']")))
driver.execute_script("arguments[0].click()",element)