Is there a way to scroll to the final product on the page? I can only fetch first 20 products from the page

Question

I am trying to fetch and print all the listed product from the given url 'xyz' but currently facing issue with the scroll. It is fetching only first 20 products from the given link and rest I am unable to fetch I have tried with the given code

 last_height = driver.execute_script("return document.body.scrollHeight")
#
# while True:
#     driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
#     time.sleep(20)  # Wait for the page to load new content
#     new_height = driver.execute_script("return document.body.scrollHeight")
#     if new_height == last_height:
#         break
#     last_height = new_height

Answer 1

I gave this a shot and it was successful.

def scroll_down_to_last_element():
    # Wait for the elements to be visible
    wait = WebDriverWait(driver, 30)

    # Initialize an empty list to store elements
    elements = []

    # Scroll down until no new elements are loaded within 10 seconds
    start_time = time.time()
    while True:
        # Scroll to the last loaded element
        elements = wait.until(EC.presence_of_all_elements_located(
            (
            By.XPATH, "//button[@class='novMV' and contains(text(), 'double tap to go back to filter list')]")))

        # Get the last element
        last_element = elements[-1]

        # Scroll to the last element
        driver.execute_script("arguments[0].scrollIntoView();", last_element)

        # Wait briefly to allow new elements to potentially load
        time.sleep(1)

        # Check if new elements have stopped loading or if 10 seconds have