how to automatically switch pages using selenium?

Question

I'm trying to automatically click on each page, but I get the following error::

endPage=5
for page in range(1, endPage):
    print("Page: ", page)
    wait.until(EC.visibility_of_element_located((By.XPATH, "//table[@class='gridview']/tbody/tr[22]/td/table/tbody/tr//td/span//following::td["+str(page)+"]/a"))).click()

Errors:

Traceback (most recent call last):
  File "./code.py", line 52, in <module>
    wait.until(EC.visibility_of_element_located((By.XPATH, "//table[@class='gridview']/tbody/tr[22]/td/table/tbody/tr//td/span//following::td["+str(page)+"]/a"))).click()
  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 80, in click
    self._execute(Command.CLICK_ELEMENT)
  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
    return self._parent.execute(command, params)
  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementClickInterceptedException: Message: element click intercepted: Element <a href="javascript:__doPostBack('ctl00$C$CtlList','Page$...')">4</a> is not clickable at point (665, 579). Other element would receive the click: <iframe id="ipcc_chat_iframe" src="https://econtact.viettel.vn:8907/?key=P2RvbWFpbj1LSERUX0RWX1RUJnVzZXJuYW1lPSZjb2xvcj1ncmVlbiZpbnRlcm5hbD0wJmNsb3NlX2JveD11bmRlZmluZWQmbW9iaWxlX2FwcD11bmRlZmluZWQ%3D" style="border: 0px; width: 190px; height: 35px; position: fixed; bottom: 20px; right: 25px; z-index: 9999;"></iframe>
  (Session info: headless chrome=92.0.4515.131)

I hope you help me

Answer 1

Does this work out correctly:

import requests
import pandas as pd

s = requests.Session()
s.get('https://xxxx/')

url = 'https://xxxxx'
headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Mobile Safari/537.36'}

tables = []
for page in range(1,11):
    try:
        payload = {
        '__EVENTTARGET': 'ctl00$C$CtlList',    
        '__EVENTARGUMENT': 'Page$%s' %page,
        'ctl00$searchtype': '1'
        }
        
        response = s.post(url, headers=headers, data=payload)
        tables.append(pd.read_html(response.text)[0][:-1])
        print('Page: ',page)

    except Exception as e:
        print(e)
        break

df = pd.DataFrame()
for table in tables:
    df = df.append(table, sort=False)
df = df.reset_index(drop=True)

Output:

print(df)
                 Thời gian  ... Unnamed: 4
0    09/08/2021 7:39:46 CH  ...        NaN
1    09/08/2021 6:39:57 CH  ...        NaN
2    09/08/2021 6:39:54 CH  ...        NaN
3    09/08/2021 6:39:52 CH  ...        NaN
4    09/08/2021 6:39:50 CH  ...        NaN
..                     ...  ...        ...
195  07/08/2021 9:39:45 SA  ...        NaN
196  07/08/2021 8:39:50 SA  ...        NaN
197  07/08/2021 8:39:49 SA  ...        NaN
198  07/08/2021 8:39:46 SA  ...        NaN
199  07/08/2021 8:39:43 SA  ...        NaN

[200 rows x 5 columns]