The script I am running is as follows:
#import necessary packages
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
#run on headless browser
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(options=chrome_options)
####
#driver = webdriver.Chrome()
# Open the website
driver.get('https://euclid.eba.europa.eu/register/pir/search')
driver.set_window_size(1200, 800)
driver.maximize_window()
#find element using the link text
xpath_institution_search = "//a[@href='/register/pir/search' and contains(text(), 'InstitutionSearch')]"
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, xpath_institution_search))
)
finally:
institution_search = driver.find_element_by_xpath(xpath_institution_search)
driver.execute_script("return arguments[0].scrollIntoView();", institution_search)
institution_search.click()
#select all appropriate National Competent Authorities
xpath_nca_list_new = '(//p-listbox[@class="col-md-9 ng-untouched ng-pristine ng-valid" and @name="ncaList"])'
#find the listbox element using it's xpath
nca_listbox = driver.find_element_by_xpath(xpath_nca_list_new)
#find elements within list box using tag names
nca_div_tags = nca_listbox.find_elements_by_tag_name("div")
#find element representing "select all" checkbox
select_all_ncas = nca_div_tags[5]
#find all list content (National Competent Authorities)
nca_content_list = nca_div_tags[0].find_elements_by_tag_name("ul")
#store list content in an ordered list
ncas = nca_content_list[0].find_elements_by_tag_name("li")
#make web driver wait until element is visible before further action
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, xpath_nca_list_new)))
finally:
count = 1
#print out list of ncas
for item in ncas:
text = item.text
print(str(count) + ". " + text)
count=count+1
I'm very new to Python so please excuse any terrible code. The code sometimes prints a list of all the listbox elements like so:
But sometimes the list that is returned is empty. There are two other listboxes on the same webpage and this very same code works consistently for these elements.
Can anyone shed some light on why this is happening? Thanks in advance. Below is the HTML code for the listbox element I am trying to work with.
<p-listbox class="col-md-9 ng-untouched ng-pristine ng-valid" name="ncaList" optionlabel="CodeDisplayValue" _ngcontent-bwu-c7="">
<div class="ui-listbox ui-inputtext ui-widget ui-widget-content ui-corner-all" style="width: 100%;">
<div class="ui-helper-hidden-accessible">
<input type="text" readonly="readonly">
</div>
<!---->
<!---->
<div class="ui-widget-header ui-corner-all ui-listbox-header ui-helper-clearfix ui-listbox-header-w-checkbox ng-star-inserted">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-helper-hidden-accessible">
<input type="checkbox" readonly="readonly">
</div>
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!---->
</div>
<div class="ui-listbox-list-wrapper" style="max-height: 250px;">
<ul class="ui-listbox-list">
<!---->
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">AT - Austrian Financial Market Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">BE - National Bank of Belgium</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">BG - Bulgarian National Bank</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">CY - Central Bank of Cyprus</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">CZ - Czech National Bank</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">DE - Federal Financial Supervisory Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">DK - Danish Financial Supervisory Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">EE - Estonian Financial Supervision Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">ES - Bank of Spain</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">FI - Finnish Financial Supervisory Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">FR - Prudential Supervisory and Resolution Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">GB - Financial Conduct Authority (FCA)</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">GR - Bank of Greece</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">HR - Croatian National Bank</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">HU - Central Bank of Hungary</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">IE - Central Bank of Ireland</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">IS - Financial Supervisory Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">IT - Bank of Italy</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">LI - Financial Market Authority Liechtenstein</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">LT - Bank of Lithuania</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">LU - Commission for the Supervision of Financial Sector</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">LV - Financial and Capital Market Commission</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">MT - Malta Financial Services Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">NL - The Netherlands Bank</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">NO - The Financial Authority of Norway</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">PL - Polish Financial Supervision Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">PT - Bank of Portugal</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">RO - National Bank of Romania</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">SE - Swedish Financial Supervisory Authority</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">SI - Bank of Slovenia</span>
<!---->
</li>
<li tabindex="0" class="ui-listbox-item ui-corner-all ng-star-inserted" style="display: block;">
<!---->
<div class="ui-chkbox ui-widget ng-star-inserted">
<div class="ui-chkbox-box ui-widget ui-corner-all ui-state-default"><span class="ui-chkbox-icon ui-clickable"></span></div>
</div>
<!----><span class="ng-star-inserted">SK - National Bank of Slovakia</span>
<!---->
</li>
</ul>
</div>
<!---->
</div>
Improved locators and with wait until visibility_of_all_elements_located of the liist:
wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "Institution Search"))).click()
search_button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[label=Search] button")))
driver.execute_script("arguments[0].scrollIntoView()", search_button)
search_button.click()
nca_list = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "[name=ncaList] li")))
for nca in nca_list:
print(nca.text)