I'm just trying to pull the first column of urls from the table off this site. And i keep running into KeyError: 0. I'm just starting to learn python.
Traceback (most recent call last):
File "riscribble.py", line 13, in <module>
lic_link = soup_data[0].find('a').text
File "C:\Users\rkrouse\Desktop\Python\lib\site-packages\bs4\element.py", line 1071, in __getitem__
return self.attrs[key]
KeyError: 0
Any ideas on why i'm getting this error and/or how to correct would be appreciated.
from bs4 import BeautifulSoup as soup
import requests as r
import pandas as pd
url = 'http://www.crb.state.ri.us/verify_CRB.php?page=0&letter='
data = r.get(url)
page_data = soup(data.text, 'html.parser')
soup_data = page_data.find('table')
lic_link = soup_data[0].find('a').text
df = pd.DataFrame()
for each in soup_data:
lic_link = each.find('a').text
df=df.append(pd.DataFrame({'LicenseURL': lic_link}, index=[0]))
df.to_csv('RI_License_urls.csv', index=False)
Imports:
from bs4 import BeautifulSoup as soup
import requests as r
import pandas as pd
import re
Get your page:
url = 'http://www.crb.state.ri.us/verify_CRB.php?page=0&letter='
data = r.get(url)
page_data = soup(data.text, 'html.parser')
Choose your link:
links = [link.text for link in page_data.table.tr.find_all('a') if re.search('licensedetail.php', str(link))]
links -> 32922
# or
links = [link for link in page_data.table.tr.find_all('a') if re.search('licensedetail.php', str(link))]
links -> <a href="licensedetail.php?link=32922&type=Resid">32922</a>
# or
links = [link['href'] for link in page_data.table.tr.find_all('a') if re.search('licensedetail.php', str(link))]
links -> licensedetail.php?link=32922&type=Resid
# or
links = [r'www.crb.state.ri.us/' + link['href'] for link in page_data.table.tr.find_all('a') if re.search('licensedetail.php', str(link))]
links -> www.crb.state.ri.us/licensedetail.php?link=32922&type=Resid
Finish:
df = pd.DataFrame(links, columns=['LicenseURL'])
df.to_csv('RI_License_urls.csv', index=False)