pythonseleniumweb-scrapingbeautifulsoupexport-csv
Web Scraping Selenium and Beautiful soup - unable to export into csv file
I am trying to web scrape the data (price + brand) from this website. The code actually works but I can only see the data on my sublime text editor and cannot convert it into a CSV file. Additionally, I get this error message:
AttributeError: 'NoneType' object has no attribute 'div'
Here is my code:
from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd
PATH = "/Users/Ziye/Desktop/Python/chromedriver"
def get_html(url):
driver = webdriver.Chrome(PATH)
driver.get(url)
return driver.page_source
def main ():
rows = []
url = "https://www.yoox.com/de/damen/kleidung/shoponline/michael%20kors_md#/Md=403&d=10321&dept=clothingwomen&gender=D&page=2&season=X"
html = get_html(url)
soup = BeautifulSoup(html, "lxml")
cards = soup.find_all("div", {"class": "col-8-24"})
print(len(cards))
for card in cards:
print(card.find(class_="itemData text-center").div.get_text())
print(card.find(class_="price").get_text())
row = {'Brand': card.find(class_="brand font-bold text-uppercase").get_text(),
'Price': card.find(class_="price").get_text()}
rows.append(row)
df = pd.DataFrame(rows)
df.to_csv('file.csv', index=False)
if __name__ == "__main__":
main()
Solution
To get items from second page, replace # in URL for ?:
import requests
import pandas as pd
from bs4 import BeautifulSoup
url = "https://www.yoox.com/de/damen/kleidung/shoponline/michael%20kors_md?/Md=403&d=10321&dept=clothingwomen&gender=D&page=2&season=X"
soup = BeautifulSoup(requests.get(url).content, "lxml")
data = []
for item in soup.select(".itemlink"):
data.append(
{
"Brand": item.select_one(".brand").get_text(strip=True),
"Category": item.select_one(".microcategory").get_text(strip=True),
"Price": item.select_one(".fullprice, .newprice").get_text(
strip=True
),
}
)
df = pd.DataFrame(data)
print(df)
df.to_csv("data.csv", index=False)
Prints:
Brand Category Price
0 MICHAEL MICHAEL KORS Knielange Röcke EUR 99,00
1 MICHAEL MICHAEL KORS Lange Kleider EUR 188,00
2 MICHAEL MICHAEL KORS Pullover EUR 180,00
3 MICHAEL MICHAEL KORS Kurze Kleider EUR 184,00
4 MICHAEL MICHAEL KORS Oversize-T-Shirts EUR 75,00
5 MICHAEL MICHAEL KORS Lange Kleider EUR 235,00
6 MICHAEL MICHAEL KORS Midi-Röcke EUR 156,00
7 MICHAEL MICHAEL KORS T-Shirts EUR 76,00
8 MICHAEL MICHAEL KORS Leggings EUR 154,00
9 MICHAEL MICHAEL KORS Looks für zuhause EUR 85,00
10 MICHAEL MICHAEL KORS Steppjacken EUR 400,00
11 MICHAEL MICHAEL KORS Overalls EUR 180,00
12 MICHAEL MICHAEL KORS Rollkragenpullover EUR 131,00
13 MICHAEL MICHAEL KORS Lange Kleider EUR 141,00
14 MICHAEL MICHAEL KORS Hemden und Blusen einfarbig EUR 132,00
15 MICHAEL MICHAEL KORS Kurze Kleider EUR 212,00
16 MICHAEL MICHAEL KORS Kurze Kleider EUR 124,00
17 MICHAEL MICHAEL KORS T-Shirts EUR 95,00
18 MICHAEL MICHAEL KORS Kurze Kleider EUR 119,00
19 MICHAEL MICHAEL KORS T-Shirts EUR 62,00
20 MICHAEL MICHAEL KORS Bikerjacken EUR 550,00
21 MICHAEL MICHAEL KORS Kurze Kleider EUR 139,00
22 MICHAEL MICHAEL KORS Kurze Kleider EUR 83,00
23 MICHAEL MICHAEL KORS Looks für zuhause EUR 75,00
24 MICHAEL MICHAEL KORS Hosen EUR 99,00
25 MICHAEL MICHAEL KORS Hemden und Blusen aus Seide EUR 109,00
26 MICHAEL MICHAEL KORS Hosen EUR 145,00
27 MICHAEL MICHAEL KORS Lange Jacken EUR 226,00
28 MICHAEL MICHAEL KORS T-Shirts EUR 66,00
29 MICHAEL MICHAEL KORS T-Shirts EUR 43,00
30 MICHAEL MICHAEL KORS Lange Kleider EUR 66,00
31 MICHAEL MICHAEL KORS Kaschmirpullover EUR 273,00
32 MICHAEL MICHAEL KORS Blusen EUR 74,00
33 MICHAEL MICHAEL KORS Kurze Kleider EUR 64,00
34 MICHAEL MICHAEL KORS Hemden und Blusen mit Blumen EUR 99,00
35 MICHAEL MICHAEL KORS Steppjacken EUR 247,00
36 MICHAEL MICHAEL KORS Bomberjacken EUR 325,00
37 MICHAEL MICHAEL KORS Looks für zuhause EUR 95,00
38 MICHAEL MICHAEL KORS Knielange Kleider EUR 347,00
39 MICHAEL MICHAEL KORS Pullover EUR 195,00
40 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 264,00
41 MICHAEL MICHAEL KORS Bikinis EUR 99,00
42 MICHAEL MICHAEL KORS Sweatshirts EUR 150,00
43 MICHAEL MICHAEL KORS Knielange Kleider EUR 192,00
44 MICHAEL MICHAEL KORS T-Shirts EUR 74,00
45 MICHAEL MICHAEL KORS Hemden und Blusen aus Spitze EUR 118,00
46 MICHAEL MICHAEL KORS Lange Jacken EUR 225,00
47 MICHAEL MICHAEL KORS Sweatshirts EUR 94,00
48 MICHAEL MICHAEL KORS Overalls EUR 235,00
49 MICHAEL MICHAEL KORS Hosen EUR 134,00
50 MICHAEL MICHAEL KORS T-Shirts EUR 59,00
51 MICHAEL MICHAEL KORS Hosen EUR 79,00
52 MICHAEL MICHAEL KORS Midi-Röcke EUR 109,00
53 MICHAEL MICHAEL KORS Hosen EUR 90,00
54 MICHAEL MICHAEL KORS Steppjacken EUR 225,00
55 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 234,00
56 MICHAEL MICHAEL KORS Hosen EUR 104,00
57 MICHAEL MICHAEL KORS Overalls EUR 129,00
58 MICHAEL MICHAEL KORS Hosen EUR 94,00
59 MICHAEL MICHAEL KORS Cropped-Hosen & Culottes EUR 99,00
60 MICHAEL MICHAEL KORS Hosen EUR 84,00
61 MICHAEL MICHAEL KORS Hosen EUR 94,00
62 MICHAEL MICHAEL KORS Hosen EUR 94,00
63 MICHAEL MICHAEL KORS Hemden und Blusen aus Seide EUR 250,00
64 MICHAEL MICHAEL KORS Hosen EUR 94,00
65 MICHAEL MICHAEL KORS Jacketts EUR 194,00
66 MICHAEL MICHAEL KORS Hosen EUR 94,00
67 MICHAEL MICHAEL KORS Midi-Röcke EUR 99,00
68 MICHAEL MICHAEL KORS Overalls EUR 235,00
69 MICHAEL MICHAEL KORS Cabanjacke & Zweireiher EUR 525,00
70 MICHAEL MICHAEL KORS Lange Kleider EUR 67,00
71 MICHAEL MICHAEL KORS Miniröcke EUR 150,00
72 MICHAEL MICHAEL KORS Mäntel EUR 395,00
73 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 175,00
74 MICHAEL MICHAEL KORS Kurze Kleider EUR 124,00
75 MICHAEL MICHAEL KORS Hemden und Blusen mit Muster EUR 94,00
76 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 94,00
77 MICHAEL MICHAEL KORS Midi-Kleider EUR 225,00
78 MICHAEL MICHAEL KORS Kurze Kleider EUR 138,00
79 MICHAEL MICHAEL KORS Kurze Kleider EUR 450,00
80 MICHAEL MICHAEL KORS Rollkragenpullover EUR 135,00
81 MICHAEL MICHAEL KORS Pullover EUR 195,00
82 MICHAEL MICHAEL KORS Midi-Röcke EUR 109,00
83 MICHAEL MICHAEL KORS Lange Kleider EUR 168,00
84 MICHAEL MICHAEL KORS Shorts & Bermudas EUR 84,00
85 MICHAEL MICHAEL KORS Strickjacken EUR 99,00
86 MICHAEL MICHAEL KORS Hosen EUR 94,00
87 MICHAEL MICHAEL KORS Kurze Kleider EUR 131,00
88 MICHAEL MICHAEL KORS Kurze Kleider EUR 144,00
89 MICHAEL MICHAEL KORS Hosen EUR 89,00
90 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 124,00
91 MICHAEL MICHAEL KORS Midi-Röcke EUR 99,00
92 MICHAEL MICHAEL KORS Hosen EUR 94,00
93 MICHAEL MICHAEL KORS Pullover EUR 249,00
94 MICHAEL MICHAEL KORS Hosen EUR 109,00
95 MICHAEL MICHAEL KORS Rollkragenpullover EUR 119,00
96 MICHAEL MICHAEL KORS Hosen EUR 94,00
97 MICHAEL MICHAEL KORS Hosen EUR 86,00
98 MICHAEL MICHAEL KORS Hemdchen EUR 109,00
99 MICHAEL MICHAEL KORS Hosen EUR 109,00
100 MICHAEL MICHAEL KORS Kurze Kleider EUR 309,00
101 MICHAEL MICHAEL KORS Midi-Kleider EUR 175,00
102 MICHAEL MICHAEL KORS Blusen EUR 84,00
103 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 139,00
104 MICHAEL MICHAEL KORS Hosen EUR 110,00
105 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 124,00
106 MICHAEL MICHAEL KORS Hosen EUR 94,00
107 MICHAEL MICHAEL KORS Gerade geschnittene Hosen EUR 129,00
108 MICHAEL MICHAEL KORS Kurze Kleider EUR 149,00
109 MICHAEL MICHAEL KORS Schulterfreies Top EUR 79,00
110 MICHAEL MICHAEL KORS Blusen EUR 250,00
111 MICHAEL MICHAEL KORS Blusen EUR 225,00
112 MICHAEL MICHAEL KORS Hosen EUR 150,00
113 MICHAEL MICHAEL KORS Lange Kleider EUR 159,00
114 MICHAEL MICHAEL KORS Jacketts EUR 210,00
115 MICHAEL MICHAEL KORS Hosen EUR 79,00
116 MICHAEL MICHAEL KORS Blusen EUR 103,00
117 MICHAEL MICHAEL KORS Hosen EUR 111,00
118 MICHAEL MICHAEL KORS Jacketts EUR 154,00
119 MICHAEL MICHAEL KORS Kurze Kleider EUR 144,00
And creates data.csv (screenshot from LibreOffice):
EDIT: To get oldprice, newprice and fullprice into separate columns:
import requests
import pandas as pd
from bs4 import BeautifulSoup
url = "https://www.yoox.com/de/damen/kleidung/shoponline/michael%20kors_md?/Md=403&d=10321&dept=clothingwomen&gender=D&page=2&season=X"
soup = BeautifulSoup(requests.get(url).content, "lxml")
data = []
for item in soup.select(".itemlink"):
data.append(
{
"Brand": item.select_one(".brand").get_text(strip=True),
"Category": item.select_one(".microcategory").get_text(strip=True),
"Old Price": item.select_one(".oldprice").get_text(strip=False)
if item.select_one(".oldprice")
else "",
"New Price": item.select_one(".newprice").get_text(strip=False)
if item.select_one(".newprice")
else "",
"Full Price": item.select_one(".fullprice").get_text(strip=False)
if item.select_one(".fullprice")
else "",
}
)
df = pd.DataFrame(data)
print(df)
df.to_csv("data.csv", index=False)
Prints:
Brand Category Old Price New Price Full Price
0 MICHAEL MICHAEL KORS Knielange Röcke EUR 99,00
1 MICHAEL MICHAEL KORS Lange Kleider EUR 235,00 EUR 188,00
2 MICHAEL MICHAEL KORS Pullover EUR 225,00 EUR 180,00
3 MICHAEL MICHAEL KORS Kurze Kleider EUR 184,00
4 MICHAEL MICHAEL KORS Oversize-T-Shirts EUR 75,00
5 MICHAEL MICHAEL KORS Lange Kleider EUR 235,00
6 MICHAEL MICHAEL KORS Midi-Röcke EUR 195,00 EUR 156,00
7 MICHAEL MICHAEL KORS T-Shirts EUR 95,00 EUR 76,00
8 MICHAEL MICHAEL KORS Leggings EUR 175,00 EUR 154,00
9 MICHAEL MICHAEL KORS Looks für zuhause EUR 85,00
10 MICHAEL MICHAEL KORS Steppjacken EUR 450,00 EUR 400,00
11 MICHAEL MICHAEL KORS Overalls EUR 225,00 EUR 180,00
...