My result is empty not sure what went wrong with my code:
Here my code:
#trying to grab data from this site https://www.watsons.com.sg/health/c/2100000?currentPage=1
import requests
from bs4 import BeautifulSoup
import pandas as pd
# Setting display options for pandas
pd.options.display.width = 1000
pd.options.display.max_rows = 1000
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}
# Create lists
items = []
prices = []
# Function to scrape a page
def scrape_page(page):
try:
url = f'https://www.watsons.com.sg/health/c/2100000?currentPage={page}'
response = requests.get(url, headers=HEADERS)
response.raise_for_status() # Check for request errors
soup = BeautifulSoup(response.text, 'html.parser')
# Find product items
product_items = soup.find_all('e2-product-tile', class_='ng-star-inserted hasPromotion-2')
print(soup)
for item in product_items:
# Extract product name
name = item.find('h2', class_='productName').get_text(strip=True)
items.append(name)
# Extract product price
price = item.find('div', class_='formatted-value ng-star-inserted').get_text(strip=True)
prices.append(price)
except requests.RequestException as e:
print(f"An error occurred: {e}")
for page in range(1, 2):
scrape_page(page)
df = pd.DataFrame({'Item': items, 'Price': prices})
print(df)
#The above is my code
U started on the wrong way, all the information u need (Name and price) is initially loaded on the page as json. So we can just take this data from script block.
import json
import requests
import pandas as pd
from bs4 import BeautifulSoup
headers = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'accept-language': 'en-US,en;q=0.9,en-US;q=0.8,en;q=0.7',
'cache-control': 'no-cache',
'priority': 'u=0, i',
'referer': 'https://www.watsons.com.sg/health/c/2100000?currentPage=1',
'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
}
def get_info_from_page(page: int) -> list:
response = requests.get(f"https://www.watsons.com.sg/health/c/2100000?currentPage={page}", headers=headers)
soup = BeautifulSoup(response.text, 'lxml')
products = []
json_data = json.loads(soup.find('script', class_='structured-data').get_text())
for product in json_data:
if product['@type'] == 'Product':
products.append({'Name': product['name'],
'Price': product['offers']['price'],
'Lowest price': product['offers']['lowPrice'],
'Brand': product['brand']['name'],
'SKU': product['sku']})
return products
df = pd.DataFrame(get_info_from_page(1))
print(df.to_string())
OUTPUT:
Name Price Lowest price Brand SKU
0 3ply Surgical Disposable Face Mask Earloop Adult (BFE >99% + PFE >99%) 50s 10.10 8.00 ASSURE BP_30368
1 Medicated Oil (Effective Relief Dizziness Headache Stuffy Nose) 57ml 6.65 6.65 KWAN LOONG OIL BP_10117
2 Daily Nutritional Drink Naturally Cholesterol Free Vegetarian with Plant Based Ingredients 850g 46.95 46.95 DR OATCARE BP_75204
3 Crystal Luxe 6500mg Red Snapper Marine Collagen Drink Berries Sachets (Awarded Gold Quality by Monde Selection 2021) 30ml x 20s 86.70 65.02 DR ORA BP_48016
4 Medicated Oil No.3 3ml 1.30 1.30 EAGLE BP_11663
5 Mos-Bye Mosquito Repellent Air Freshener (Lemongrass) 275ml 7.00 7.00 SAWADAY BP_38451
6 BreatheEasy Patch 5s 7.60 7.60 SANTECARE BP_79625
7 Yoko Yoko 82ml 9.20 9.20 AMMELTZ BP_17827
8 Plus Lubricant Eye Drops 0.4ml x 30s 24.20 24.20 REFRESH BP_11709
9 Tablets 250mg Clinically Proven FAST Relief Medicinal Charcoal (Treats Diarrhoea and Food Poisoning) 20s 9.05 9.05 ULTRACARBON BP_10048
10 Alcon Systane Ultra HD High Performance Lubricant Eye Drops 0.7ml x 30s 26.00 26.00 ALCON BP_63561
11 50 Billion Probiotics Complex Vegetarian Capsules (Limits Stomach and Digestion Issues) 30s 34.95 24.46 21ST CENTURY BP_82443
12 Ultimate Enzyme 720ml 113.55 94.50 AFC BP_64679
13 Tablets 400mg (Supporting Healthy Liver Function & Healthy Mood) 30s 87.05 87.05 HEPTRAL BP_18066
14 Apple Cider Vinegar Fat Burner Juice Sachet (Aids In Weight Loss + Boosts Metabolism + Blocks Carb Absorption)35ml X 7s 26.00 19.50 SHORTCUTX BP_51421
15 Multi Purpose Solution (Extra Comfort, Made In Singapore, Lens Case Included) 360ml X 3s 23.10 16.17 WATSONS BP_43042
16 RheumaSalve Medicated Balm 50g 10.10 9.09 HERITAGE BP_14661
17 Anti Bacterial Foaming Hand Soap Refill Lychee (99.9% Anti-Bacterial Protection + Gentle On Delicate Skin) 200ml 2.75 2.75 KIREI KIREI BP_48923
18 Mosquito Repellent Patch (12 Hours Protection, Natural Ingredients) 24s 6.90 4.83 WATSONS BP_44750
19 VapoDrops +Immune Support Lozenges Blackcurrant Menthol 16 Lozengers 7.35 4.92 VICKS BP_77193
20 Sea Coconut Cough Mixture (Relieves Coughs & Soothes Throat) 177ml 5.90 5.90 AFRICAN SEA-COCONUT BRAND BP_10554
21 Joint Sensei Supreme Dietary Supplement Tablet (To Support Cartilage And Bone Integrity, For Joint Neck Shoulder Knee Back Pain) 540s 82.25 64.90 AFC BP_82962
22 Vitamin C + Vitamin D3 + Zinc Effervescent Tablets Natural Orange Flavour (For Immune Health) 15s x 3 Tubes 25.30 25.30 WATSONS BP_32470
23 Panadol Extend for Muscle & Joint Pain Caplets 18s 9.95 9.95 PANADOL BP_13813
24 MenstruHeat Menstrual Cramp Relief 2 Pieces 4.95 4.95 BLOOD BP_88685
25 Eagle Fresh On Original Citrus Roll On 8ml 3.90 3.50 EAGLE BP_37936
26 30 Patch Light Scent (Gentle to Skin + Pain Relief) 20s 7.20 7.20 SALONPAS® BP_13155
27 Watsons Washproof Plasters 100s 4.50 4.50 WATSONS BP_93866
28 WOODS P/MINT CURE 200ML ADULT 11.45 11.45 WOODS BP_84270
29 Turmeric Concentrated Extract 60s 14.65 10.90 AFC BP_31424
30 Diamond Vision Pro 4X Dietary Supplement (Floraglo Lutein 4X Eye Supplement For Dry Eye, Tired Eyes, Blurred Vision, Blue Light Protection, Eye Fatigue) 30s 60.45 60.45 AFC BP_62521
31 Blackmores Buffered C Tablets 200s 65.30 39.18 BLACKMORES BP_55774