Search code examples
pythonweb-scrapingbeautifulsouppaginationcategories

Trying to scrape other category with beautifulsoup


this is the website i am trying to scrape: [https://www.jurongpoint.com.sg/store-directory/]

This is my code,as u can see i don't know how to fill both of the {} for the url variable as the 4 category that i want to scrape especially url for service is very different. The comment above url variable shows the link of the 4 category when clicked in. Appreciate any help,thank you!

from bs4 import BeautifulSoup
import requests

def parse():


    cate=["Service","Food & Beverage","Fashion & Accessories","Electronics & Technology"]

    #cate=Food+%26+Beverage
    #cate=Electronics+%26+Technology
    #cate=Fashion+%26+Accessories
    #cate=Services


    url="https://www.jurongpoint.com.sg/store-directory/?level=&cate={}+%26+{}"

    for cat in cate:
    
    
        for page in range(1,14):
            print(page)

            soup = BeautifulSoup(requests.get(url).text ,"html.parser")


            for link in soup.find_all('div',class_='entry-content'):

                try:
                    shops=soup.find_all('div',class_="col-9")
                    names=soup.find_all('tr',class_="clickable")

                    for n, k in zip(names, shops):
                        name = n.find_all('td')[1].text.replace(' ','')
                        desc = k.text.replace(' ','')
                        print(name + "\n")
                        print(desc)

                except AttributeError as e:
                    print(e)


                next_button = soup.select_one('.PagedList-skipToNext a')
                if next_button:
                    url = next_button.get('href')
                else:
                    break


parse() 

Solution

  • Use parameters of your request and avoid to manage escape characters (like %26)

    url = "https://www.jurongpoint.com.sg/store-directory"
    
    for cat in cate:
        for page in range(1, 14):
            print(f'Scraping category {cat} page {page}')
            payload = {
                'level': '',
                'cate': cat,
                'page': page
            }
            resp = requests.get(url, params=payload)
            soup = BeautifulSoup(resp.text, 'html.parser')
            # your code here
    
    >>> resp.url
    'https://www.jurongpoint.com.sg/store-directory/?level=&cate=Electronics+%26+Technology&page=8'