Search code examples
pythonhtmlscrapycss-selectorsparsel

How can scrapy get the second <a href> inside of div class?


I just trying to get url from in a div class. But this div has two <a href>. So, that can get the first one but that should get second one. How can I do this?

This is the html code of website:

 <div class="active"> 
          <a href="javascript:;" class=" add-favorite  j-lightbox-popup-button " data-target="howGoingCheckoutNotRedirect"> <i data-feather="heart"></i> </a> 
          <a href="/urun/yohegoalhareketli-kale-112432"> 

This is my Python code:

product_link = container.a["href"]

It gets javascript:; inside of a["href"]

All my code:

from bs4 import BeautifulSoup as soup
import pandas as pd
from tabulate import tabulate
import os
from joker import joker
import requests


product_joker_array = pd.DataFrame(columns = ['Shop Name','Product Name','Product Price','Product Image Url','Product Url']) 


for categories in range(0,len(joker.joker_category_names)):
    url = joker.joker_url  + joker.joker_category_names[categories]
    print(url)


    for pages in range(0,250):
        url = (joker.joker_url  + joker.joker_category_names[categories] + "#/page={}".format(pages))
        print(url)
        response = requests.get(url)
        data = response.text

        page_soup = soup(data, 'lxml')

        #time.sleep(10)
        containers = page_soup.findAll("div",{"class": "active"})
        #print(containers)
        len(containers)
        try:
            container = containers[0]
        except IndexError:
            print("Index Error")
            break

        #print(container)

        for container in containers:

            product_image = container.findAll("div", {"class": "image-link"})
            product_final_image = product_image[0].img["src"].strip()
            print("Product Image Url : " + product_final_image )

            product_desc = container.findAll("div", {"class": "image-link"})
            #print("asdasdas", product_desc)
            product_final_desc = product_desc[0].img["alt"].strip()
            print("product_final_desc : ",product_final_desc)

            #product_link = container.a["href"]
            product_link = container.xpath('.//a[2]/@href').extract_first()
            print("link : ",product_link)

            product_final_link = joker.joker_url + product_link
            print("final link : ",product_final_link)     



            product_price = container.findAll("span",{"class": "discount-price"})

            product_final_price = product_price[0].text.replace(" ", "")
            product_final_price = product_final_price.strip()
            print("product_final : ",product_final_price)
            #if "," not in product_final_price:
                                         #product_final_price = product_final_price + ",00"
                                         #print("product ,00",product_final_price)
            product_market_name = 'Joker'

            #print("Product Price:" + product_final_price )
            product_joker_array.loc[len(product_joker_array)] = [product_market_name,product_final_desc,product_final_price,product_final_image,product_final_link]

Solution

  • I solved this problem with these codes:

    product_link = container.find("div", {"class": "active"})
                product_link = product_link.findAll("a")
                product_final_link = joker.joker_url + product_link[1]["href"]