Trouble fetching results using yield while going for multiprocessing

Question

I'm trying to create a script using python applying multiprocessing within it to fetch the link of different users from a webpage. Although the link of the users are available in it's landing page, I'm trying to dig them out from their inner pages. However, when I use yield within get_links() function and print() within get_target_link(), I can get the results as expected.

My question is: how can I achieve the same using yield within both of the functions?

I've tried:

import requests
import concurrent.futures
from urllib.parse import urljoin
from bs4 import BeautifulSoup

def get_links(url):
    res = requests.get(url)
    soup = BeautifulSoup(res.text,"lxml")
    for item in soup.select(".summary .question-hyperlink"):
        yield urljoin(base,item.get("href"))

def get_target_link(targeturl):
    res = requests.get(targeturl)
    soup = BeautifulSoup(res.text,"lxml")
    name_link = urljoin(base,soup.select_one(".user-details > a").get("href"))
    yield name_link

if __name__ == '__main__':
    base = 'https://stackoverflow.com'
    mlink = "https://stackoverflow.com/questions/tagged/web-scraping"

    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        future_to_url = {executor.submit(get_target_link, url): url for url in get_links(mlink)}
        concurrent.futures.as_completed(future_to_url)

The above script produces no result at all.

Answer 1

A few problems with your initial approach that causing "no result at all":

• BeautifulSoup(res.text,"lxml") - change parser to html.parser (you are parsing html web-pages)
• there's no benefit of making the function get_target_link as generator cause it's not supposed to become an iterator and it already produces out a single result at once.
• concurrent.futures.as_completed returns an iterator over the Future instances, not the final result

The corrected approach would look as below:

import requests
import concurrent.futures as futures
from urllib.parse import urljoin
from bs4 import BeautifulSoup


def get_links(url):
    res = requests.get(url)
    soup = BeautifulSoup(res.text, "html.parser")
    for link in soup.select(".summary .question-hyperlink"):
        yield urljoin(base, link.get("href"))


def get_target_link(target_url):
    res = requests.get(target_url)
    soup = BeautifulSoup(res.text, "html.parser")
    name_link = urljoin(base, soup.select_one(".user-details a").get("href"))
    return name_link


if __name__ == '__main__':
    base = 'https://stackoverflow.com'
    mlink = "https://stackoverflow.com/questions/tagged/web-scraping"

    with futures.ThreadPoolExecutor(max_workers=10) as executor:
        future_to_url = {executor.submit(get_target_link, url): url for url in get_links(mlink)}
        for future in futures.as_completed(future_to_url):
            url = future_to_url[future]
            try:
                data = future.result()
            except Exception as ex:
                print(f'Failed to extract user details from url: {url}')
            else:
                print(data)

The output:

https://stackoverflow.com/users/10035985/andrej-kesely
https://stackoverflow.com/users/11520568/rachit-gupta
https://stackoverflow.com/users/10568531/robots-txt
https://stackoverflow.com/users/10664939/logan-anderson
https://stackoverflow.com/users/688393/c%c3%a9sar
https://stackoverflow.com/users/903061/gregor
https://stackoverflow.com/users/9950503/saraherceg
https://stackoverflow.com/users/80851/gmile
https://stackoverflow.com/users/11793150/saurabh-rawat
https://stackoverflow.com/users/11793061/xzatar
https://stackoverflow.com/users/11759292/rachel9866
https://stackoverflow.com/users/2628114/user2628114
https://stackoverflow.com/users/9810397/bart
https://stackoverflow.com/users/838355/ir2pid
https://stackoverflow.com/users/10629482/shreya
https://stackoverflow.com/users/11669928/thor-is
https://stackoverflow.com/users/7660288/acro2142
https://stackoverflow.com/users/3342430/freddiev4
https://stackoverflow.com/users/11767045/k-%c3%96sterlund
https://stackoverflow.com/users/11781213/mohamed-shire
https://stackoverflow.com/users/5412619/a-nonymous
https://stackoverflow.com/users/4354477/forcebru
https://stackoverflow.com/users/10568531/robots-txt
https://stackoverflow.com/users/6622587/eyllanesc
https://stackoverflow.com/users/10568531/robots-txt
https://stackoverflow.com/users/3273177/casabonita
https://stackoverflow.com/users/1540328/dipesh-parmar
https://stackoverflow.com/users/6231957/perth
https://stackoverflow.com/users/11400264/workin-4weekend
https://stackoverflow.com/users/1000551/vadim-kotov
https://stackoverflow.com/users/331508/brock-adams
https://stackoverflow.com/users/11300154/helloworld1990
https://stackoverflow.com/users/11786268/mohsine-jirou
https://stackoverflow.com/users/9707561/fatima-tt
https://stackoverflow.com/users/11759292/rachel9866
https://stackoverflow.com/users/6622587/eyllanesc
https://stackoverflow.com/users/11485683/titan
https://stackoverflow.com/users/11593630/supek
https://stackoverflow.com/users/11717116/raja-kishore-patnayakuni
https://stackoverflow.com/users/975887/madushan
https://stackoverflow.com/users/10568531/robots-txt
https://stackoverflow.com/users/283366/phil
https://stackoverflow.com/users/8677101/bpdesilva
https://stackoverflow.com/users/3504096/programmerper
https://stackoverflow.com/users/6303216/akhlaq-ahmed
https://stackoverflow.com/users/11457578/sh-student
https://stackoverflow.com/users/11783947/alexis-cruz-cruz
https://stackoverflow.com/users/3579212/adnanmuttaleb
https://stackoverflow.com/users/1060350/anony-mousse
https://stackoverflow.com/users/8100732/khadija-saeed