soup.find unable to find text on yahoo finance

Question

I am receiving the following error:

Invalid figure 'Total Cash Flow From Operating Activities' passed.

From this url:

Scrape Yahoo Finance Income Statement with Python

"This is made a little more difficult because the "Net Income" in enclosed in a strong tag"

Can someone explain to me why this code fails to work for Total Cash Flow From Operating Activities and how I can determine that something has a strong tag?

Code:

import re, requests
from bs4 import BeautifulSoup
import sys
"""
import os  #  file system operations
import re  # regular expressions
import pandas as pd # pandas... the best time series library out there
import datetime as dt # date and time functions
import io
"""
# search with regular expressions

# "CrumbStore":\{"crumb":"(?<crumb>[^"]+)"\}

def get_crumb():
    url = 'https://uk.finance.yahoo.com/quote/AAPL/history' # url for a ticker symbol, with a download link
    r = requests.get(url)  # download page

    txt = r.text # extract html

    cookie = r.cookies['B'] # the cooke we're looking for is named 'B'
    print('Cookie: ', cookie)

    # Now we need to extract the token from html. 
    # the string we need looks like this: "CrumbStore":{"crumb":"lQHxbbYOBCq"}
    # regular expressions will do the trick!

    pattern = re.compile('.*"CrumbStore":\{"crumb":"(?P<crumb>[^"]+)"\}')

    for line in txt.splitlines():
        m = pattern.match(line)
        if m is not None:
            crumb = m.groupdict()['crumb']


    print('Crumb=',crumb)
    return crumb

def periodic_figure_values(soup, yahoo_figure):

    values = []
    pattern = re.compile(yahoo_figure)

    title = soup.find("strong", text=pattern)    # works for the figures printed in bold
    if title:
        row = title.parent.parent
    else:
        title = soup.find("td", text=pattern)    # works for any other available figure
        if title:
            row = title.parent
        else:
            sys.exit("Invalid figure '" + yahoo_figure + "' passed.")

    cells = row.find_all("td")[1:]    # exclude the <td> with figure name
    for cell in cells:
        if cell.text.strip() != yahoo_figure:    # needed because some figures are indented
            str_value = cell.text.strip().replace(",", "").replace("(", "-").replace(")", "")
            if str_value == "-":
                str_value = 0
            value = int(str_value) * 1000
            values.append(value)
    return values

def financials_soup(ticker_symbol, statement, quarterly=False):
    if statement == "is" or statement == "bs" or statement == "cf":
        crumb = get_crumb()
        url = "https://finance.yahoo.com/q/" + statement + "?s=" + ticker_symbol + "&crumb=" + crumb
        if not quarterly:
            url += "&annual"
        return BeautifulSoup(requests.get(url).text, "html.parser")

    return sys.exit("Invalid financial statement code '" + statement + "' passed.")

print(periodic_figure_values(financials_soup("AAPL", "cf"), "Total Cash Flow From Operating Activities"))

Edit:

I was able to obtain results by changing the financials_soup function to the following:

def financials_soup(ticker_symbol, statement, quarterly=False):
    if statement == "financials" or statement == "balance-sheet" or statement == "cash-flow":
        crumb = get_crumb()
        url = "https://finance.yahoo.com/quote/" + ticker_symbol + "/" + statement + "?p=" + ticker_symbol + "&crumb=" + crumb
        if not quarterly:
            url += "&annual"
        return BeautifulSoup(requests.get(url).text, "html.parser")

    return sys.exit("Invalid financial statement code '" + statement + "' passed.")

Answer 1

If you inspect the source returned by your link (on line 69):

url = "https://finance.yahoo.com/q/" + statement + "?s=" + ticker_symbol + "&crumb=" + crumb

You'll see that it hasn't any usefull information about Total Cash Flow From Operating Activities. If you change that into

url = "https://finance.yahoo.com/quote/" + ticker_symbol + "/cash-flow?p=" + ticker_symbol

It'll return the information that you are seeking:

[65824000000, 81266000000, 59713000000]