How do I get a dataframe to write all data pulled to csv and not skip any rows with missing data?

Question

The following is working great for hundreds of stocks. But some stocks have incomplete data. E.g., blank shortRatio amount, etc. When that happens, it skips the entire row instead of just leaving certain fields blank. For example, out of symbols = ['AI', 'AG', 'MA'], only 'MA' outputs any data. How do I get it to write to csv whatever data is available and not skip rows just because one field is blank?

import pandas as pd
from yahooquery import Ticker
import csv

symbols = ['AI', 'AG', 'MA']

modules = 'financialData defaultKeyStatistics price summaryDetail earningsTrend'

for tick in symbols:
    tickers = Ticker(tick)
    try:
        df = pd.DataFrame([{'symbol': ticker,
'financialCurrency': data['financialData']['financialCurrency'], 
'shortRatio': data['defaultKeyStatistics']['shortRatio'], 
'shortPercentOfFloat': data['defaultKeyStatistics']['shortPercentOfFloat'], 
'priceToBook': data['defaultKeyStatistics']['priceToBook'], 
'earningsQuarterlyGrowth': data['defaultKeyStatistics']['earningsQuarterlyGrowth'], 
'marketCap': data['price']['marketCap'], 
'shortName': data['price']['shortName'], 
'fiftyTwoWeekLow': data['summaryDetail']['fiftyTwoWeekLow'], 
'fiftyTwoWeekHigh': data['summaryDetail']['fiftyTwoWeekHigh'], 
        'growth': data['earningsTrend']['trend'][-1]['growth']}                     
                           for ticker, data in tickers.get_modules(modules).items()])
    except Exception:
        continue
    try:
        df.to_csv('output.csv', mode='a', index=True, header=False)
    except (KeyError, TypeError):
        continue

Answer 1

This code does what you expect:

import pandas as pd
from yahooquery import Ticker
import csv

symbols = ['AI', 'AG', 'MA']

modules = 'financialData defaultKeyStatistics price summaryDetail earningsTrend'

with open('output.csv', mode='w', newline='') as file:
    writer = csv.writer(file)
    writer.writerow(['symbol', 'financialCurrency', 'shortRatio', 'shortPercentOfFloat', 
                     'priceToBook', 'earningsQuarterlyGrowth', 'marketCap', 'shortName', 
                     'fiftyTwoWeekLow', 'fiftyTwoWeekHigh', 'growth'])

    for tick in symbols:
        tickers = Ticker(tick)
        try:
            data = tickers.get_modules(modules)
            data_dict = {'symbol': tick}
            financial_data = data.get(tick, {}).get('financialData', {})
            data_dict['financialCurrency'] = financial_data.get('financialCurrency', '')
            default_statistics = data.get(tick, {}).get('defaultKeyStatistics', {})
            data_dict['shortRatio'] = default_statistics.get('shortRatio', '')
            data_dict['shortPercentOfFloat'] = default_statistics.get('shortPercentOfFloat', '')
            data_dict['priceToBook'] = default_statistics.get('priceToBook', '')
            data_dict['earningsQuarterlyGrowth'] = default_statistics.get('earningsQuarterlyGrowth', '')
            price_data = data.get(tick, {}).get('price', {})
            data_dict['marketCap'] = price_data.get('marketCap', '')
            data_dict['shortName'] = price_data.get('shortName', '')
            summary_data = data.get(tick, {}).get('summaryDetail', {})
            data_dict['fiftyTwoWeekLow'] = summary_data.get('fiftyTwoWeekLow', '')
            data_dict['fiftyTwoWeekHigh'] = summary_data.get('fiftyTwoWeekHigh', '')
            earnings_trend = data.get(tick, {}).get('earningsTrend', {})
            data_dict['growth'] = earnings_trend.get('trend', [])[-1].get('growth', '')
            writer.writerow(data_dict.values())
        except KeyError:
            continue

Now, if you open the ouput.csv file:

df=pd.read_csv('output.csv')

you will get:

  symbol financialCurrency  shortRatio  shortPercentOfFloat  priceToBook  \
0     AI               USD        0.71               0.2460     2.556160   
1     AG               USD        2.24                  NaN     1.413673   
2     MA               USD        2.13               0.0062    53.075290   

   earningsQuarterlyGrowth     marketCap                    shortName  \
0                      NaN    2476463104                  C3.ai, Inc.   
1                      NaN    1995270912  First Majestic Silver Corp.   
2                    0.061  331728388096      Mastercard Incorporated   

   fiftyTwoWeekLow  fiftyTwoWeekHigh    growth  
0            10.16             30.92        {}  
1             5.85             14.59  -0.33077  
2           276.87            390.00   0.12854