Search code examples
excelvbaweb-scrapingxmlhttprequest

Get headers of table when using XMLHTTP approach


I have a code that grabs a table from this url

https://www.reuters.com/companies/AAPL.OQ/financials/income-statement-annual

The code is OK and no problem at all except one point. The code gets the table but doesn't get the header

    With http
    .Open "Get", sURL, False
    .send
    html.body.innerHTML = .responseText
End With

   Set tbl = html.getElementsByTagName("Table")(0)

        For Each rw In tbl.Rows
            r = r + 1: c = 1
            For Each cl In rw.Cells
                ws.Cells(r, c).Value = cl.innerText
                c = c + 1
            Next cl
    Next rw

When inspecting the URL, I found that API URL supported

https://www.reuters.com/companies/api/getFetchCompanyFinancials/AAPL.OQ

How can I extract the desired data "annual" for "income" from the JSON response?

I tried to refer to the section I desire but got an error

Const strUrl As String = "https://www.reuters.com/companies/api/getFetchCompanyFinancials/AAPL.OQ"

Sub Test()
Dim a, json As Object, colData As Collection, sFile As String, i As Long

With CreateObject("MSXML2.ServerXMLHTTP.6.0")
    .Open "GET", strUrl
    .send
    Set json = JSONConverter.ParseJson(.responseText)
End With


Set colData = json("market_data")("financial_statements")

Stop
End Sub

Solution

  • logic similar to this should work in vba:

    Dim data As Scripting.Dictionary, key As Variant, block As Collection, r As Long, item As Object
    
    Set data = json("market_data")("financial_statements")("financial_statements")("income")("annual") ' dict of collections
    
    r = 1
    
    For Each key In data.keys
        Set block = data(key)  'each block (section of info) is a row
        r = r + 1: c= 2
        For each item In block 'loop columns in block         
            With Activesheet
                If r = 2 then 'write out headers to row 1,starting col2 and then values to row 2 starting from col 2, and key goes in row , col 1
                    .Cells(1,c) = item("date")
                End If
                .Cells(r,1) = Key
                .Cells(r,c) = item("value")
            End With
            c = c + 1
        Next
    Next
    

    I can't test in VBA but if I write the python (long hand) equivalent I get the same table:

    import requests
    import pandas as pd
    
    json = requests.get('https://www.reuters.com/companies/api/getFetchCompanyFinancials/AAPL.OQ').json()
    data = json["market_data"]["financial_statements"]["income"]["annual"]
    rows = len(data.keys()) + 1
    columns = len(data["Revenue"]) + 1
    r = 0
    df = pd.DataFrame(["" for c in range(columns)] for r in range(rows))
    
    for key in data.keys():
        block = data[key]
        r+=1 ; c = 1
        for item in block:
            if r == 1:
                df.iloc[0 , c] = item["date"]
            df.iloc[r,c] = item["value"]
            df.iloc[r,0] = key
            c+=1
    print(df)