Search code examples
vbaweb-scrapingxmlhttprequestweb-crawler

Run time error 91, object variable or with block variable not set


My intention is to scrape all the app name from that page and the app link leading to the next page. However, when i run it, i see that after looping once it produces the following error "Run time error 91, object variable or with block variable not set".Here is the full code. Any help would really be appreciated. Thanks in advance.

Sub app_crawler()
    Dim xmlpage As New XMLHTTP60, htmldoc As New HTMLDocument
    Dim htmlas As Object, htmla As Object, sstr As String

    xmlpage.Open "GET", "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8", False
    xmlpage.send
    htmldoc.body.innerHTML = xmlpage.responseText

    For Each htmlas In htmldoc.getElementsByClassName("lockup-info")(0).getElementsByTagName("a")
        sstr = htmlas.href

        xmlpage.Open "GET", sstr, False
        xmlpage.send
        htmldoc.body.innerHTML = xmlpage.responseText

        For Each htmla In htmldoc.getElementsByClassName("intro")(1).getElementsByTagName("h1")
            x = x + 1: Cells(x, 1) = htmla.innerText
        Next htmla
    Next htmlas
End Sub

Solution

  • This is the answer which fixes all the problems I was having:

    Sub app_crawler()
        Dim http As New XMLHTTP60, hdoc As New HTMLDocument, hdoc_one As New HTMLDocument
        Dim elem As Object, post As Object, sstr As String
    
        With http
            .Open "GET", "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8", False
            .send
            hdoc.body.innerHTML = .responseText
        End With
    
        For Each elem In hdoc.getElementsByClassName("lockup-info")
            With elem.getElementsByTagName("li")(0).getElementsByTagName("a")
                If .Length Then sstr = .Item(0).href
            End With
            With http
                .Open "GET", sstr, False
                .send
                hdoc_one.body.innerHTML = .responseText
            End With
    
            For Each post In hdoc_one.getElementsByClassName("intro")
                With post.getElementsByTagName("h1")
                    If .Length Then i = i + 1: Cells(i, 1) = .Item(0).innerText
                End With
            Next post
        Next elem
    End Sub