Search code examples
excelvbaweb-scrapingscreen-scraping

Web page pagination not changing

I am trying to navigate to the next page on a website, normally this works for me. However I am struggling at the moment. Currenly with this line of code Set nextPageElement = HTML.getElementsByClassName("paginationMini--right__active")(0) I can loop X amount of times however it is NOT changing the page, the page always remains page 1, therefore if I stated 3 pages it will pull the same data off page1 THREE times. When it should change the page 3 times.

I have tried several variations and have left a few commented out in the code below. All off the attempts end after the first page, the above line of code is the only one that loops the code 3 times but is not changing the page. I have always used this code so I do know that it works. Please could someone point out the correct class.

Link

image

'Searches Number of Pages entered in Sheet20 
    If pageNumber >= Replace(Worksheets("Sheet20").Range("J9").Value, "", "+") Then Exit Do
        On Error Resume Next
         Set nextPageElement = HTML.getElementsByClassName("paginationMini--right__active")(0)  ' THIS LINE
       
'Set nextPage = HTML.getElementsByClassName("pagination--ul")(0).getElementsByClassName("pagination--li")(0).getElementsByTagName("a")(0)
'Set nextPage = HTML.querySelector(".pagination--ul > li.pagination--li > a")
'Set nextPage = HTML.getElementsByClassName("pagination--ul")(0).getElementsByClassName("pagination--li")(0).getElementsByClassName("paginationMini--right__active")(0)
'Set nextPageElement = HTML.getElementsByClassName("paginationMini--ul")(0).getElementsByTagName("li")(2).getElementsByTagName("a")(0)
'Set nextPageElement = HTML.getElementsByClassName("paginationMini")(0).getElementsByTagName("li")(2).getElementsByTagName("a")(0)
    If nextPageElement Is Nothing Then Exit Do
        nextPageElement.Click 'next web page 
    Do While objIE.Busy = True Or objIE.readyState <> 4
    Loop
        Set Html = objIE.document
        pageNumber = pageNumber + 1


<div class="search-page__pagination-wrapper">
  <nav class="pagination">
    <ul class="pagination--ul">
      <li class="pagination--li">
        <span class="pagination--left__inactive">
                    <i class="icon">
                        <svg><use xlink:href="/templates/_generated/svg_icons/common.svg#icon-arrow-left"></use></svg>
                    </i>
                    <span class="pagination__text">Previous</span>
        </span>
      </li>
      <li class="pagination--li">
        1
      </li>
      <li class="pagination--li">
        <a href="https://www.autotrader.co.uk:443/car-search/page/2" rel="nofollow" data-paginate="2" data-to-top="true">2</a>
      </li>

      <li class="pagination--li">
        <a href="https://www.autotrader.co.uk:443/car-search/page/3" rel="nofollow" data-paginate="3" data-to-top="true">3</a>
      </li>
      <li class="pagination--li">
        <a href="https://www.autotrader.co.uk:443/car-search/page/4" rel="nofollow" data-paginate="4" data-to-top="true">4</a>
      </li>
      <li class="pagination--li">
        <a href="https://www.autotrader.co.uk:443/car-search/page/5" rel="nofollow" data-paginate="5" data-to-top="true">5</a>
      </li>
      <li class="pagination--li">
        <a href="https://www.autotrader.co.uk:443/car-search/page/6" rel="nofollow" data-paginate="6" data-to-top="true">6</a>
      </li>

      <li class="pagination--li">
        <a class="pagination--right__active" href="https://www.autotrader.co.uk:443/car-search/page/2" rel="nofollow" data-paginate="2" data-to-top="true">
          <i class="icon">
                    <svg><use xlink:href="/templates/_generated/svg_icons/common.svg#icon-arrow-right"></use></svg>
                </i>
          <span class="pagination__text">Next</span>
        </a>
      </li>
    </ul>
  </nav>

</div>

Solution

  • Try this way to grab content from next pages. The links connected to next pages are invalid ones. When you click on the next page links, they get redirected to some other url. However, the following is one of the easy ways to get things done:

    Sub FetchNextPageContent()
    Dim IE As Object, post As Object, Url$, I&
    
    Set IE = CreateObject("InternetExplorer.Application")

    Url = "https://www.autotrader.co.uk/car-search?sort=relevance&postcode=W1K%203RA&radius=1500&include-delivery-option=on&page="

    For I = 1 To 5
        IE.Visible = True
        IE.navigate Url & I
        While IE.Busy = True Or IE.readyState < 4: DoEvents: Wend
        
        For Each post In IE.document.getElementsByClassName("search-page__result")
            With post.getElementsByClassName("listing-fpa-link")
                If .Length Then Debug.Print .Item(0).getAttribute("href")
            End With
        Next post
    Next I
End Sub

    If clicking on the next page button is what you wanna stick with, the following should do that:

    Sub FetchNextPageContent()
    Dim IE As Object, post As Object, Url$, I&, nextPage As Object
    Dim Html As HTMLDocument
    
    Set IE = CreateObject("InternetExplorer.Application")

    Url = "https://www.autotrader.co.uk/car-search?sort=relevance&postcode=W1K%203RA&radius=1500&include-delivery-option=on&page=1"
    
    IE.Visible = True
    IE.navigate Url
    
    Do
        While IE.Busy = True Or IE.readyState < 4: DoEvents: Wend
        Set Html = IE.document
        
        For Each post In Html.getElementsByClassName("search-page__result")
            With post.getElementsByClassName("listing-fpa-link")
                If .Length Then Debug.Print .Item(0).getAttribute("href")
            End With
        Next post
        
        Set nextPage = Html.querySelector("a.pagination--right__active")
        If Not nextPage Is Nothing Then
            nextPage.Click
            Application.Wait Now + TimeValue("00:00:05")
        Else:
            Exit Do
        End If
    Loop
End Sub