How to click next button while scraping a webpage

Question

I am scraping a webpage using scrapy that has multiple pages of information and I need the program to click the next button and then scrape the next page and then keep doing that until all the pages have been scraped. But I cannot figure out how to do that, I can only scrape the first page.

from scrapy_splash import SplashRequest
from ..items import GameItem

class MySpider(Spider):
        name = 'splash_spider' # Name of Spider
        start_urls = ['http://www.starcitygames.com/catalog/category/10th%20Edition'] # url(s)
        def start_requests(self):
                for url in self.start_urls:
                        yield SplashRequest(url=url, callback=self.parse, args={"wait": 3})
        #Scraping
        def parse(self, response):
                item = GameItem()
                for game in response.css("tr"):
                        # Card Name
                        item["Name"] = game.css("a.card_popup::text").extract_first()
                        # Price
                        item["Price"] = game.css("td.deckdbbody.search_results_9::text").extract_first()
                        yield item

Answer 1

Documentation is pretty explicit about it :

from scrapy_splash import SplashRequest
from ..items import GameItem

class MySpider(Spider):
        name = 'splash_spider' # Name of Spider
        start_urls = ['http://www.starcitygames.com/catalog/category/10th%20Edition'] # url(s)
        def start_requests(self):
            for url in self.start_urls:
                yield SplashRequest(url=url, callback=self.parse, args={"wait": 3})
        #Scraping
        def parse(self, response):
            item = GameItem()
            for game in response.css("tr"):
                # Card Name
                item["Name"] = game.css("a.card_popup::text").extract_first()
                # Price
                item["Price"] = game.css("td.deckdbbody.search_results_9::text").extract_first()
                yield item

            next_page = response.css(<your css selector to find next page>).get()
            if next_page is not None:
                yield response.follow(next_page, self.parse)