Scrapy Splash is always returning the same page

Question

For each of several Disqus users, whose profile urls are known in advance, I want to scrape their names and usernames of their followers. I'm using scrapy and splash do to so. However, when I'm parsing the responses, it seems that it is always scraping the page of the first user. I tried setting wait to 10 and dont_filter to True, but it isn't working. What should I do now?

Here is my spider:

import scrapy
from disqus.items import DisqusItem

class DisqusSpider(scrapy.Spider):
    name = "disqusSpider"
    start_urls = ["https://disqus.com/by/disqus_sAggacVY39/", "https://disqus.com/by/VladimirUlayanov/", "https://disqus.com/by/Beasleyhillman/", "https://disqus.com/by/Slick312/"]
    splash_def = {"endpoint" : "render.html", "args" : {"wait" : 10}}

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url = url, callback = self.parse_basic, dont_filter = True, meta = {
                "splash" : self.splash_def,
                "base_profile_url" : url
            })

    def parse_basic(self, response):
        name = response.css("h1.cover-profile-name.text-largest.truncate-line::text").extract_first()
        disqusItem = DisqusItem(name = name)
        request = scrapy.Request(url = response.meta["base_profile_url"] + "followers/", callback = self.parse_followers, dont_filter = True, meta = {
            "item" : disqusItem,
            "base_profile_url" : response.meta["base_profile_url"],
            "splash": self.splash_def
        })
        print "parse_basic", response.url, request.url
        yield request

    def parse_followers(self, response):
        print "parse_followers", response.meta["base_profile_url"], response.meta["item"]
        followers = response.css("div.user-info a::attr(href)").extract()

DisqusItem is defined as follows:

class DisqusItem(scrapy.Item):
    name = scrapy.Field()
    followers = scrapy.Field()

Here are the results:

2017-08-07 23:09:12 [scrapy.core.engine] DEBUG: Crawled (200) <POST http://localhost:8050/render.html> (referer: None)
parse_followers https://disqus.com/by/disqus_sAggacVY39/ {'name': u'Trailer Trash'}
2017-08-07 23:09:14 [scrapy.extensions.logstats] INFO: Crawled 5 pages (at 5 pages/min), scraped 0 items (at 0 items/min)
2017-08-07 23:09:18 [scrapy.core.engine] DEBUG: Crawled (200) <POST http://localhost:8050/render.html> (referer: None)
parse_followers https://disqus.com/by/VladimirUlayanov/ {'name': u'Trailer Trash'}
2017-08-07 23:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <POST http://localhost:8050/render.html> (referer: None)
parse_followers https://disqus.com/by/Beasleyhillman/ {'name': u'Trailer Trash'}
2017-08-07 23:09:40 [scrapy.core.engine] DEBUG: Crawled (200) <POST http://localhost:8050/render.html> (referer: None)
parse_followers https://disqus.com/by/Slick312/ {'name': u'Trailer Trash'}

Here is the file settings.py:

# -*- coding: utf-8 -*-

# Scrapy settings for disqus project
#

BOT_NAME = 'disqus'

SPIDER_MODULES = ['disqus.spiders']
NEWSPIDER_MODULE = 'disqus.spiders'

ROBOTSTXT_OBEY = False

SPLASH_URL = 'http://localhost:8050' 

DOWNLOADER_MIDDLEWARES = {
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}

DUPEFILTER_CLASS = 'scrapyjs.SplashAwareDupeFilter'
DUPEFILTER_DEBUG = True

DOWNLOAD_DELAY = 10

Answer 1

I was able to get it to work using SplashRequest instead of scrapy.Request.

ex:

import scrapy
from disqus.items import DisqusItem
from scrapy_splash import SplashRequest


class DisqusSpider(scrapy.Spider):
    name = "disqusSpider"
    start_urls = ["https://disqus.com/by/disqus_sAggacVY39/", "https://disqus.com/by/VladimirUlayanov/", "https://disqus.com/by/Beasleyhillman/", "https://disqus.com/by/Slick312/"]

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(url, self.parse_basic, dont_filter = True, endpoint='render.json',
                        args={
                            'wait': 2,
                            'html': 1
                        })