Scrapy pipeline only save one page of results

Question

I have a spider to crawl course_tal which has a pipeline to save two types of items:

moocs.csv which contains the course data. moocs_review.csv which contains the reviews data.

This is the spider code I have:

import scrapy
from scrapy import  Request
from scrapy.loader import ItemLoader

from urlparse import urljoin 
from moocs.items import MoocsItem,MoocsReviewItem



class MoocsSpiderSpider(scrapy.Spider):
    name = "moocs_spider"
    #allowed_domains = ["https://www.coursetalk.com/subjects/data-science/courses"]
    start_urls = (
        'https://www.coursetalk.com/subjects/data-science/courses',
    )


    def parse(self, response):
        courses_xpath = '//*[@class="course-listing-card"]//a[contains(@href, "/courses/")]/@href'
        courses_url = [urljoin(response.url,relative_url)  for relative_url in response.xpath(courses_xpath).extract()]  
        for course_url in courses_url[0:3]:
            print course_url
            yield Request(url=course_url, callback=self.parse_reviews)
        next_page_url =   response.xpath('//*[@class="js-course-pagination"]//a[contains(@aria-label,"Next")]/@href').extract()
        yield Request(url=next_page_url, callback=self.parse)


    def parse_reviews(self, response):
        #print response.body
        l = ItemLoader(item=MoocsItem(), response=response)
        l.add_xpath('course_title', '//*[@class="course-header-ng__main-info__name__title"]//text()')
        l.add_xpath('course_description', '//*[@class="course-info__description"]//p/text()')
        l.add_xpath('course_instructors', '//*[@class="course-info__instructors__names"]//text()')
        l.add_xpath('course_key_concepts', '//*[@class="key-concepts__labels"]//text()')
        l.add_value('course_link', response.url)
        l.add_value('course_provider', response.url)
        l.add_xpath('course_cost', '//*[@class="course-details-panel__course-cost"]//text()')
        l.add_xpath('university', '//*[@class="course-info__school__name"]//text()[2]')
        #'//*[@class="course-info__school__name"]'
        item = l.load_item()

        for review in response.xpath('//*[@class="review-body"]'):
            r = ItemLoader(item=MoocsReviewItem(), response=response, selector=review)
            r.add_value('course_title', item['course_title'])
            r.add_xpath('review_body', './/div[@class="review-body__content"]//text()')
            r.add_xpath('course_stage', './/*[@class="review-body-info__course-stage--completed"]//text()')
            r.add_xpath('user_name', './/*[@class="review-body__username"]//text()')
            r.add_xpath('review_date', './/*[@itemprop="datePublished"]/@datetime')
            r.add_xpath('score', './/*[@class="sr-only"]//text()')

            yield r.load_item()

        yield item  

Which goes to each course page and save the details into the corresponding item. I'm getting the pagination here:

    next_page_url =   response.xpath('//*[@class="js-course-pagination"]//a[contains(@aria-label,"Next")]/@href').extract()

The spider goes to next pages but the result is not saved in the output file.

I'm guessing the problem is in the pipeline, where the files are created:

class MultiCSVItemPipeline(object):
    CSVDir = '/moocs/scripts/moocs/moocs/'
    SaveTypes = ['moocs','moocsreview']

    def __init__(self):
        dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)

    def spider_opened(self, spider):
        self.files = dict([ (name, open(CSVDir+name+'.csv','w+b')) for name in self.SaveTypes ])
        self.exporters = dict([ (name,CsvItemExporter(self.files[name])) for name in self.SaveTypes])
        [e.start_exporting() for e in self.exporters.values()]

    def spider_closed(self, spider):
        [e.finish_exporting() for e in self.exporters.values()]
        [f.close() for f in self.files.values()]

    def process_item(self, item, spider):
        what = item_type(item)
        if what in set(self.SaveTypes):
            self.exporters[what].export_item(item)
        return item

Answer 1

Are you sure the spider is doing the pagination properly?

When you do this:

next_page_url =   response.xpath('//*[@class="js-course-pagination"]//a[contains(@aria-label,"Next")]/@href').extract()

extract() returns a list of results that you are then passing into the url parameter of the Request:

yield Request(url=next_page_url, callback=self.parse)

But url must be a string or unicode value, so, doing that will generate the following error:

TypeError: Request url must be str or unicode, got list:

It can be solved by using the extract_first() method, and I would also check that the value is not None:

next_page_url =   response.xpath('//*[@class="js-course-pagination"]//a[contains(@aria-label,"Next")]/@href').extract_first()
if next_page_url:
    yield Request(url=next_page_url) # parse is the callback by default

Please, try this and tell me if it solved your problem