Running multiple spiders per one process - no output data saved

Question

Please help! Trying to create currency parser using Scrapy. Created two spiders which works properly if run them separately. Trying to run two spiders per one process - gives no data in output. No matter if trying to save it to txt, json or database.

Using MySQL database.

But running one spider per one process - saves data succesfully.

The only way to save data to database or whatever - is to run scrapy crawl Liga && scrapy crawl IFinance.

Main spiders code:

from scrapy.selector import Selector
from scrapy.crawler import CrawlerProcess

from ..items import CurparserItem

class LigaSpider(Spider):
    name = "Liga"
    allowed_domains = ["finance.liga.net"]
    start_urls = [
        "https://finance.liga.net/currency/nbu",
    ]

    def parse(self, response):
        cur_list = Selector(response).xpath('//table[contains(@class, "default-table-finance course-two-col")]/tbody/tr')
        for currency in cur_list:
            item = CurparserItem()

            cur_code_raw = currency.xpath('./td[1]/a/text()').extract_first()
            item['cur_code'] = " ".join(cur_code_raw.split())
            item['cur_name'] = currency.xpath('./td[2]/text()').extract_first()
            multiple_value = currency.xpath('./td[4]/div/text()').extract_first()
            hrn_points = currency.xpath('./td[3]/text()').extract_first()
            item['cur_value'] = round(float(multiple_value)/int(hrn_points), 3)
            item['cur_behavior'] = currency.xpath('./td[4]/span/text()').extract_first()

            yield item


class IFinance(Spider):
    name = "IFinance"
    allowed_domains = ["finance.i.ua"]
    start_urls = [
        "https://finance.i.ua/nbu/",
    ]

    def parse(self, response):
        cur_list = Selector(response).xpath('//table[contains(@class, "table-data")]/tbody/tr')
        for currency in cur_list:
            item = CurparserItem()

            item['cur_code'] = currency.xpath('./th/text()').extract_first()
            item['cur_name'] = currency.xpath('./td[2]/text()').extract_first()

            raw_behav_sign = currency.xpath('./td[3]/span/@class').extract_first()
            behav_sign = ""
            if raw_behav_sign == "value -increase":
                behav_sign = "+ "
            elif raw_behav_sign == "value -decrease":
                behav_sign = "- "
            else:
                behav_sign = ""

            item['cur_value'] = currency.xpath('./td[3]/span/span[1]/text()').extract_first()
            raw_behav_value = currency.xpath('./td[3]/span/span[2]/text()').extract_first()
            item['cur_behavior'] = behav_sign + raw_behav_value

            yield item


process = CrawlerProcess()
process.crawl(LigaSpider)
process.crawl(IFinance)
process.start()

Pipelines file:

import mysql.connector


class CurparserPipeline(object):

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.spider.name)

    def __init__(self, spider_name):
        self.create_connection()
        self.create_table(spider_name)

    def create_connection(self):
        self.conn = mysql.connector.connect(
            host='localhost',
            user='root',
            passwd='3480stfgDev',
            database='currencydb'
        )
        self.curr = self.conn.cursor()

    def create_table(self, spider_name):
        self.curr.execute("""DROP TABLE IF EXISTS %s""" % spider_name)
        self.curr.execute("""create table %s(
                            cur_code text,
                            cur_name text,
                            cur_value float,
                            cur_behavior text
                            )""" % spider_name)

    def process_item(self, item, spider):
        self.store_db(item, spider)
        return item

    def store_db(self, item, spider):
        spider_name = spider.name
        self.curr.execute("""insert into """ + spider_name + """ values (%s,%s,%s,%s)""", (
            item['cur_code'],
            item['cur_name'],
            item['cur_value'],
            item['cur_behavior']
        ))
        self.conn.commit()```

Answer 1

When you use CrawlerProcess, project settings (i.e. from a settings.py file in your project folder) are not used.

See https://docs.scrapy.org/en/latest/topics/practices.html#run-scrapy-from-a-script for information on how to pass your project settings:

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

process = CrawlerProcess(get_project_settings())