I'm trying to scrape data from this website https://pigeon-ndb.com/races/. At first, I thought the problem would be easy to solve if I figured out how to select elements from a drop-down menu, but it's ending up being more complicated than anticipated.
I ideally want to iterate through all the years & seasons (2010-2019) and then through all the records of all the organizations and the races. In summary, scrape the data from all the tables located in the website using scrapy (no selenium).
I know that the problem involves utilizing the GET requests for the drop-down menus (3 total) like so:
https://pigeon-ndb.com/api/?request=get_databases (somehow select the json elements for year and season for the next request)
https://pigeon-ndb.com/api/?request=get_organizations&database=2010%20OB&_=1557098607652 (requires the year and season from the previous request to work)
https://pigeon-ndb.com/api/?request=get_races&organization=&_=1557098607653 (requires the name of the organization from the previous request (#2) to work)
The following code is a basic outline of the scrapy spider I plan to use, subject to change:
from scrapy import Spider
from scrapy.http import Request
class PigeonSpider(Spider):
name = 'pigeonspider'
allowed_domains = ['pigeon-ndb.com']
start_urls = ['https://pigeon-ndb.com/races/']
def parse(self, response):
pass
def parse2(self,response):
pass
def parse3(self,response):
pass
Since it's a GET request, I'm expecting to use this multiple times (or some variation):
yield Request(url,callback=self.parse2)
I think I'll need to incorporate json for the dynamic parts of the scraping process, but not sure if it's the best approach
In scrapy shell:
import json
jsonresponse = json.loads(response.body)
This is the json output for the first request ( https://pigeon-ndb.com/api/?request=get_databases):
{'data': [{'year': '2010', 'season': 'OB'}, {'year': '2010', 'season': 'YB'}, {'year': '2011', 'season': 'OB'}, {'year': '2011', 'season': 'YB'}, {'year': '2012', 'season': 'OB'}, {'year': '2012', 'season': 'YB'}, {'year': '2013', 'season': 'OB'}, {'year': '2013', 'season': 'YB'}, {'year': '2014', 'season': 'OB'}, {'year': '2014', 'season': 'YB'}, {'year': '2015', 'season': 'OB'}, {'year': '2015', 'season': 'YB'}, {'year': '2016', 'season': 'OB'}, {'year': '2016', 'season': 'YB'}, {'year': '2017', 'season': 'OB'}, {'year': '2017', 'season': 'YB'}, {'year': '2018', 'season': 'OB'}, {'year': '2018', 'season': 'YB'}, {'year': '2019', 'season': 'OB'}], 'jsonapi': {'version': 2.2, 'db': 'pigeon-ndb'}, 'meta': {'copyright': 'Copyright 2019 Craig Vander Galien', 'authors': ['Craig Vander Galien']}}
I'm still learning scrapy so would appreciate example code on how to approach this problem. Thanks!
Edit:
So I tried implementing the following code but I'm running into errors:
from scrapy import Spider
from scrapy.http import Request
import json
class PigeonSpider(Spider):
name = 'pigeonspider'
allowed_domains = ['pigeon-ndb.com']
start_urls = ['https://pigeon-ndb.com/races/']
def parse(self, response):
result = json.loads(response.body)
for node in result['data']:
yield Request(
url = 'https://pigeon-ndb.com/api/?request=get_organizations&database={year}%20{season}'.format(year=node["year"], season=node["season"]),
callback = self.parse_organizations,
cookies = {'database':'{year} {season}'.format(year=node['year'],season=node['season'])},
meta = {
'year': node['year'],
'season': node['season'],
}
)
def parse_organizations(self,response):
result = json.loads(response.body)
for node in result['data']:
org_num = node['orgNum']
if node['orgNum'] is None:
org_num = 'null'
yield Request(
url='https://pigeon-ndb.com/api/?request=get_races&organization={org_name}&orgNum={org_num}'.format(org_name=node["Sys"], org_num=org_num),
callback=self.parse_races,
headers={'x-requested-with': 'XMLHttpRequest'},
cookies={'database':'{year} {season}'.format(year=response.meta["year"], season=response.meta["season"])}
)
def parse_races(self,response):
result = json.loads(response.body)
for node in result['clockings']['data']:
yield {
'race':node['racename'],
'season':node['season'],
'date':node['date'],
'year':node['year'],
'time':node['Time'],
'complevel':node['CompLevel'],
'class': node['class'],
'city': node['City'],
'zip': node['Zip'],
'state': node['State'],
'entry': node['entry'],
'first_name':node['FirstName'],
'last_name':node['LastName'],
'line_num':node['LineNum'],
'band_num':node['band_no'],
'color': node['BB'],
'sex': node['sex'],
'arrival_time':node['arri_time'],
'distance':node['distance'],
'speed':node['speed'],
'reg_points':node['reg_points'],
'std_points':node['std_points'],
'unirate':node['unirate'],
'place': node['Place'],
}
When running spider (error):
Traceback (most recent call last):
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/site-packages/scrapy/utils/defer.py", line 102, in iter_errback
yield next(it)
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/site-packages/scrapy/spidermiddlewares/offsite.py", line 30, in process_spider_output
for x in result:
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/site-packages/scrapy/spidermiddlewares/referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/site-packages/scrapy/spidermiddlewares/urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/site-packages/scrapy/spidermiddlewares/depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "/home/glenn/Projects/pigeonscraper/pigeonscraper/spiders/pigeonspider.py", line 13, in parse
result = json.loads(response.body)
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/json/__init__.py", line 348, in loads
return _default_decoder.decode(s)
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/home/glenn/anaconda3/envs/scraperenv/lib/python3.7/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
First of all you need to set two params (database and season) using cookies. After that you can iterate over JSON results:
from scrapy import Spider
from scrapy.http import Request
import json
class PigeonSpider(Spider):
name = 'pigeonspider'
allowed_domains = ['pigeon-ndb.com']
start_urls = ['https://pigeon-ndb.com/api/?request=get_databases']
def parse(self, response):
result = json.loads(response.body)
for node in result["data"]:
yield Request(
url="https://pigeon-ndb.com/api/?request=get_organizations&database={year}%20{season}".format(year=node["year"], season=node["season"]),
callback=self.parse_organizations,
# headers={'x-requested-with': "XMLHttpRequest", 'referer': "https://pigeon-ndb.com/races/"},
cookies={'database':'{year} {season}'.format(year=node["year"], season=node["season"])},
meta={
"year": node["year"],
"season": node["season"],
}
)
pass
def parse_organizations(self,response):
result = json.loads(response.body)
for node in result["data"]:
org_num = node["orgNum"]
if node["orgNum"] is None:
org_num = "null"
yield Request(
url="https://pigeon-ndb.com/api/?request=get_races&organization={org_name}&orgNum={org_num}".format(org_name=node["Sys"], org_num=org_num),
callback=self.parse_races,
headers={'x-requested-with': "XMLHttpRequest"},
cookies={'database':'{year} {season}'.format(year=response.meta["year"], season=response.meta["season"])}
)
pass
def parse_races(self,response):
result = json.loads(response.body)
for race_key in result["data"].keys():
race_date = result["data"][race_key]["date"]
race_release_time = result["data"][race_key]["release_time"]
race_bird_attend = result["data"][race_key]["bird_attend"]
# etc.
pass
Update You're completely ignoring my comments. parse_race_details was not implemented in your code at all!
from scrapy import Spider
from scrapy.http import Request
import json
class PigeonSpider(Spider):
name = 'pigeonspider'
allowed_domains = ['pigeon-ndb.com']
start_urls = ['https://pigeon-ndb.com/api/?request=get_databases']
debug = False
def parse(self, response):
result = json.loads(response.body)
for node in result["data"]:
yield Request(
url="https://pigeon-ndb.com/api/?request=get_organizations&database={year}%20{season}".format(
year=node["year"], season=node["season"]),
callback=self.parse_organizations,
# headers={'x-requested-with': "XMLHttpRequest", 'referer': "https://pigeon-ndb.com/races/"},
cookies={
'database': '{year} {season}'.format(
year=node["year"],
season=node["season"])},
meta={
"year": node["year"],
"season": node["season"],
},
dont_filter=True,
)
# Debug
if self.debug:
break
pass
def parse_organizations(self, response):
result = json.loads(response.body)
for node in result["data"]:
org_num = node["orgNum"]
if node["orgNum"] is None:
org_num = "null"
yield Request(
url="https://pigeon-ndb.com/api/?request=get_races&organization={org_name}&orgNum={org_num}".format(org_name=node["Sys"], org_num=org_num),
callback=self.parse_races,
headers={'x-requested-with': "XMLHttpRequest"},
cookies={'database': '{year} {season}'.format(year=response.meta["year"], season=response.meta["season"])},
dont_filter=True,
# meta={
# "year": response.meta["year"],
# "season": response.meta["season"],
# },
)
# Debug
if self.debug:
break
pass
def parse_races(self, response):
result = json.loads(response.body)
if result["response"] == "failed":
print("Failed response!")
for race_key in result["data"].keys():
race_name = result["data"][race_key]["racename"]
race_date = result["data"][race_key]["date"].replace("/", "%2F")
race_time = result["data"][race_key]["Time"]
yield Request(
url="https://pigeon-ndb.com/api/?request=get_race_details&racename={race_name}&date={race_date}&time={race_time}".format(race_name=race_name, race_date=race_date, race_time=race_time),
callback=self.parse_race_details,
headers={'x-requested-with': "XMLHttpRequest"},
# cookies={'database': '{year} {season}'.format(year=response.meta["year"], season=response.meta["season"])},
dont_filter=True,
)
# Debug
if self.debug:
break
pass
def parse_race_details(self, response):
result = json.loads(response.body)
if result["response"] == "failed":
print("Failed response!")
for node in result['data']['clockings']['data']:
yield {
'race':node['racename'],
'season':node['season'],
'date':node['date'],
'year':node['year'],
'time':node['Time'],
'complevel':node['CompLevel'],
'class': node['Class'],
'city': node['City'],
'zip': node['Zip'],
'state': node['State'],
'entry': node['entry'],
'first_name':node['FirstName'],
'last_name':node['LastName'],
'line_num':node['LineNum'],
'band_num':node['band_no'],
'color': node['BB'],
'sex': node['sex'],
'arrival_time':node['arri_time'],
'distance':node['distance'],
'speed':node['speed'],
'reg_points':node['reg_points'],
'std_points':node['std_points'],
'unirate':node['unirate'],
'place': node['Place'],
}
pass