Search code examples
pythonweb-scraping

Python stuck on page 1


What is the best way to get this to read each next page of results? Currently data is pulling but only page 1

import requests
import json

page = 1
url = "https://api-prod.grip.events/1/container/4368/search?search=&sort=name&order=asc&type_id=4907,4906,5265,4964,4904,1026,4908&page=%d"
headers = {
    'x-authorization': 'a422cc2a-31fb-4b4e-a1bd-a34b561adc6c'
}
with open("list.txt", "w") as f:
    for page in range(1, 1000):
        response = requests.get(url % page, headers=headers).json()
        contacts = response["data"]
        for contact in contacts:
            target = "%s\t%s\t%s\t%s" % (contact["company_name"], contact["job_title"], contact["name"],  contact["job_industry"])
            f.write(target + "\n")
            print(target)

Solution

  • found your site's encoding was utf-8 so maybe try this:

    import requests
    import json
    
    page = 1
    url = "https://api-prod.grip.events/1/container/4368/search?search=&sort=name&order=asc&type_id=4907,4906,5265,4964,4904,1026,4908&page=%d"
    headers = {
        'x-authorization': 'a422cc2a-31fb-4b4e-a1bd-a34b561adc6c'
    }
    with open("list.txt", "w", encoding='utf-8') as f: #added " encoding='utf-8' "
        for page in range(1, 1000):
            response = requests.get(url % page, headers=headers).json()
            contacts = response["data"]
            for contact in contacts:
                target = "%s\t%s\t%s\t%s" % (contact["company_name"], contact["job_title"], contact["name"],  contact["job_industry"])
                f.write(target + "\n")
                print(target)