firstly i am very new to python the issue i get is the in the output url, the "limit" variable doesn't update on the second loop it shows : https://website.com/table1/?key=xxxx&limit=0,+100 when i want it to show https://website.com/table1/?key=xxxx&limit=100,+100
i can see the limit variable is incrementing by 100 but it is just not passing it through to the api url
import requests
import json
import csv
import pandas as pd
import os
os.remove('ladder.csv')
os.remove('ladder.json')
# Define the base URL and parameters
base_url = 'https://website.com/table1/?key=xxxx'
query_params = ",+"
limit = 0
offset = 100
full_url = f'{base_url}{limit}{query_params}{offset}'
# Function to fetch data from the API
def fetch_data(full_url):
response = requests.get(full_url)
print (response.url)
print (offset)
if response.status_code == 200:
return response.json()
else:
print(f"Error: {response.status_code}")
print("Response content:", response.content)
return None
# Initialize variables
all_data = []
has_more_data = True
# Loop to fetch data with pagination
while has_more_data:
# Fetch data
data = fetch_data(full_url)
if data:
# Debug: Print the response structure
print(limit,offset)
# Check if 'results' key exists in the response
if 'ranking' in data:
# Process the data (example: extend the all_data list with new records)
all_data.extend(data['ranking'])
# Check if there is more data to fetch
if offset >= 1000:
has_more_data = False
else:
if len(data['ranking']) < limit:
has_more_data = False
else:
# Update the offset for the next request
limit += offset
else:
print("Key 'ranking' not found in the response")
has_more_data = False
else:
has_more_data = False
# Example: Save all data to a JSON file
with open('ladder.json', 'w') as f:
json.dump(all_data, f, indent=4)
print("Data fetching complete. Total records fetched:", len(all_data))
with open('ladder.json', encoding='utf-8') as inputfile:
df = pd.read_json(inputfile)
df.to_csv('ladder.csv', encoding='utf-8', index=False)
Your issue seems to be with how the full_url
variable is constructed inside the loop. The full_url
is being set before the loop starts and is not updated within the loop, so the updated limit value isn't being used in subsequent requests.