making a variable update in an API Loop

Question

firstly i am very new to python the issue i get is the in the output url, the "limit" variable doesn't update on the second loop it shows : https://website.com/table1/?key=xxxx&limit=0,+100 when i want it to show https://website.com/table1/?key=xxxx&limit=100,+100

i can see the limit variable is incrementing by 100 but it is just not passing it through to the api url

import requests
import json
import csv
import pandas as pd
import os
 
os.remove('ladder.csv')
os.remove('ladder.json')
 
 
# Define the base URL and parameters
base_url = 'https://website.com/table1/?key=xxxx'
 
query_params = ",+"
limit = 0
offset = 100
full_url = f'{base_url}{limit}{query_params}{offset}'
 
 
# Function to fetch data from the API
def fetch_data(full_url):
    response = requests.get(full_url)
    print (response.url)
 
    print (offset)
    if response.status_code == 200:
        return response.json()
    else:
        print(f"Error: {response.status_code}")
        print("Response content:", response.content)
        return None
 
# Initialize variables
all_data = []
has_more_data = True
     
# Loop to fetch data with pagination
while has_more_data:
    # Fetch data
    data = fetch_data(full_url)
     
    if data:
        # Debug: Print the response structure
         
        print(limit,offset)
         
        # Check if 'results' key exists in the response
        if 'ranking' in data:
            # Process the data (example: extend the all_data list with new records)
            all_data.extend(data['ranking'])
             
            
            # Check if there is more data to fetch
            if offset >= 1000:
                    has_more_data = False
            else:
 
                if len(data['ranking']) < limit:
                  has_more_data = False
                 
                else:
                # Update the offset for the next request
                  limit += offset
        else:
            print("Key 'ranking' not found in the response")
            has_more_data = False
             
    else:
        has_more_data = False
         
# Example: Save all data to a JSON file
with open('ladder.json', 'w') as f:
    json.dump(all_data, f, indent=4)
 
print("Data fetching complete. Total records fetched:", len(all_data))
 
with open('ladder.json', encoding='utf-8') as inputfile:
    df = pd.read_json(inputfile)
 
df.to_csv('ladder.csv', encoding='utf-8', index=False)

Answer 1

Your issue seems to be with how the full_url variable is constructed inside the loop. The full_url is being set before the loop starts and is not updated within the loop, so the updated limit value isn't being used in subsequent requests.