GET request with chunks in Python

Question

I'm getting the data from API using requests library doing like this:

import requests

url = "Some String"

headers = {

'Authorization':"Some Token"}

response = requests.request("GET", url, headers=headers)

But the file that I'm trying to get is very large so I receive the time exceptions error. How can I get it using chunks in request?

Thanks!

Answer 1

import requests    
import datetime
import pandas as pd


url = "some URL"
headers = {

'Authorization':"Some Token"}

start_date = datetime.datetime(2018, 6, 1)
end_date = datetime.datetime.now()
temp_end_date = start_date + datetime.timedelta(days=7)

output = dict()
while temp_end_date <= end_date:
    temp_url = url % (start_date.timestamp()*1000, temp_end_date.timestamp()*1000)
    response = requests.get(temp_url, headers=headers)
    temp_data = response.json()
    for key, value in temp_data.items():
        output_arr = output.get(key, [])
        output_arr.extend(value)
        output[key] = output_arr
    start_date = temp_end_date + datetime.timedelta(seconds=1)
    temp_end_date += datetime.timedelta(days=7)

data=output
df=pd.DataFrame(data)

df.head()