I'm setting up a Django
Views that requests products data from an API, parse them with BeautifulSoup
, apply the googletrans
module and save the response into my Postgresql database. I have a 429
HTTP error when I run my script because I send to many requests at once and I was adviced to add a time.sleep()
method of 2 seconds to fix the issue.
The only thing is that I'm not sure where should I place this method in my script, and I need to make sure that its the right way to do it before risking of getting my IP address blocked by Google tranlsate API
again.
This is my Views:
from bs4 import BeautifulSoup
from googletrans import Translator
import requests
import json
def api_data(request):
if request.GET.get('mybtn'): # to improve, == 'something':
resp_1 = requests.get(
"https://www.headout.com/api/public/v1/product/listing/list-by/city?language=fr&cityCode=PARIS&limit=5000¤cyCode=CAD",
headers={
"Headout-Auth": HEADOUT_PRODUCTION_API_KEY
})
resp_1_data = resp_1.json()
base_url_2 = "https://www.headout.com/api/public/v1/product/get/"
translator = Translator()
for item in resp_1_data['items']:
print('translating item {}'.format(item['id']))
# concat ID to the URL string
url = '{}{}'.format(base_url_2, item['id'] + '?language=fr')
# make the HTTP request
resp_2 = requests.get(
url,
headers={
"Headout-Auth": HEADOUT_PRODUCTION_API_KEY
})
resp_2_data = resp_2.json()
descriptiontxt = resp_2_data['contentListHtml'][0]['html'][0:2040] + ' ...'
#Parsing work
soup = BeautifulSoup(descriptiontxt, 'lxml')
parsed = soup.find('p').text
#Translation doesn't work
translation = translator.translate(parsed, dest='fr')
titlename = item['name']
titlefr = translator.translate(titlename, dest='fr')
destinationname = item['city']['name']
destinationfr = translator.translate(destinationname, dest='fr')
Product.objects.get_or_create(
title=titlefr.text,
destination=destinationfr.text,
description=translation.text,
link=item['canonicalUrl'],
image=item['image']['url']
)
return render(request, "form.html")
This is what I came out with, is this okay?
Product.objects.get_or_create(
title=titlefr.text,
destination=destinationfr.text,
description=translation.text,
link=item['canonicalUrl'],
image=item['image']['url']
)time.sleep(2) #here
or like this:
resp_1 = requests.get(
"https://www.headout.com/api/public/v1/product/listing/list-by/city?language=fr&cityCode=PARIS&limit=5000¤cyCode=CAD",
headers={
"Headout-Auth": HEADOUT_PRODUCTION_API_KEY
}, time.sleep(2)) #or here
Please help.
EDIT
Based on @jay answer, Is this correct?
Product.objects.get_or_create(
title=titlefr.text,
destination=destinationfr.text,
description=translation.text,
link=item['canonicalUrl'],
image=item['image']['url']
)
#at the very end of the for loop
time.sleep(2)
I would recommend at the very end of the for loop, so that it waits 2 seconds before relooping.
destinationname = item['city']['name']
destinationfr = translator.translate(destinationname, dest='fr')
Product.objects.get_or_create(
title=titlefr.text,
destination=destinationfr.text,
description=translation.text,
link=item['canonicalUrl'],
image=item['image']['url']
)
time.sleep(2)
return render(request, "form.html")