I am writing a helper class for handling multiple urls request in asynchronous way. The code is following.
class urlAsyncClient(object):
def __init__(self, url_arr):
self.url_arr = url_arr
async def async_worker(self):
result = await self.__run()
return result
async def __run(self):
pending_req = []
async with aiohttp.ClientSession() as session:
for url in self.url_arr:
r = self.__fetch(session, url)
pending_req.append(r)
#Awaiting the results altogether instead of one by one
result = await asyncio.wait(pending_req)
return result
@staticmethod
async def __fetch(session, url):
async with session.get(url) as response: #ERROR here
status_code = response.status
if status_code == 200:
return await response.json()
else:
result = await response.text()
print('Error ' + str(response.status_code) + ': ' + result)
return {"error": result}
As awaiting the result one by one seems meaningless in asynchronous. I put them into an array and wait together by await asyncio.wait(pending_req).
But seems like it is not the correct way to do it as I get the following error
in __fetch async with session.get(url) as response: RuntimeError: Session is closed
May I know the correct way to do it? Thanks.
because session has closed before you await it
async with aiohttp.ClientSession() as session:
for url in self.url_arr:
r = self.__fetch(session, url)
pending_req.append(r)
#session closed hear
you can make session an argument to __run, like this
async def async_worker(self):
async with aiohttp.ClientSession() as session:
result = await self.__run(session)
return result
# session will close hear
async def __run(self, session):
pending_req = []
for url in self.url_arr:
r = self.__fetch(session, url)
pending_req.append(r)
#Awaiting the results altogether instead of one by one
result = await asyncio.wait(pending_req)
return result