I'm new to asynco but I am one long user of multithreding and multiprocessing. I really was trying to figure out what am I doing wrong here that this error is appeararing.
This is fully runnable code in python 3.7 that returns the error, so anyone can test it. Can anyone tell me what am I doing wrong, where did my logic went bad. I would really appreciate feedback.
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
try this:
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
#loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
result:
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://example.org/birthday
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://www.example.com/
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
Process finished with exit code 0
you may only remove the loop.close() from SecondClass() because that will prevent the Script from running all the tasks...