I have downloaded the code here https://github.com/aosabook/500lines.
This is an educational open source project with many sub-projects. The one I am studying currently is the 'crawler' project. It requires python 3.4+ and aiohttp 1.2+. My current setup is Windows 10, Python 3.6.1, and aiohttp 2.0.4.
Strangely, my tests do not run and it is related to aiohttp. Can anybody familiar with aiohttp tell me why?
Here is the output I get:
C:\Users\Primary User\Desktop\500lines-master\crawler\code>python test.py
C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web.py:54: ResourceWarning: loop argument is deprecated
warnings.warn("loop argument is deprecated", ResourceWarning)
E.E..EEEEEEEEEE..
======================================================================
ERROR: test_content_type (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 314, in test_content_type
self.add_page(content_type='foo')
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_encoding (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 309, in test_encoding
test_charset('', 'utf-8')
File "test.py", line 305, in test_charset
self.add_page(url, content_type=content_type)
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_link (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 126, in test_link
self.add_page('/', ['/foo'])
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_link_cycle (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 137, in test_link_cycle
url = self.add_page('/foo', ['/bar'])
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_max_tasks (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 264, in test_max_tasks
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_max_tries (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 281, in test_max_tries
self.add_handler('/', handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_non_html (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 322, in test_non_html
self.add_page('/xml', body=body, content_type='application/xml')
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_non_http (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 333, in test_non_http
self.add_page(body=body)
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_prohibited_host (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 152, in test_prohibited_host
self.add_page('/', ['http://example.com'])
File "test.py", line 93, in add_page
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_redirect (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 196, in test_redirect
url = self.add_redirect('/', foo)
File "test.py", line 101, in add_redirect
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_redirect_cycle (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 214, in test_redirect_cycle
url = self.add_redirect('/bar', foo)
File "test.py", line 101, in add_redirect
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
======================================================================
ERROR: test_redirect_join (__main__.TestCrawler)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test.py", line 230, in test_redirect_join
self.add_redirect('/foo', baz)
File "test.py", line 101, in add_redirect
self.add_handler(url, handler)
File "test.py", line 76, in add_handler
self.app.router.add_route('GET', url, handler)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 823, in add_route
resource = self.add_resource(path, name=name)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 787, in add_resource
self.register_resource(resource)
File "C:\Users\Primary User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\web_urldispatcher.py", line 762, in register_resource
"Cannot register a resource into frozen router.")
RuntimeError: Cannot register a resource into frozen router.
----------------------------------------------------------------------
Ran 17 tests in 0.034s
FAILED (errors=12)
C:\Users\Primary User\Desktop\500lines-master\crawler\code>
I am also learning it. I found this issue https://github.com/hangoutsbot/hangoutsbot/pull/655. Then code like this
@asyncio.coroutine
def _create_server(self):
app = web.Application(loop=self.loop)
return app
def add_handler(self, url, handler):
self.app.router.add_route('GET', url, handler)
handler_factory = self.app.make_handler(debug=True)
srv = yield from self.loop.create_server(
handler_factory, '127.0.0.1', self.port)
# Prevent "Task destroyed but it is pending" warnings.
self.addCleanup(lambda: self.loop.run_until_complete(
handler_factory.finish_connections()))
self.addCleanup(srv.close)
I think you should run crawl.py. Another problem, aiohttp 2.0.4 has no results after running for me. Using aiohttp 0.21 fixed it.