Stop Gracefully Tornado ioLoop

Question

I have this async worker functionality using tornado's ioloop. I'm trying to shutdown the loop gracefully on Ctrl+C but getting the following error

tornado.ioloop.TimeoutError: Operation timed out after None seconds

I know I can catch it but I do want to finish the process in a graceful way, how can I achieve that?

#!/usr/bin/env python
import time
import signal
import random

from tornado import gen, ioloop, queues

concurrency = 10

def sig_exit(signum, frame):
    ioloop.IOLoop.current().add_callback_from_signal(shutdown)

def shutdown():
    print('Will shutdown in few seconds ...')
    io_loop = ioloop.IOLoop.current()

    deadline = time.time() + 3

    def stop_loop():
        now = time.time()
        if now < deadline and (io_loop._callbacks or io_loop._timeouts):
            io_loop.add_timeout(now + 1, stop_loop)
        else:
            io_loop.stop()
            print('Shutdown')

    stop_loop()

@gen.coroutine
def main():
    q = queues.Queue()
    q.put(1)

    @gen.coroutine
    def do_stuff():
        print("doing stuff")
        yield gen.Task(ioloop.IOLoop.instance().add_timeout, time.time() + random.randint(1, 5))
        print("done doing stuff")

    @gen.coroutine
    def worker():
        while True:
            yield do_stuff()

    for _ in range(concurrency):
        worker()

    yield q.join()


if __name__ == '__main__':
    signal.signal(signal.SIGTERM, sig_exit)
    signal.signal(signal.SIGINT, sig_exit)

    io_loop = ioloop.IOLoop.instance()
    io_loop.run_sync(main)

Answer 1

If you're using run_sync, you can no longer call IOLoop.stop - run_sync is now responsible for that. So if you want to make this shutdown "graceful" (instead of just raising a KeyboardInterrupt at the point where you now call stop() and exiting with a stack trace), you need to change the coroutine passed to run_sync so it exits.

One possible solution is a tornado.locks.Event:

# Create a global Event
shutdown_event = tornado.locks.Event()

def shutdown():
    # Same as in the question, but instead of `io_loop.stop()`:
    shutdown_event.set()

@gen.coroutine
def main():
    # Use a WaitIterator to exit when either the queue 
    # is done or shutdown is triggered. 
    wait_iter = gen.WaitIterator(q.join(), shutdown_event.wait())
    # In this case we just want to wait for the first one; we don't
    # need to actually iterate over the WaitIterator. 
    yield wait_iter.next()