Search code examples
pythonpython-multiprocessing

How to exit all the joined processes in case any one has an exception - Python


I have a python pool of processes , if an exception occurs in any one of the process i want to exit the execution of the pool

I have joined all the processes in the pool, so the join waits for every process to finish. If i raise sys.exit(1) inside the target function the system goes on infinite wait because the join is still waiting for process to complete.

How can exit the execution while using join in the code

from multiprocessing import Pool
import time
import sys

def printer(ip):
    try:
        for _ in xrange(5):
            print ip+str(_)
            time.sleep(1.0)
    except Exception as e:
        print e
        sys.exit(2)

def test():
    pool = Pool(processes=2)
    for i in ["hello",5]:
        result = pool.apply_async(printer,(i,))
    pool.close()    
    pool.join()
    print "good bye"

test()

Solution

  • Just return to the parent process the status of the operation and use a callback to react to failures.

    import time
    from multiprocessing import Pool
    
    
    def printer(ip):
        try:
            for _ in xrange(5):
                print ip+str(_)
                time.sleep(1.0)
            return True
        except Exception as e:
            print e
            return False
    
    
    class Worker():
        def __init__(self):
            self.pool = Pool(processes=2)
    
        def callback(self, result):
            if not result:
                print "Error raised in child process. Terminating.."
                self.pool.terminate()
    
        def do_job(self):
            for i in ["hello", 5]:
                self.pool.apply_async(printer, (i,), callback=self.callback)
            self.pool.close()
            self.pool.join()
            print "good bye"
    
    
    def test():
        w = Worker()
        w.do_job()