python 3 pool map does not follow processes instruction in mac

Question

I have the following code

import multiprocessing, subprocess

def runGate(alias):
    #worker function
    command = ['Gate','-a',alias,'main.mac']
    return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

p = multiprocessing.Pool(processes=4)
p.map(runGate, aliases)
p.close()
p.join()

where aliases is a set of 1k+ combinations, obviously I dont want to launch 1k+ processes at the same time, only 4. But that is not what it is happening, actually 1k+ processes are launched and my mac restarts. How can I launch ONLY 4 processes at the time?

Answer 1

This was the solution that actually worked

def call_proc(cmd):
    """ This runs in a separate thread. """
    #subprocess.call(shlex.split(cmd))  # This will block until cmd finishes
    p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    return (out, err)

pbar = ProgressBar(widgets=[SimpleProgress()], maxval=len(aliases)).start()

pool = multiprocessing.Pool(processes=6)  #(multiprocessing.cpu_count())
results = []
for alias in aliases:
    command = 'Gate -a '+alias+' main.mac'
    pool.apply_async(call_proc, (command,), callback=results.append)

while len(results) != len(aliases):
    pbar.update(len(results))
    sleep(0.5)
pbar.finish()

# Close the pool and wait for each running task to complete
pool.close()
pool.join()