I have several scripts which I run using Multiprocessing pool I am trying to do a progress bar based on the scripts completed.
I checked https://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console#=
but I cannot figure out how I can combine the scripts completed in the counter
import os
from multiprocessing import Pool
def run_process(process):
os.system('python {}'.format(process))
processes = ('script1.py', 'script2.py','script3.py','script4.py')
if __name__ == "__main__":
pool = Pool(processes=2)
pool.map(run_process, processes)
Here's a slightly differrent approach which uses concurrent.futures.ThreadPoolExecutor
instead of a multiprocessing.Pool
which make it simpler and more efficient than what's in my other answer.
Note it uses the same print_progress_bar.py
module that's in my other answer.
import concurrent.futures
import os
import subprocess
import sys
from print_progress_bar import print_progress_bar
progress_bar_kwargs = dict(prefix='Progress:', suffix='Complete', length=40)
# To simplify testing just using one script multiple times.
processes = ('./mp_scripts/script1.py', './mp_scripts/script1.py',
'./mp_scripts/script1.py', './mp_scripts/script1.py')
process_count = 0
def run_process(process):
global process_count
subprocess.run([sys.executable, process])
# Update process count and progress bar when it's done.
process_count += 1
print_progress_bar(process_count, len(processes), **progress_bar_kwargs)
print_progress_bar(0, len(processes), **progress_bar_kwargs) # Print 0% progress.
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
future_to_process = {executor.submit(run_process, process): process
for process in processes}
for future in concurrent.futures.as_completed(future_to_process):
process = future_to_process[future]
try:
_ = future.result()
except Exception as exc:
print()
print(f'{process} generated an exception: {exc}')
print('Done')