terminal partitioning for each of subprocesses prints
Say we have multiple subprocesses like the following which has some results printed in real time to sys.stdout or sys.stderr.
proc1 = subprocess.Popen(['cmd1'],
env=venv1,
stdout=sys.stdout,
stderr=sys.stderr,
)
proc2 = subprocess.Popen(['cmd2'],
env=venv2,
stdout=sys.stdout,
stderr=sys.stderr,
)
However, after executing this script in the terminal, while looking at what is being printed, it is not easy to distinguish which print is from the first process and which is from the second.
Is there a solution for this to see the stdout of each process separately, like if the terminal screen could be partitioned and each partition would have shown the printing results from each process?
I have written for you a curses application which will do what you request: divide the terminal window into a number of partitions and then watch the different output streams in the different partitions.
The function watch_fd_in_panes will take a list of lists, where the sub-lists specify which file descriptors to watch inside each partition.
Here is what your example calling code will look like:
import subprocess
from watcher import watch_fds_in_panes
proc1 = subprocess.Popen('for i in `seq 30`; do date; sleep 1 ; done',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
# this process also writes something on stderr
proc2 = subprocess.Popen('ls -l /asdf; for i in `seq 20`; do echo $i; sleep 0.5; done',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
proc3 = subprocess.Popen(['echo', 'hello'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
watch_fds_in_panes([[proc1.stdout.fileno(), proc1.stderr.fileno()],
[proc2.stdout.fileno(), proc2.stderr.fileno()],
[proc3.stdout.fileno(), proc3.stderr.fileno()]],
sleep_at_end=3.)
except KeyboardInterrupt:
print("interrupted")
proc1.kill()
proc2.kill()
proc3.kill()
To run it you will need these two files:
panes.py
import curses
class Panes:
"""
curses-based app that divides the screen into a number of scrollable
panes and lets the caller write text into them
"""
def start(self, num_panes):
"set up the panes and initialise the app"
# curses init
self.num = num_panes
self.stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
# split the screen into number of panes stacked vertically,
# drawing some horizontal separator lines
scr_height, scr_width = self.stdscr.getmaxyx()
div_ys = [scr_height * i // self.num for i in range(1, self.num)]
for y in div_ys:
self.stdscr.addstr(y, 0, '-' * scr_width)
self.stdscr.refresh()
# 'boundaries' contains y coords of separator lines including notional
# separator lines above and below everything, and then the panes
# occupy the spaces between these
boundaries = [-1] + div_ys + [scr_height]
self.panes = []
for i in range(self.num):
top = boundaries[i] + 1
bottom = boundaries[i + 1] - 1
height = bottom - top + 1
width = scr_width
# create a scrollable pad for this pane, of height at least
# 'height' (could be more to retain some scrollback history)
pad = curses.newpad(height, width)
pad.scrollok(True)
self.panes.append({'pad': pad,
'coords': [top, 0, bottom, width],
'height': height})
def write(self, pane_num, text):
"write text to the specified pane number (from 0 to num_panes-1)"
pane = self.panes[pane_num]
pad = pane['pad']
y, x = pad.getyx()
pad.addstr(y, x, text)
y, x = pad.getyx()
view_top = max(y - pane['height'], 0)
pad.refresh(view_top, 0, *pane['coords'])
def end(self):
"restore the original terminal behaviour"
curses.nocbreak()
self.stdscr.keypad(0)
curses.echo()
curses.endwin()
and watcher.py
import os
import select
import time
from panes import Panes
def watch_fds_in_panes(fds_by_pane, sleep_at_end=0):
"""
Use panes to watch output from a number of fds that are writing data.
fds_by_pane contains a list of lists of fds to watch in each pane.
"""
panes = Panes()
npane = len(fds_by_pane)
panes.start(npane)
pane_num_for_fd = {}
active_fds = []
data_tmpl = {}
for pane_num, pane_fds in enumerate(fds_by_pane):
for fd in pane_fds:
active_fds.append(fd)
pane_num_for_fd[fd] = pane_num
data_tmpl[fd] = bytes()
try:
while active_fds:
all_data = data_tmpl.copy()
timeout = None
while True:
fds_read, _, _ = select.select(active_fds, [], [], timeout)
timeout = 0
if fds_read:
for fd in fds_read:
data = os.read(fd, 1)
if data:
all_data[fd] += data
else:
active_fds.remove(fd) # saw EOF
else:
# no more data ready to read
break
for fd, data in all_data.items():
if data:
strng = data.decode('utf-8')
panes.write(pane_num_for_fd[fd], strng)
except KeyboardInterrupt:
panes.end()
raise
time.sleep(sleep_at_end)
panes.end()
Finally, here is a screenshot of the above code in action:
In this example, we are monitoring both stdout and stderr of each process in the relevant partition. In the screenshot, the line that proc2 wrote to stderr before the start of the loop (regarding /asdf) has appeared after the first line that proc2 wrote to stdout during the first iteration of the loop (i.e. the 1 which has since scrolled off the top of the partition), but this is cannot be controlled because they were written to different pipes.