Stopping the execution of threads in Python. The thread is blocked before join method

Question

I'm writing the program which execute child programs in a new process and read stdout of these programs in the separate thread and write it in StyledTextCtrl control. I faced with problem of stopping the execution of threads. I have the following code:

import subprocess
from threading import Thread, Lock
import wx
import wx.stc


import logging
logging.basicConfig(level=logging.DEBUG,
                    format='[%(levelname)s] (%(threadName)-10s) %(module)10s:%(funcName)-15s %(message)s',
                    )

class ReadThread(Thread):
    def __init__(self, subp, fd, append_text_callback):
        Thread.__init__(self)
        self.killed = False
        self.subp = subp
        self.fd = fd
        self.append_text_callback = append_text_callback

    def run(self):
        chunk = None
        while chunk != '' and not self.killed:
            self.subp.poll()
            chunk = self.fd.readline()
            if chunk:
                self.append_text_callback('%d: ' % self.subp.pid + chunk)
            if chunk == "":
                break
        logging.debug('END')


class MyForm(wx.Frame):

    def __init__(self):
        wx.Frame.__init__(self, None, wx.ID_ANY, "Example1")
        panel = wx.Panel(self, wx.ID_ANY)
        self.log = wx.stc.StyledTextCtrl(panel, wx.ID_ANY, size=(300,100), style=wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL)
        run_button = wx.Button(panel, wx.ID_ANY, 'Run')
        self.Bind(wx.EVT_BUTTON, self.onRun, run_button)
        self.Bind(wx.EVT_CLOSE, self.OnCloseFrame)
        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.log, 1, wx.ALL|wx.EXPAND, 5)
        sizer.Add(run_button, 0, wx.ALL|wx.CENTER, 5)
        panel.SetSizer(sizer)
        self.read_thread_1 = None
        self.read_thread_2 = None
        self.log_lock = Lock()            

    def OnCloseFrame(self, event):
        logging.debug('CLOSE')
        if self.read_thread_1:
            self.read_thread_1.killed = True
        if self.read_thread_2:
            self.read_thread_2.killed = True
        if self.read_thread_1:
            self.read_thread_1.join(timeout=1)
            logging.debug('after 1 join')
        if self.read_thread_2:
            self.read_thread_2.join(timeout=1)
            logging.debug('after 2 join')
        self.read_thread_1 = None
        self.read_thread_2 = None
        event.Skip()

    def onRun(self, event):
        cmd1 = "E:/threading_block/stdout_emulator.exe 50 500"
        subp1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        self.read_thread_1 = ReadThread(subp1, subp1.stdout, self.AppendText)
        self.read_thread_1.start()
        cmd2 = "E:/threading_block/stdout_emulator.exe 21 400"
        sp2 = subprocess.Popen(cmd2, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        self.read_thread_2 = ReadThread(sp2, sp2.stdout, self.AppendText)
        self.read_thread_2.start()
        logging.debug('END')

    def AppendText(self, text):
        self.log_lock.acquire()
        logging.debug('%s' % text)
        self.log.AddText(text)
        self.log_lock.release()

if __name__ == "__main__":
    app = wx.PySimpleApp()
    frame = MyForm().Show()
    app.MainLoop()

I stop the threads in OnCloseFrame method. I set killed attribute in True value and as a result it cause the finishing the execution of while loop in run method of ReadThread class. There is no problem when I close this application (under Windows) during the execution of threads. But If I remove timeout=1 from self.read_thread_1.join(timeout=1) and self.read_thread_2.join(timeout=1) there will be a blocking of program.

The program stuck before self.read_thread_1.join() in OnCloseFrame method.

What's the problem with this blocking? Why I should use timeout for join?

The C code of std_emulator.exe is the following:

#include <stdio.h>
#include <stdlib.h>
#include <windows.h>
#include <time.h>

int main(int argc, char *argv[])
{
  int s;
  int n;
  if (argc > 1)
  {    
    s = atoi(argv[1]);
    n = atoi(argv[2]);
  }
  else
  {
    s = 1;
    n = 50;
  }

  int i;
  for (i = s; i < n; ++i)
  {
    printf("This is %d line\n", i);
    fflush(stdout);
    if (i % 2 == 0) 
    {       
      srand(time(NULL));
      int r = rand() % 7;
      printf("sleep_type_2 with %d\n", r);
      Sleep(30 * r);
      fflush(stdout); 
    }     
      if (i % 3 == 0) 
      {     
        int r = rand() % 7;
        printf("sleep_type_3 with %d\n", r);
        Sleep(40 * r);
        fflush(stdout); 
      }
      if (i % 5 == 0) 
      {
        int r = rand() % 9;
        printf("sleep_type_5 with %d\n", r);
        Sleep(50);
        fflush(stdout); 
      }   
    }
}

Answer 1

In your case, you're not actually killing the threads when you use join(timeout=1). Instead, the join operation is waiting for one second to see if the thread exits, and when it doesn't, it just gives up. The join won't work because your thread is most likely blocking on this line:

chunk = self.fd.readline()

So it's never looping around to see if killed has been set to True. I think the best way for you to cleanly shutdown the threads is to terminate the subprocess being run in ReadThread in OnCloseFrame:

if self.read_thread_1:
    self.read_thread_1.subp.terminate()
    self.read_thread_1.join()
if self.read_thread_2:
    self.read_thread_2.subp.terminate()
    self.read_thread_2.join()

Terminating the subprocess will unblock the readline call, and allow you to cleanly terminate the thread.