So I have some simple code that is using multithreading. It's working just fine however I'm noticing that the threads aren't being destroyed after returning the values since everytime the script runs, the Thread number in console goes up and the RAM being used also goes up after the script is done processing (which implies that something was left running after the script was done processing).
After researching this, this, this and this, I've noticed that my threads are probably aren't joining (?) since my script never prints "Threads Destroyed". Can anyone suggest what could be going wrong?
if __name__ == "__main__":
def run_selenium1(a, b, c, d, e):
@st.cache_data(show_spinner=False)
def get_links(i, resumeContent):
#stufff happens
for something1, something2, something3, something4, something5, something6, something7 in zip(Final_Something1, Final_Something2, Final_Something3, Final_Something4, Final_Something5, Final_Something6, Final_Something7):
Final_Array.append((something1, something2, something3, something4, something5, something6, something7))
driver.close()
driver.quit()
except:
driver.close()
driver.quit()
with webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) as driver:
try:
#links are obtained
except:
driver.close()
driver.quit()
threads = []
for i in links:
t = threading.Thread(target=get_links, args=(i, Content))
t.daemon = True
threads.append(t)
t.start()
for t in threads:
t.join()
print("Threads destroyed") #<---- this isn't printed
EDit: After Eureka's answer I get this:
Starting thread 0
Starting thread 1
Starting thread 2
Starting thread 3
Starting thread 4
Starting thread 5
Starting thread 6
Starting thread 7
Starting thread 8
Starting thread 9
Starting thread 10
Starting thread 11
Starting thread 12
Starting thread 13
Starting thread 14
Starting thread 15
Starting thread 16
Starting thread 17
Starting thread 18
Starting thread 19
Starting thread 20
Starting thread 21
Starting thread 22
Starting thread 23
Starting thread 24
Total number of threads was 25
Trying to Join thread # 0
Joined thread # 0
Trying to Join thread # 1
Joined thread # 1
Trying to Join thread # 2
Joined thread # 2
Trying to Join thread # 3
Joined thread # 3
Trying to Join thread # 4
Joined thread # 4
Trying to Join thread # 5
Joined thread # 5
Trying to Join thread # 6
Joined thread # 6
Trying to Join thread # 7
Joined thread # 7
Trying to Join thread # 8
Joined thread # 8
Trying to Join thread # 9
Joined thread # 9
Trying to Join thread # 10
Joined thread # 10
Trying to Join thread # 11
Joined thread # 11
Trying to Join thread # 12
Joined thread # 12
Trying to Join thread # 13
Joined thread # 13
Trying to Join thread # 14
Joined thread # 14
Trying to Join thread # 15
Joined thread # 15
Trying to Join thread # 16
Joined thread # 16
Trying to Join thread # 17
Joined thread # 17
Trying to Join thread # 18
Joined thread # 18
Trying to Join thread # 19
Joined thread # 19
Trying to Join thread # 20
Joined thread # 20
Trying to Join thread # 21
Joined thread # 21
Trying to Join thread # 22
Joined thread # 22
Trying to Join thread # 23
Joined thread # 23
Trying to Join thread # 24
Joined thread # 24
All threads have now been joined
To test this, try adding notifications that the threads are finishing:
if __name__ == "__main__":
def run_selenium1(a, b, c, d, e):
@st.cache_data(show_spinner=False)
def get_links(iterator, i, resumeContent):
#stufff happens
for something1, something2, something3, something4, something5, something6, something7 in zip(Final_Something1, Final_Something2, Final_Something3, Final_Something4, Final_Something5, Final_Something6, Final_Something7):
Final_Array.append((something1, something2, something3, something4, something5, something6, something7))
print("About to close ",iterator)
driver.close()
driver.quit()
print("Closed and quit ",iterator)
except:
print("Error on ",iterator)
driver.close()
driver.quit()
print("Error closed and quit ",iterator)
with webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) as driver:
try:
#links are obtained
except:
driver.close()
driver.quit()
threads = []
for iterator,i in enumerate(links):
t = threading.Thread(target=get_links, args=(iterator,i, Content))
t.daemon = True
threads.append(t)
print("Starting thread",iterator)
t.start()
print("Total number of threads was ",len(threads)
for i,t in enumerate(threads):
print("Trying to Join thread #",i)
t.join()
print("Joined thread #",i)
print("All threads have now been joined")
threads = []
t = None