I'm trying to download files using the Request module using Multi threading and simultaneously need to Update a Progress bar as well. But while doing so, whenever I press submit button, my GUI stops working and in the background the files start to download and only after Downloading, my Progress bar updates and GUI continues to work Normal.
I have tried to apply multi-threading to the GUI as well so that the progress of downloaded files will be displayed, but it is only updating the progress after the whole Process gets over.
from datetime import datetime
from threading import Thread
import dask.dataframe as dd
from pubsub import pub
from queue import Queue
import pandas as pd
import requests
import shutil
import sqlite3
import time
import os
import csv
import wx
class MainPage(wx.Dialog):
def __init__(self):
wx.Dialog.__init__(self, None, title="File Selector")
user_sizer = wx.BoxSizer(wx.HORIZONTAL)
user_label = wx.StaticText(self, label='Number of Threads:')
user_sizer.Add(user_label, 0, wx.ALL | wx.CENTER, 5)
self.threads = wx.TextCtrl(self)
user_sizer.Add(self.threads, 5, wx.ALL, 5)
button_sizer = wx.BoxSizer(wx.HORIZONTAL)
file_label = wx.StaticText(self, label='Select File:')
button_sizer.Add(file_label, 0, wx.ALL | wx.CENTER, 5)
self.select = wx.TextCtrl(self)
button_sizer.Add(self.select, 0, wx.ALL | wx.CENTER, 10)
file_select = wx.Button(self, label='Browse')
file_select.Bind(wx.EVT_BUTTON, self.file)
button_sizer.Add(file_select, 0, wx.ALL | wx.CENTER, 5)
link_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
link_choice_label = wx.StaticText(self, label='Link Column:')
link_choice_sizer.Add(link_choice_label, 0, wx.ALL | wx.CENTER, 5)
self.link_choice = wx.TextCtrl(self)
link_choice_sizer.Add(self.link_choice, 0, wx.ALL | wx.CENTER, 5)
file_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
file_choice_label = wx.StaticText(self, label='File column:')
file_choice_sizer.Add(file_choice_label, 0, wx.ALL | wx.CENTER, 5)
self.file_choice = wx.TextCtrl(self)
file_choice_sizer.Add(self.file_choice, 0, wx.ALL | wx.CENTER, 5)
submit_sizer = wx.BoxSizer(wx.HORIZONTAL)
submit_button = wx.Button(self, label='Submit')
submit_button.Bind(wx.EVT_BUTTON, self.process)
submit_sizer.Add(submit_button, 0, wx.TOP, 20)
main_box = wx.BoxSizer(wx.VERTICAL)
main_box.Add(user_sizer, 0, wx.ALL, 3)
main_box.Add(button_sizer, 0, wx.LEFT, 43)
main_box.Add(link_choice_sizer, 0, wx.LEFT, 39)
main_box.Add(file_choice_sizer, 0, wx.LEFT, 43)
main_box.Add(submit_sizer, 0, wx.LEFT, 135)
self.user_sizer = user_sizer
self.button_sizer = button_sizer
self.SetSizer(main_box)
self.Center()
def file(self, event):
dialog = wx.FileDialog(None, "Choose a File:", wildcard="CSV files (*.csv)|*.csv", style=wx.DD_DEFAULT_STYLE | wx.DD_NEW_DIR_BUTTON)
if dialog.ShowModal() == wx.ID_OK:
print(dialog.GetPath())
path = dialog.GetPath()
self.select.ChangeValue(path)
print(path)
dialog.Destroy()
def process(self, event):
selected_path = self.select.GetValue()
thread = self.threads.GetValue()
link = self.link_choice.GetValue()
file = self.file_choice.GetValue()
print(thread)
print(link)
print(file)
if (thread != ""):
if (selected_path != "") :
if (link != "") :
if (file != "") :
if (thread.isdigit()) and (link.isdigit()) and (file.isdigit()) :
if (int(link) != int(file)):
if selected_path.endswith('.csv'):
self.threads.Clear()
self.select.Clear()
self.link_choice.Clear()
self.file_choice.Clear()
print("out")
print(file)
wx.LogMessage("File processed Sucessfully")
# self.csv_processing(selected_path, thread, link, file)
self.progress_show(selected_path, thread, link, file)
else:
wx.LogError("File Extension Not Valid")
else:
wx.LogError("Both Link Column and File Column Can't be Same")
else:
wx.LogError("Please Enter Valid Number")
else:
wx.LogError("Please Enter Number in File Column")
else:
wx.LogError("Please Enter Number in Link Column")
else:
wx.LogError("Please Select File")
else:
wx.LogError("Please Enter Number in Thread")
def progress_show(self, selected_path, thread, link, file):
pub.sendMessage('frameListener', message='show', selected_path = selected_path, thread = thread, link = link, file = file)
class Download_file():
def download_image(self, url, file_path, data_load):
print("In Class")
try:
wx.CallAfter(self.new_update, data_load)
response = requests.get(url)
if response.status_code == 200:
if (response.raw) != None:
with open(file_path, 'wb') as out_file:
out_file.write(response.content)
del response
else:
print("not found")
except requests.exceptions.RequestException as e:
print(e)
class DownloadWorker(Thread):
def __init__(self, queue):
Thread.__init__(self)
self.queue = queue
def run(self):
while True:
print("In queue")
url, file_path, data_load = self.queue.get()
download = Download_file
try:
# pub.sendMessage('progress', message='show', data_load=data_load)
# print((url))
download.download_image(self, url, file_path, data_load)
# wx.CallAfter(self.new_update, data_load)
time.sleep(1)
finally:
self.queue.task_done()
print("Downloaded")
def new_update(self, data_load):
print("upload")
print(data_load)
pub.sendMessage('progress', message='show', data_load=data_load)
class Progressbar(wx.Dialog):
def __init__(self, selected_path, thread, link, file):
wx.Dialog.__init__(self, None, title='Progess Bar', size=(300,100))
print("INnnn")
progress_sizer = wx.BoxSizer(wx.HORIZONTAL)
self.progress_bar = wx.Gauge(self, range=100, size=(250,25), style= wx.GA_HORIZONTAL)
progress_sizer.Add(self.progress_bar, 0, wx.CENTER)
progress_box = wx.BoxSizer(wx.VERTICAL)
progress_box.Add(progress_sizer, wx.ALIGN_CENTRE, 25)
print(selected_path, thread, link, file)
self.SetSizer(progress_box)
self.Center()
self.Show(True)
self.call_progress(selected_path, thread, link, file)
pub.subscribe(self.progressListener, 'progress')
self.Destroy()
def call_progress(self, selected_path, thread, link, file):
start = datetime.now()
count = 1
thread = int(thread)
dask_data = dd.read_csv(selected_path, sep=',', error_bad_lines=False, quoting=csv.QUOTE_NONE, dtype="unicode")
data_length = len(dask_data)
data = pd.read_csv(selected_path, delimiter=',', chunksize=thread)
count = 1
date = str(datetime.date(datetime.now()))
mydir = os.getcwd()
image_dir = os.path.join(mydir, "Downloaded_Images")
if not os.path.exists(image_dir):
os.mkdir(image_dir)
date_dir = os.path.join(image_dir, date)
if not os.path.exists(date_dir):
os.mkdir(date_dir)
link_name = 'Link_column' + '___' + str(link)
link_column = os.path.join(date_dir, link_name)
if not os.path.exists(link_column):
os.mkdir(link_column)
queue = Queue()
for x in range(thread):
worker = DownloadWorker(queue)
worker.daemon = True
worker.start()
thread_length = thread
for df in data:
data_load = round((thread_length / data_length) * 100)
if data_load > 100:
data_load = 100
for i in range(thread):
try:
print(df.iloc[i][int(link) - 1])
url = (df.iloc[i][int(link) - 1])
print(df.iloc[i][int(file) - 1])
file_name = (df.iloc[i][int(file) - 1])
print(count)
# print(data_load)
file_path = os.path.join(link_column, file_name)
queue.put((url, file_path, data_load))
count += 1
except IndexError:
pass
print("-------------------------")
queue.join()
thread_length += thread
# data_load = round((thread / data_length) * 100)
# if data_load > 100:
# data_load = 100
# self.progress_bar.SetValue(data_load)
# queue.join()
print((datetime.now()) - start)
def progressListener(self, message, data_load):
print("listener")
self.progress_bar.SetValue(data_load)
class MyPanel(wx.Panel):
def __init(self, parent):
wx.Panel.__init__(self, parent)
class MainFrame(wx.Frame):
def __init__(self):
wx.Frame.__init__(self, None)
MyPanel(self)
pub.subscribe(self.myListener, 'frameListener')
self.Center()
dlg = MainPage()
dlg.ShowModal()
def myListener(self, message, selected_path, thread, link, file):
print(selected_path, thread, link, file)
# self.center()
dlg = Progressbar(selected_path = selected_path, thread = thread, link = link, file = file)
dlg.ShowModal()
def myapp():
app = wx.App(False)
MainFrame()
# app.MainLoop()
if __name__ == '__main__':
myapp()
Expected result is that the File Should be Downloaded and at the same time the Progress Bar should be Updated as well.
With apologies to Mike for highjacking his code for a skeleton, here is a rough version using urlretrieve and a wxPython event to do the updating.
import os
import wx
import time
from threading import Thread
from urllib.request import FancyURLopener
from urllib.request import urlopen
import wx.lib.newevent
urlretrieve = FancyURLopener().retrieve
progress_event, EVT_PROGRESS_EVENT = wx.lib.newevent.NewEvent()
class DownloadThread(Thread):
def __init__(self, parent, url, fsize):
"""Constructor"""
Thread.__init__(self)
self.fsize = fsize
self.url = url
self.parent = parent
self.start()
def run(self):
local_fname = os.path.basename(self.url)
urlretrieve(self.url, local_fname, self.reporthook)
def reporthook(self,blocknum, blocksize, totalsize):
readsofar = blocknum * blocksize
if totalsize > 0:
percent = int((readsofar / totalsize) * 100)
else: # total size is unknown
percent = 0
evt = progress_event(count=percent)
#Send back current count for the progress bar
try:
wx.PostEvent(self.parent, evt)
except: # The parent frame has probably been destroyed
pass
class MyPanel(wx.Panel):
def __init__(self, parent):
"""Constructor"""
wx.Panel.__init__(self, parent)
self.lbl = wx.StaticText(self, label="Download URL:")
self.dl_txt = wx.TextCtrl(self)
btn = wx.Button(self, label="Download")
self.gauge = wx.Gauge(self, -1, range=100)
self.main_sizer = wx.BoxSizer(wx.VERTICAL)
dl_sizer = wx.BoxSizer(wx.HORIZONTAL)
dl_sizer.Add(self.dl_txt, 1, wx.EXPAND|wx.ALL, 5)
dl_sizer.Add(btn, 0, wx.ALL, 5)
self.main_sizer.Add(dl_sizer, 0, wx.EXPAND)
self.main_sizer.Add(self.lbl, 0, wx.ALL|wx.CENTER, 5)
self.main_sizer.Add(self.gauge, 0, wx.ALL|wx.EXPAND, 5)
self.SetSizer(self.main_sizer)
#Bind to the progress event issued by the thread
self.Bind(EVT_PROGRESS_EVENT, self.OnProgress)
btn.Bind(wx.EVT_BUTTON, self.OnDownload)
def OnDownload(self, event):
url = self.dl_txt.GetValue()
self.lbl.SetLabel(os.path.basename(url))
try:
header = urlopen(url)
fsize = int(header.headers['Content-Length'])
# start thread
d_thread = DownloadThread(self, url, fsize)
while d_thread.isAlive():
time.sleep(0.1)
wx.GetApp().Yield()
continue
# Finished re-set
self.dl_txt.SetValue("")
self.lbl.SetLabel("Download URL:")
self.gauge.SetValue(0)
except Exception as e:
print ("Error: ", e)
def OnProgress(self, event):
""""""
self.gauge.SetValue(event.count)
class DownloaderFrame(wx.Frame):
def __init__(self):
"""Constructor"""
wx.Frame.__init__(self, None, title="Downloader", size=(800, 400))
panel = MyPanel(self)
self.Show()
if __name__ == "__main__":
app = wx.App(False)
frame = DownloaderFrame()
app.MainLoop()