I am writing a program to download a bunch of files, the program is very long, I download the files by calling aria2c.exe via subprocess, and I have encountered a problem.
Specifically, when using aria2c + subprocess + QThread to download the file in the background, the GUI hangs and the progressbar and related labels don't update while the download is running, the GUI remains unresponsive until the download is complete.
I used the same method to download the files in console without the GUI using aria2c + subprocess + threading.Thread, the download completed successfully and all stats are updated correctly, and the threads complete without errors.
This is the minimal code required to reproduce the issue, though it is rather long:
import re
import requests
import subprocess
import sys
import time
from PyQt6.QtCore import Qt, QThread, pyqtSignal, pyqtSlot
from PyQt6.QtGui import (
QFont,
QFontMetrics,
)
from PyQt6.QtWidgets import (
QApplication,
QGridLayout,
QGroupBox,
QHBoxLayout,
QLabel,
QProgressBar,
QPushButton,
QSizePolicy,
QVBoxLayout,
QWidget,
)
BASE_COMMAND = [
"aria2c",
"--async-dns=false",
"--connect-timeout=3",
"--disk-cache=256M",
"--disable-ipv6=true",
"--enable-mmap=true",
"--http-no-cache=true",
"--max-connection-per-server=16",
"--min-split-size=1M",
"--piece-length=1M",
"--split=32",
"--timeout=3",
]
url = "http://ipv4.download.thinkbroadband.com/100MB.zip"
UNITS_SIZE = {"B": 1, "KiB": 1 << 10, "MiB": 1 << 20, "GiB": 1 << 30}
DOWNLOAD_PROGRESS = re.compile(
"(?P<downloaded>\d+(\.\d+)?[KMG]iB)/(?P<total>\d+(\.\d+)?[KMG]iB)"
)
UNITS = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB", "RiB", "QiB")
ALIGNMENT = Qt.AlignmentFlag.AlignLeft | Qt.AlignmentFlag.AlignTop
class Font(QFont):
def __init__(self, size: int = 10) -> None:
super().__init__()
self.setFamily("Times New Roman")
self.setStyleHint(QFont.StyleHint.Times)
self.setStyleStrategy(QFont.StyleStrategy.PreferAntialias)
self.setPointSize(size)
self.setBold(True)
self.setHintingPreference(QFont.HintingPreference.PreferFullHinting)
FONT = Font()
FONT_RULER = QFontMetrics(FONT)
class Box(QGroupBox):
def __init__(self) -> None:
super().__init__()
self.setAlignment(ALIGNMENT)
self.setContentsMargins(3, 3, 3, 3)
self.setSizePolicy(QSizePolicy.Policy.Minimum, QSizePolicy.Policy.Minimum)
self.vbox = make_vbox(self)
class Button(QPushButton):
def __init__(self, text: str) -> None:
super().__init__()
self.setFont(FONT)
self.setFixedSize(72, 20)
self.setText(text)
def make_box(
box_type: type[QHBoxLayout] | type[QVBoxLayout] | type[QGridLayout],
parent: QWidget,
margin: int,
) -> QHBoxLayout | QVBoxLayout | QGridLayout:
box = box_type(parent) if parent else box_type()
box.setAlignment(ALIGNMENT)
box.setContentsMargins(*[margin] * 4)
return box
def make_vbox(parent: QWidget = None, margin: int = 0) -> QVBoxLayout:
return make_box(QVBoxLayout, parent, margin)
def make_hbox(parent: QWidget = None, margin: int = 0) -> QHBoxLayout:
return make_box(QHBoxLayout, parent, margin)
class Label(QLabel):
def __init__(self, text: str) -> None:
super().__init__()
self.setFont(FONT)
self.set_text(text)
def autoResize(self) -> None:
self.Height = FONT_RULER.size(0, self.text()).height()
self.Width = FONT_RULER.size(0, self.text()).width()
self.setFixedSize(self.Width + 3, self.Height + 8)
def set_text(self, text: str) -> None:
self.setText(text)
self.autoResize()
class ProgressBar(QProgressBar):
def __init__(self) -> None:
super().__init__()
self.setFont(FONT)
self.setValue(0)
self.setFixedSize(1000, 25)
class DownThread(QThread):
update = pyqtSignal(dict)
def __init__(self, parent: QWidget, url: str, folder: str) -> None:
super().__init__(parent)
self.url = url
self.folder = folder
self.line = ""
self.stats = {}
def run(self) -> None:
self.total = 0
res = requests.head(url)
if res.status_code == 200 and (total := res.headers.get("Content-Length")):
self.total = int(total)
self.process = subprocess.Popen(
BASE_COMMAND + [f"--dir={self.folder}", self.url],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
self.monitor()
self.quit()
def monitor(self) -> None:
self.start = self.elapsed = time.time_ns()
self.downloaded = 0
output = self.process.stdout
self.buffer = ""
while self.process.poll() is None:
char = output.read(1)
if char in (b"\n", b"\r"):
self.line = self.buffer
self.buffer = ""
self.update_stats()
else:
self.buffer += char.decode()
self.finish()
def update_stats(self) -> None:
if match := DOWNLOAD_PROGRESS.search(self.line):
current = time.time_ns()
new, total = map(self.parse_size, match.groupdict().values())
delta = (current - self.elapsed) / 1e9
speed = (new - self.downloaded) / delta
self.stats = {
"downloaded": new,
"total": total,
"speed": speed,
"elapsed": (current - self.start) / 1e9,
"eta": ((total - new) / speed) if speed != 0 else 1e309,
}
self.elapsed = current
self.downloaded = new
self.update.emit(self.stats)
@staticmethod
def parse_size(size: str) -> int:
unit = size[-3:]
size = size.replace(unit, "")
return (float if "." in size else int)(size) * UNITS_SIZE[unit]
def finish(self):
self.elapsed = (time.time_ns() - self.start) // 1e9
total = self.total or self.stats["total"]
self.stats["downloaded"] = total
self.stats["total"] = total
self.stats["elapsed"] = self.elapsed
self.stats["eta"] = 0
self.stats["speed"] = total / self.elapsed
self.update.emit(self.stats)
class Underbar(Box):
def __init__(self):
super().__init__()
self.setFixedHeight(256)
self.progressbar = ProgressBar()
self.hbox = make_hbox()
self.hbox.addWidget(self.progressbar)
self.displays = {}
for name in ("Downloaded", "Total", "Speed", "Elapsed", "ETA"):
self.hbox.addWidget(Label(name))
widget = Label("0")
self.hbox.addWidget(widget)
self.displays[name] = widget
self.vbox.addLayout(self.hbox)
self.button = Button("Test")
self.vbox.addWidget(self.button)
self.button.clicked.connect(self.test)
def test(self):
self.progressbar.setValue(0)
down = DownThread(self, url, "D:/downloads")
down.update.connect(self.update_displays)
down.run()
def update_displays(self, stats):
self.progressbar.setValue(100 * int(stats["downloaded"] / stats["total"] + 0.5))
for name, suffix in (("Downloaded", ""), ("Total", ""), ("Speed", "/s")):
self.displays[name].setText(
f"{round(stats[name.lower()] / 1048576, 2)}MiB{suffix}"
)
self.displays["Elapsed"].setText(f'{round(stats["elapsed"], 2)}s')
self.displays["ETA"].setText(f'{round(stats["eta"], 2)}s')
for label in self.displays.values():
label.autoResize()
if __name__ == "__main__":
app = QApplication([])
app.setStyle("Fusion")
window = Underbar()
window.show()
sys.exit(app.exec())
How to fix this?
I have solved the problem. I have discarded the idea of using subprocess to call aria2c, instead I use aiohttp + aiofiles to download the file in multiple parts using HTTP Range header.
And I then use QEventLoop from qasync to make asyncio work with QThread, and it works without problems.
import aiofiles
import asyncio
import sys
from aiohttp import ClientSession
from bisect import bisect
from PyQt6.QtCore import Qt, QThread, pyqtSignal, pyqtSlot
from PyQt6.QtGui import (
QFont,
QFontMetrics,
)
from PyQt6.QtWidgets import (
QApplication,
QGridLayout,
QGroupBox,
QHBoxLayout,
QLabel,
QProgressBar,
QPushButton,
QSizePolicy,
QVBoxLayout,
QWidget,
)
from qasync import QEventLoop
from tqdm import tqdm
CHUNK = 524288
url = "http://ipv4.download.thinkbroadband.com/100MB.zip"
UNITS = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB", "RiB", "QiB")
UNIT_SIZES = [1 << (i * 10) for i in range(11)]
ALIGNMENT = Qt.AlignmentFlag.AlignLeft | Qt.AlignmentFlag.AlignTop
def to_time(n: float) -> str:
n = int(n + 0.5)
segments = []
for _ in (0, 1):
n, d = divmod(n, 60)
segments.insert(0, d)
if n:
segments.insert(0, n)
return ":".join(map(str, segments))
def format_size(size: int) -> str:
i = bisect(UNIT_SIZES, size) - 1
return f"{round(size/UNIT_SIZES[i], 3)}{UNITS[i]}"
class Downloader(QThread):
update = pyqtSignal()
refresh = pyqtSignal()
def __init__(self, url: str, filepath: str, links: int = 16) -> None:
super().__init__()
self.url = url
self.filepath = filepath
self.links = links
self.update.connect(self.update_values)
async def preprocess(self, session: ClientSession) -> None:
resp = await session.head(self.url)
self.total = int(resp.headers["Content-Length"])
self.progress = tqdm(
total=self.total, unit_scale=True, unit_divisor=1024, unit="B"
)
async def start_download(self) -> None:
async with ClientSession(
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"
}
) as session:
await self.preprocess(session)
self.chunk = self.total // self.links
ends = range(0, self.total, self.chunk)[: self.links]
self.ranges = [
*((start, end - 1) for start, end in zip(ends, ends[1:])),
(ends[-1], self.total - 1),
]
await asyncio.gather(
*(self.download_worker(i, session) for i in range(self.links))
)
async def download_worker(self, index: int, session: ClientSession) -> None:
async with aiofiles.open(self.filepath, "wb") as file:
start, end = self.ranges[index]
await file.seek(start)
async with session.get(
url=self.url,
headers={"Range": f"bytes={start}-{end}"},
) as resp:
async for chunk in resp.content.iter_chunked(CHUNK):
self.progress.update(len(chunk))
self.update.emit()
await file.write(chunk)
def update_values(self) -> None:
d = self.progress.format_dict
self.stats = {
"Downloaded": d["n"],
"Total": d["total"],
"Elapsed": d["elapsed"],
"Speed": d["rate"],
"ETA": (d["total"] - d["n"]) / d["rate"],
}
self.refresh.emit()
def run(self) -> None:
loop = QEventLoop(self)
asyncio.set_event_loop(loop)
loop.run_until_complete(self.start_download())
class Font(QFont):
def __init__(self, size: int = 10) -> None:
super().__init__()
self.setFamily("Times New Roman")
self.setStyleHint(QFont.StyleHint.Times)
self.setStyleStrategy(QFont.StyleStrategy.PreferAntialias)
self.setPointSize(size)
self.setBold(True)
self.setHintingPreference(QFont.HintingPreference.PreferFullHinting)
FONT = Font()
FONT_RULER = QFontMetrics(FONT)
class Box(QGroupBox):
def __init__(self) -> None:
super().__init__()
self.setAlignment(ALIGNMENT)
self.setContentsMargins(3, 3, 3, 3)
self.setSizePolicy(QSizePolicy.Policy.Minimum, QSizePolicy.Policy.Minimum)
self.vbox = make_vbox(self)
class Button(QPushButton):
def __init__(self, text: str) -> None:
super().__init__()
self.setFont(FONT)
self.setFixedSize(72, 20)
self.setText(text)
def make_box(
box_type: type[QHBoxLayout] | type[QVBoxLayout] | type[QGridLayout],
parent: QWidget,
margin: int,
) -> QHBoxLayout | QVBoxLayout | QGridLayout:
box = box_type(parent) if parent else box_type()
box.setAlignment(ALIGNMENT)
box.setContentsMargins(*[margin] * 4)
return box
def make_vbox(parent: QWidget = None, margin: int = 0) -> QVBoxLayout:
return make_box(QVBoxLayout, parent, margin)
def make_hbox(parent: QWidget = None, margin: int = 0) -> QHBoxLayout:
return make_box(QHBoxLayout, parent, margin)
class Label(QLabel):
def __init__(self, text: str) -> None:
super().__init__()
self.setFont(FONT)
self.set_text(text)
def autoResize(self) -> None:
self.Height = FONT_RULER.size(0, self.text()).height()
self.Width = FONT_RULER.size(0, self.text()).width()
self.setFixedSize(self.Width + 3, self.Height + 8)
def set_text(self, text: str) -> None:
self.setText(text)
self.autoResize()
class ProgressBar(QProgressBar):
def __init__(self) -> None:
super().__init__()
self.setFont(FONT)
self.setValue(0)
self.setFixedSize(1000, 25)
class Underbar(Box):
def __init__(self):
super().__init__()
self.setFixedHeight(256)
self.progressbar = ProgressBar()
self.hbox = make_hbox()
self.hbox.addWidget(self.progressbar)
self.displays = {}
for name in ("Downloaded", "Total", "Speed", "Elapsed", "ETA"):
self.hbox.addWidget(Label(name))
widget = Label("0")
self.hbox.addWidget(widget)
self.displays[name] = widget
self.vbox.addLayout(self.hbox)
self.button = Button("Test")
self.vbox.addWidget(self.button)
self.button.clicked.connect(self.test)
def test(self):
self.progressbar.setValue(0)
self.down = Downloader(url, "D:/speedtest/100MB.zip")
self.down.refresh.connect(self.update_values)
self.down.run()
def update_values(self):
stats = self.down.stats
self.displays["Downloaded"].setText(format_size(stats["Downloaded"]))
self.displays["Total"].setText(format_size(stats["Total"]))
self.displays["Speed"].setText(format_size(stats["Speed"]) + "/s")
self.displays["Elapsed"].setText(to_time(stats["Elapsed"]))
self.displays["ETA"].setText(to_time(stats["ETA"]))
self.progressbar.setValue(int(stats["Downloaded"] / stats["Total"] * 100 + 0.5))
for label in self.displays.values():
label.autoResize()
if __name__ == "__main__":
app = QApplication([])
app.setStyle("Fusion")
window = Underbar()
window.show()
sys.exit(app.exec())