Handling thousands of items in a QListWidget and reducing lag

Question

I have QListWidget and a dictionary which I am looping through using a for loop. Each iteration of the for loop adds an item to the QListWidget along with a few buttons and labels attached to each item. Everything is working fine, but I have the problem of the list taking a long time (about 20 seconds for 1k items) to load every time I refresh the list. During this time the GUI is completely irresponsive (which I'm fine with as long as it isn't taking too long). One solution that I found was that the refresh time was dramatically decreased if I hid (self.hide()) the QMainWindow while it is doing its iterations, then showing (self.show()) it after it was completed (about 1.5 seconds for 1k items), so I'm assuming that it is a problem with the resources. Would it be possible to get refresh times of around the 1.5 seconds while still keeping the GUI visible (and irresponsive) by like freezing the GUI so it doesn't use up as many resources while the list is being refreshed.

Example code:

import sys
import time
from PyQt5.QtWidgets import QApplication, QMainWindow,  QListWidget, QListWidgetItem, QPushButton, QVBoxLayout, QWidget


class window(QMainWindow):
    def __init__(self):
        super(window, self).__init__()
        self.show()
        self.setFixedSize(800, 500)
        self.listwidget()
        self.refreshlist()

    def listwidget(self):
        self.list = QListWidget(self)
        self.list.setFixedSize(800, 500)
        self.list.show()

    def refreshlist(self): # uncomment self.hide() and self.show() to see how much faster it is
        start = time.time()
        # self.hide()
        for i in range(1000):
            item = QListWidgetItem(str(i))
            self.list.addItem(item)
            widget = QWidget(self.list)
            layout = QVBoxLayout(widget)
            layout.addWidget(QPushButton())
            self.list.setItemWidget(item, widget)
        # self.show()
        print(f"took {time.time() - start} seconds")
        """
        average time with hiding and showing was 0.3 seconds
        average time without hiding and showing was 14 seconds
        """

if __name__ == '__main__':
    app = QApplication([])
    Gui = window()
    sys.exit(app.exec_())

Answer 1

The initial problem is that when it is showing then every time you add an item (and widget) it repaints everything again unlike the task of hiding it and showing it where there is only one painting.

An alternative that does not reduce the loading time but does make the GUI visible is to add blocks of items every T seconds using a queue and a timer. You could also add a gif that indicates to the user that information is being loaded.

from collections import deque
from functools import cached_property
import sys


from PyQt5.QtCore import pyqtSignal, QTimer
from PyQt5.QtGui import QMovie
from PyQt5.QtWidgets import (
    QApplication,
    QMainWindow,
    QLabel,
    QListWidget,
    QListWidgetItem,
    QPushButton,
    QStackedWidget,
    QVBoxLayout,
    QWidget,
)


class ListWidget(QListWidget):
    started = pyqtSignal()
    finished = pyqtSignal()

    CHUNK = 50
    INTERVAL = 0

    def __init__(self, parent=None):
        super().__init__(parent)

        self.timer.timeout.connect(self.handle_timeout)

    @cached_property
    def queue(self):
        return deque()

    @cached_property
    def timer(self):
        return QTimer(interval=self.INTERVAL)

    def fillData(self, data):
        self.started.emit()
        self.queue.clear()
        self.queue.extend(data)
        self.timer.start()

    def handle_timeout(self):
        for i in range(self.CHUNK):
            if self.queue:
                value = self.queue.popleft()
                self.create_item(str(value))
            else:
                self.timer.stop()
                self.finished.emit()
                break

    def create_item(self, text):
        item = QListWidgetItem(text)
        self.addItem(item)
        widget = QWidget()
        layout = QVBoxLayout(widget)
        button = QPushButton(text)
        layout.addWidget(button)
        layout.setContentsMargins(0, 0, 0, 0)
        self.setItemWidget(item, widget)


class Window(QMainWindow):
    def __init__(self):
        super(Window, self).__init__()
        self.setFixedSize(800, 500)
        self.setCentralWidget(self.stackedWidget)
        self.stackedWidget.addWidget(self.gifLabel)
        self.stackedWidget.addWidget(self.listWidget)

        self.listWidget.started.connect(self.handle_listwidget_started)
        self.listWidget.finished.connect(self.handle_listwidget_finished)

    @cached_property
    def stackedWidget(self):
        return QStackedWidget()

    @cached_property
    def listWidget(self):
        return ListWidget()

    @cached_property
    def gifLabel(self):
        label = QLabel(scaledContents=True)
        movie = QMovie("loading.gif")
        label.setMovie(movie)
        return label

    def handle_listwidget_started(self):
        self.gifLabel.movie().start()
        self.stackedWidget.setCurrentIndex(0)

    def handle_listwidget_finished(self):
        self.gifLabel.movie().stop()
        self.stackedWidget.setCurrentIndex(1)


if __name__ == "__main__":
    app = QApplication([])

    w = Window()
    w.show()

    w.listWidget.fillData(range(1000))

    sys.exit(app.exec_())