How can i insert a massive amount of data From a DataBase into a TableWidget in a optimized way?

Question

I have a PushButton that when clicked, it displays the data of the DataBase to the TableWidget by runing the funcion add_values, and it works fine if you have a few thousands of rows, But let's say you have milions of rows, it's very ineficiente because I am using the method feachall() appending the DataBase's data into a varible and using a for loop to insert the data into the TableWidget.

here is my code.

DataBase = sqlite3.connect("DataBase.db",check_same_thread=False)
cursor = DataBase.cursor()

class TableWidget(QTableWidget):
    def __init__(self):
        super().__init__(1, 5)

        self.setHorizontalHeaderLabels(list('ABCDE'))


class AppDemo(QWidget):
    def __init__(self):
        super().__init__()
        self.resize(600, 600)

        self.mainLayout = QVBoxLayout()
        self.table = TableWidget()
        self.button = QPushButton("add items")
        self.button.clicked.connect(self.add_values)
        self.mainLayout.addWidget(self.table)
        self.mainLayout.addWidget(self.button)
        self.setLayout(self.mainLayout)


    def add_values(self):
        
        cursor.execute("select * from Peaple")

        data = cursor.fetchall()

        self.table.setRowCount(len(data))
        self.table.setColumnCount(5)

        for i in range(0, len(data)):
            for j in range(0,5):
                self.table.setItem(i,j , QtWidgets.QTableWidgetItem(str(data[i][j])))



app = QApplication(sys.argv)
demo = AppDemo()
demo.show()
app.exit(app.exec_())

i looked at DB Browser SQLite to see how it handles this when you Browse Data, and the DB Browser SQLite only loads the data when you actually see it and not 'Pre-loads' everything in advence.

Illustrating what i mean

Is there a way to do something like this in pyqt or is there another solution?

notice, i can't use thread cause even so it's gonna take some time to append the DataBase's data into the varible.

Answer 1

you want this ?

    #get cursor on app start
        cursor.execute("select * from Peaple")

    def add_values(self):        
        
        needed = 2
        data = cursor.fetchmany(needed)

        column = 5
        self.table.setColumnCount(column)

        rowCount = self.table.rowCount()

        try:
            for i in range(0, needed):
                for j in range(0, column):
                    self.table.setItem(rowCount-1+i, j, QTableWidgetItem(str(data[i][j])))
                self.table.setRowCount(rowCount+i+1)

        except IndexError:
            print("=====================")
            print("IndexError Because There is No Data to Get From Table")
            print("=====================")