I am trying to put an animated graph made with matplotlib funcAnimation into a PySide6 widget. It takes data in from a serial port, plots it, and records it into a csv. My issue is that after a set period of time, it starts to slow down significantly. While data is coming in every second, after about 20 minutes or so, it begins to only update every 2 seconds.
I tried changing the interval time and while that does make it a last a little longer, ultimately, they all still end up slowing down undesirably. I implemented a code which checks for execution time and noticed that it gradually increases until it is past over a second. I then tried taking it out of the widget but the same result ensues. Some other ideas I had were using blitting or turning it into a list/dict instead of a csv but just wanted to get some input before implementing. I saw a post on reddit with the same issue and in which blitting didn't work and he uses a csv so I'm not sure about my previous ideas: [Reddit Post](https://www.reddit.com/r/AskProgramming/comments/owde4a/matplotlib_animation_exponnetially_slower_the/ In this post they got a few suggestions which seemed to work for them but only this one applied to my code: Why are ax.set_xlabel(), ax.grid, etc. in your animate() function for each of the axes? These should be set one time when you create the axes, not on every call to animate(). So I changed that and put it into setup, this meant that I had to delete my ax.cla() in the animate loop. Now, it doesnt just show the last 10 values anymore so I need to fix that too.
My code:
`
import sys
from PySide6 import QtGui, QtCore
from PySide6.QtGui import QScreen, QPixmap
from pathlib import Path
from PySide6.QtWidgets import QWidget, QApplication, QPushButton, QVBoxLayout, QMainWindow, QHBoxLayout, QLabel
from matplotlib.backends.backend_qtagg import (
FigureCanvas, NavigationToolbar2QT as NavigationToolbar)
from matplotlib.backends.backend_qtagg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
from PySide6.QtCore import Qt
from matplotlib.animation import FuncAnimation
import pandas as pd
import serial
import matplotlib.image as image
import csv
import subprocess
import time
import openpyxl
import cProfile
#Initialize Serial Port
ser = serial.Serial()
ser.baudrate = 28800
ser.port = 'COM3'
timeout = 1
parity = serial.PARITY_ODD
bytesize = serial.EIGHTBITS
ser.open()
#Image Widget
class Widget(QWidget):
def __init__(self):
super().__init__()
#Initialize value that will select one image if the value is reached and another if not
self.int4_value = 0
#Creating the pixmap
self.image_label = QLabel()
self.original_pixmap = QPixmap("C:/Users/mlee/Downloads/mid_green_background (1).png")
self.scaled_pixmap = self.original_pixmap.scaled(20, 20)
self.image_label.setPixmap(self.scaled_pixmap)
#Layout
v_layout = QVBoxLayout()
v_layout.addWidget(self.image_label, alignment=Qt.AlignTop)
self.setLayout(v_layout)
def update_int4_value(self, new_value):
#Updating value to change image
self.int4_value = new_value
if self.int4_value == 1365:
self.original_pixmap = QPixmap("C:/Users/mlee/Downloads/mid_green_background (1).png")
else:
self.original_pixmap = QPixmap("C:/Users/mlee/Downloads/solid_red_background (1).jpg")
self.scaled_pixmap = self.original_pixmap.scaled(20, 20)
self.image_label.setPixmap(self.scaled_pixmap)
class Window(QMainWindow):
def __init__(self):
#Counts the number of screenshots
self.screenshot_counter = self.load_screenshot_counter()
super().__init__()
#Import widget
self.widget1 = Widget()
self.app = app
self.setWindowTitle("Custom")
#Add menu bar
menu_bar = self.menuBar()
file_menu = menu_bar.addMenu("&File")
quit_action = file_menu.addAction("Quit")
quit_action.triggered.connect(self.quit)
save_menu = menu_bar.addMenu("&Save")
Screenshot = save_menu.addAction("Screenshot")
Screenshot.triggered.connect(self.screenshot)
#Set up graph as widget
self._main = QWidget()
self.setCentralWidget(self._main)
layout = QHBoxLayout(self._main)
self.fig = Figure(figsize=(5, 3))
self.canvas = FigureCanvas(self.fig)
layout.addWidget(self.canvas, stretch=24)
layout.addWidget(self.widget1, stretch=1)
#Set up toolbar
self.addToolBar(NavigationToolbar(self.canvas, self))
#Creating csv file to store incoming data and adding headers
with open('csv_graph.csv', 'w', newline='') as csvfile:
fieldnames = ['Value1', 'Value2', 'Value3', 'Value4', 'Value5', 'Value6']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
#Call the loop
self.setup()
def setup(self):
#At 500ms interval, failed after 1400 frames
#At 250 ms interval, failed after 1600 frames
#At 1ms interval, failed after 1800 frames
#Create the subplot
self.ax = self.fig.add_subplot(111)
# Legend and labels
self.ax.legend(loc='upper left')
self.ax.set_xlabel('Time(seconds)')
self.ax.set_ylabel('Value')
self.ax.set_title('Random Data')
#Animation function
self.ani = FuncAnimation(self.canvas.figure, self.animate, interval=1, cache_frame_data=False)
def animate(self, i):
#Timer to check execution time
start_time = time.time()
#Reading serial port data and only going htrough the loop if length is larger than 2
# since the first input is normally all 0
bytestoread = ser.inWaiting()
new_value = ser.read(bytestoread)
if len(new_value) >= 2:
#Turning all the bytes into ints
byte1 = new_value[2:4] # Extract the first byte
int1 = int.from_bytes(byte1, byteorder='little')
print(int1)
byte2 = new_value[4:6] # Extract the second byte
int2 = int.from_bytes(byte2, byteorder='little')
print(int2)
byte3 = new_value[6:8] # Extract the third byte
int3 = int.from_bytes(byte3, byteorder='little')
print(int3)
byte4 = new_value[8:10]
int4 = int.from_bytes(byte4, byteorder='little')
print(int4)
#Pass int4 to the image widget
self.widget1.update_int4_value(int4)
byte5 = new_value[10:12] # Day & Hour
int5 = int.from_bytes(byte5, byteorder='little')
# print(int5)
byte6 = new_value[12:14] # Minutes & Seconds
int6 = int.from_bytes(byte6, byteorder='little')
print(int6)
#Write the data into the csv
with open('csv_graph.csv', 'a', newline='') as csvfile:
csv_writer = csv.writer(csvfile)
csv_writer.writerow([int1, int2, int3, int4, int5, int6])
#Read from that csv and then take the last 10 rows
data = pd.read_csv('csv_graph.csv')
last_x_rows = data.tail(10)
#Assigning the values to variables
x = last_x_rows['Value6']
y1 = last_x_rows['Value1']
y2 = last_x_rows['Value2']
y3 = last_x_rows['Value3']
y4 = last_x_rows['Value4']
# Plotting
# self.ax.cla()
self.ax.plot(x, y1, color='Red', label='Red')
self.ax.plot(x, y2, color='Blue', label='Blue')
self.ax.plot(x, y3, color='Purple', label='Purple')
self.ax.plot(x, y4, color='Green', label='Green')
#Opening the csv in an excel
with pd.ExcelWriter("C:/Users/mlee/Documents/Excel_CSV/New_Excel.xlsx", mode='a', engine='openpyxl',
if_sheet_exists='replace') as writer:
data.to_excel(writer, sheet_name='Sheet_1')
#Execution time check
end_time = time.time()
self.execution_time = end_time - start_time
print(f"Frame {i}: Execution Time = {self.execution_time:.2f} seconds")
#Was trying to use blitting but didn't do it right
return self.ax
#Functions for menu bar
def quit(self):
self.app.quit()
def load_screenshot_counter(self):
counter_file = Path("screenshot_counter.txt")
if counter_file.exists():
with counter_file.open("r") as file:
return int(file.read())
else:
return 1
def save_screenshot_counter(self):
counter_file = Path("C:/Users/mlee/Downloads/screenshot_counter.txt")
with counter_file.open("w") as file:
file.write(str(self.screenshot_counter))
def screenshot(self):
# Get the primary screen
primary_screen = QApplication.primaryScreen()
# Get the available size of the screen (excluding reserved areas)
available_size = primary_screen.availableSize()
print(available_size)
shot = QScreen.grabWindow(QApplication.primaryScreen(), 0, 90, 95, 1410, 700)
file_path = f"C:/Users/mlee/Downloads/Screenshot_{self.screenshot_counter}.png"
shot.save(file_path, "PNG")
self.screenshot_counter += 1
self.save_screenshot_counter()
app = QApplication(sys.argv)
widget = Window()
widget.show()
with cProfile.Profile() as pr:
sys.exit(app.exec())
pr.print_stats(sort='cumtime')
`
Well firstly, you are reading the .csv file inside of your animation function which is not so good because the file is getting bigger each time it stores the data inside of it and therefore when it reads the data it takes a lot of time, and re-plotting all the data each frame can be inefficient. Instead, updating the existing plot data is typically more efficient. First solve the issue of reading data from .csv you can store the data more efficiently too with the "collections" module called deque. Deque has fast appends so when you're continuously adding new data points and need to maintain only a fixed number of the most recent ones, a deque with a set maxlen automatically discards the oldest items when the new ones are added. This behaevior is ideal for your case.
from collections import deque
then you can make the buffer for the last 10 data points from deque
self.data_buffer = deque(maxlen=10) # Buffer to store the last 10 data points
self.csv_data = [] # Make a list for storing the data
and if you are going to write the information inside of the .csv then I would suggest that you write the data in batches instead on every frame.Make a variable something like this.
self.csv_batch_size = 50 # Adjust this accordingly
then in your animate function you can make an append so you store the data
new_data = [int1, int2, int3, int4, int5, int6]
self.data_buffer.append(new_data)
self.csv_data.append(new_data)
then you can make an if statement for checking the size
if len(self.csv_data) >= self.csv_batch_size:
self.write_to_csv()
self.csv_data = []
then you update the plot
df = pd.DataFrame(list(self.data_buffer), columns=['Value1', 'Value2', 'Value3', 'Value4', 'Value5', 'Value6'])
x = range(len(df))
self.ax.clear()
self.ax.plot(x, df['Value1'], color='Red')
self.ax.plot(x, df['Value2'], color='Blue')
then you can make a function that is called inside of the if statement to save the data in the csv.
def write_to_csv(self):
with open('csv_graph.csv', 'a', newline='') as csvfile:
csv_writer = csv.writer(csvfile)
csv_writer.writerows(self.csv_data)
that is the first step I would take, and not to mention that you are writing in the excel every frame that is costly too so I would do that on demand or less frequently.