Python GUI (tkinter.ttk) application slow

Question

I've got (working) application done with ttk. It uses self-created module for showing a comport-related controls and a canvas which draws a few graphs on it. When I crate an instance of my object, it starts a thread in which processes serial input and appends this to a list (one list per graph). When I have 3-6 graphs, the application gets noticeably slow. It also has got a few bugs, but I will adress them when I'm done with the general concept.

Things that may help you help me:

• comport is an instance of a self-written object that derives from LabelFrame and Serial.Serial
• coordinates for graphs are stored in a dictionary of lists: self.graphs = {} self.graphs['name1']=[] number of coordinates stored is up to the width of canvas, so about 1000-2000 per graph. Have six graphs - please multiply by 6
• With every new coordinate arriving I pop(0) from the list and append() the new coordinate
• I forgot, I also store timing of each new set of coordinates arriving in a separate list
• I use a preiodic call function to process the lists: self.after(100, func=self.periodicCall)Thus every 100ms I delete(ALL) from the canvas and I draw every graph with theset of lines. So if I have 1000 coords in 6 graps, I draw 6000 small lines
• Plus some service info of course such as a few rulers

So I guess the idea is clear. I want to figure out what would be the better approach. I'm just a started in python and in programming as well, so I'm asking for your excuse for the code that I'm going to post and for the pain in your eyes it's gonna cause. I don't have any programming style and I want to fix it. At least a bit. So any other comments on anything you'll se in the code are welcome.

#-------------------------------------------------------------------------------
# Name:        dataVisualizer
# Purpose:
#
# Author:      dccharacter
#
# Created:     23.03.2012
# Copyright:   (c) dccharacter 2012
# Licence:     <your licence>
#-------------------------------------------------------------------------------
#!/usr/bin/env python

from tkinter import *
from tkinter.ttk import *
from robowidgets.serialPortGui import *
import threading
import re
import atexit
import random
from datetime import datetime
import time

class dataVisualizer(LabelFrame):
    def __init__(self, master, comport , cnf={}, **kw):
        self.master = master
        self.comport = comport
        LabelFrame.__init__(self, *cnf, **kw)

        self.messageVar = StringVar()
        Label(self, text="Message format regexp:").pack()
        self.messagePattern = Entry(self, width = 20, text = 234234, textvariable = self.messageVar);
        self.messageVar.set(r'(-*\d+),(-*\d+),(-*\d+),(-*\d+),(-*\d+),(-*\d+)')
        self.messagePattern.pack()
        Button(self, text = "Pause", command = self.pause).pack()
        self.pauseFlag = TRUE

        self.canvWidth, self.canvHeight = 1000, 700
        self.density = 1 ##width of pixel - the bigger, the wider graph
        self.numOfDots = self.canvWidth//self.density
        self.graphs = {}
        self.graphs['name1']=[]
        self.graphs['name2']=[]
        self.graphs['name3']=[]
        self.graphs['name4']=[]
        self.graphs['name5']=[]
        self.graphs['name6']=[]
        self.timings = []
        self.zeroTiming = datetime.now()
        self.colors = ['red', 'blue', 'green', 'orange', 'violet', 'black', 'cyan']

        self.canv = Canvas(self, width = self.canvWidth, height = self.canvHeight)
        self.canv.pack()

        self.thread = threading.Thread(target = self.workerThread)
        self.thread.start()

        self.serialData = []

        self.periodicCall()

    def pause(self):
        self.pauseFlag = ~self.pauseFlag

    def redraw(self):
        self.canv.delete(ALL)

        colorIndex = 0
        for graphName in self.graphs:
            runningAverage = sum(self.graphs[graphName][-10:])//10
            text = str(runningAverage)
            self.canv.create_text(self.canvWidth-60, 20*(colorIndex+1), text = text,
                fill = self.colors[colorIndex], anchor = W)
            prev_xxx, prev_yyy = 0, 0
            for yyy in self.graphs[graphName]:
                self.canv.create_line(prev_xxx, prev_yyy, prev_xxx+self.density, self.canvHeight//2 - yyy,
                    width = 1.4, fill = self.colors[colorIndex])
                prev_xxx, prev_yyy = prev_xxx+self.density, self.canvHeight//2 - yyy
            colorIndex = colorIndex + 1
        self.drawMesh()

    def drawMesh(self):
        self.canv.create_rectangle(3, 3, self.canvWidth,
            self.canvHeight, outline = 'black', width = 2)
        self.canv.create_line(0, self.canvHeight/2, self.canvWidth,
            self.canvHeight/2, fill="black", width = 1)

        mouseX = self.canv.winfo_pointerx() - self.canv.winfo_rootx()
        mouseY = self.canv.winfo_pointery() - self.canv.winfo_rooty()

        if mouseY < 60: aaa = -1
        else: aaa = 1
        if mouseX > self.canvWidth - 200 : bbb = -12
        else: bbb = 1
        try:
            self.canv.create_rectangle(mouseX + 10*bbb - 5, mouseY - 20*aaa +10,
                mouseX + 10*bbb + 115, mouseY - 20*aaa - 30, outline = "black",
                fill = "red")
            self.canv.create_text(mouseX + 10*bbb, mouseY - 40*aaa,
                text = "t="+str(self.timings[mouseX//self.density]),
                anchor = W)
            self.canv.create_text(mouseX + 10*bbb, mouseY - 20*aaa,
                text = "value="+str(self.canvHeight//2 - mouseY),
                anchor = W)
        except IndexError:
            pass
        self.canv.create_line(mouseX, 0, mouseX,
            self.canvHeight, fill="blue", dash = [4, 1, 2, 1], width = 1)
        self.canv.create_line(0, mouseY, self.canvWidth,
            mouseY, fill="blue", dash = [4, 1, 2, 1], width = 1)


    def periodicCall(self):
        self.redraw()
        self.after(100, func=self.periodicCall)

    def workerThread(self):

        while (1):
            try:
                if self.comport.isOpen() and (self.pauseFlag == TRUE):
                    comLine = self.comport.readline()
                    if len(self.timings) == self.numOfDots:
                        self.timings.pop(0)
                    td = datetime.now() - self.zeroTiming

                    ##  b'271;-3:-50\r\n'
                    parsedLine = re.search(self.messagePattern.get(), str(comLine))
                    index = 1
                    if parsedLine:
                        self.timings.append(td)
                        for graphName in self.graphs:
                            if len(self.graphs[graphName]) == self.numOfDots:
                                self.graphs[graphName].pop(0)
                            try:
                                self.graphs[graphName].append(int(parsedLine.group(index)))
                            except IndexError:
                                self.graphs[graphName].append(0)
                            index = index + 1
                else:
                    self.comport.flush();
                    time.sleep(1)
            except TclError:
                self.thread._stop()

def main():
    root = Tk()
    mainWindow = Frame(root)
    mainWindow.pack()
    port = comPortWidget(mainWindow)
    port.pack()
    dv = dataVisualizer(mainWindow, port)
    dv.pack()
    root.mainloop()

if __name__ == '__main__':
    main()

And the serial part - may lag as well (used to lag when I used to reenumerate ports evey second or so...)

#-------------------------------------------------------------------------------
# Name:        robowidgets
# Purpose:
#
# Author:      dccharacter
#
# Created:     10.03.2012
# Copyright:   (c) dccharacter 2012
# Licence:     <your licence>
#-------------------------------------------------------------------------------
#!/usr/bin/env python

import serial
from serial.tools.list_ports_windows import comports
from tkinter import *
from tkinter.ttk import *

class comPortWidget(LabelFrame, serial.Serial):

    commonComPortSpeeds = ["1200", "2400", "4800", "9600", "14400", "19200", "38400", "57600", "115200"]

    def __init__(self, master=None, cnf={}, **kw):
        """Construct a comPortWidget widget with the parent MASTER.

        STANDARD OPTIONS

            borderwidth, cursor, font, foreground,
            highlightbackground, highlightcolor,
            highlightthickness, padx, pady, relief,
            takefocus, text, background, class, colormap, container,
            height, labelanchor, labelwidget,
            visual, width

        WIDGET-SPECIFIC OPTIONS


        """
        self.master = master
        LabelFrame.__init__(self, master, text="Serial settings", *cnf, **kw)
        serial.Serial.__init__(self)
        self.parent = master
        self.draw()

    def draw(self):
        self.strVarComPort = StringVar()
        self.comboComport = Combobox(self,
            textvariable=self.strVarComPort)

        self.comboComport.grid(row=0, column=1)
        self.labelComportName = Label(self, text="Com port:")
        self.labelComportName.grid(row=0, column=0)

        self.strVarComSpeed = StringVar()
        self.comboComSpeed = Combobox(self,
            textvariable=self.strVarComSpeed, values=self.commonComPortSpeeds)
        self.comboComSpeed.current(len(self.commonComPortSpeeds)-1)
        self.comboComSpeed.grid(row=1, column=1)
        self.labelComSpeed = Label(self, text="Com speed:")
        self.labelComSpeed.grid(row=1, column=0)

        self.buttonComOpen = Button(self, text="Open port", command=self.openPort)
        self.buttonComOpen.grid(row=0, column=2)
        self.buttonComClose = Button(self, text="Close port", command=self.closePort)
        self.buttonComClose.grid(row=1, column=2)
        self.buttonRefreshPorts = Button(self, text="Re", width=3, command=self.refreshComPortsCombo)
        ##self.buttonRefreshPorts.grid(row=0, column=2)

        self.refreshComPortsCombo()

    def refreshComPortsCombo(self):
        listComs = self.enumerateComPorts()
        if not listComs:
            listComs.append("No com ports found")
            self.disableControls(~self.isOpen())
            self.buttonComClose.configure(state=DISABLED)
        else:
            self.disableControls(self.isOpen())
        self.buttonRefreshPorts.configure(state=NORMAL)
        self.comboComport.config(values=listComs)
        self.comboComport.current(len(listComs)-1)
        ##self.after(500, func=self.refreshComPortsCombo)

    def enumerateComPorts(self):
        """
        Returns the list ofcom port names in the system or an empty list if
        no ports found
        """
        listComs = []
        for port, desc, hwid in sorted(comports()):
            listComs.append(port)
        return listComs

    def openPort(self):
        if self.isOpen():
            return
        self.port = self.comboComport.get()
        self.baudrate = int(self.comboComSpeed.get())
        self.timeout = 1
        try:
            self.open()
            self.disableControls(self.isOpen())
        except IOError:
            pass

    def closePort(self):
        if self.isOpen():
            self.flush()
            self.close()
            self.disableControls(self.isOpen())

    def disableControls(self, isConnected):
        if isConnected:
            self.labelComportName.configure(state=DISABLED)
            self.labelComSpeed.configure(state=DISABLED)
            self.comboComport.configure(state=DISABLED)
            self.comboComSpeed.configure(state=DISABLED)
            self.buttonComClose.configure(state=NORMAL)
            self.buttonComOpen.configure(state=DISABLED)
            self.buttonRefreshPorts.configure(state=DISABLED)
        else:
            self.labelComportName.configure(state=NORMAL)
            self.labelComSpeed.configure(state=NORMAL)
            self.comboComport.configure(state=NORMAL)
            self.comboComSpeed.configure(state=NORMAL)
            self.buttonComClose.configure(state=DISABLED)
            self.buttonComOpen.configure(state=NORMAL)
            self.buttonRefreshPorts.configure(state=NORMAL)

def main():
    pass

if __name__ == '__main__':
    main()

UPDATE: I did as Brian advised. Now I have two screen redraw functions. Difference between them is that first moves all the lines to the left adding new to the right and deleting those that fall off the canvas. The second one moves lines to the left and re-deploys elements that fall off the canvas to the right (without creating new ones). There's a huge improvement with any of these in respect to my initial variant, but I don't see big difference between the two wit the naked eye - mayme if I had more elements I would. The latter though works better specifically for my application as I don't have to track those who fall off the cliff.

Here the functions:

def drawGraph(self): ###needed for self.updateGraph2() only as it is creates the lines
    for graphNum in range(0, self.numOfGraphs):
        self.graphLines.append([])
        self.graphData.append([0,]*self.numOfDots)
        for iii in range(0,self.numOfDots):
            self.graphLines[graphNum].append(
                self.canv.create_line(0,0,0,0,fill=self.colors[graphNum],
                width=1.2, tags=('graphLines', 'graph'+str(graphNum)))
                )


def updateGraph2(self):
    while not self.queue.empty():
        iTuple = self.queue.get()
        self.canv.move('graphLines', -self.density,0)
        for graphNum in range(0, self.numOfGraphs):
            try: self.graphData[graphNum].append(iTuple[graphNum])
            except IndexError:
                self.graphData[graphNum].append(0)
            self.graphData[graphNum].pop(0)
            self.graphLines[graphNum].append(self.graphLines[graphNum].pop(0))
            self.canv.coords(self.graphLines[graphNum][-1],
                self.canv.winfo_width()-self.density,
                int(int(self.graphData[graphNum][-2])+int(self.canv.winfo_height()//2)),
                self.canv.winfo_width(),
                int(int(self.graphData[graphNum][-1])+int(self.canv.winfo_height()//2))
                )

def updateGraph(self):
    while not self.queue.empty():
        self.timingIndex = self.timingIndex + 1
        self.canv.move('graphLines', -self.density, 0)
        iTuple = self.queue.get()
        for iii in range(0, len(iTuple)):
            yyy = int(iTuple[iii])+self.canv.winfo_height()//2
            if yyy < 0: yyy = 0
            if yyy > self.canv.winfo_height(): yyy = self.canv.winfo_height()
            prev_yyy = int(self.prevTuple[iii])+self.canv.winfo_height()//2
            if prev_yyy < 0: prev_yyy = 0
            if prev_yyy > self.canv.winfo_height(): prev_yyy = self.canv.winfo_height()
            self.canv.create_line(
                self.canv.winfo_width()-self.density, prev_yyy,
                self.canv.winfo_width(), yyy,
                width = 1.4, fill = self.colors[iii], tags=('graphLines','graph'+str(iii)))
        self.prevTuple = iTuple

        self.canv.addtag_overlapping('todelete',-1,-1,-3,self.canv.winfo_height()+1)
        self.canv.dtag('preserve','todelete')
        self.canv.delete('todelete')

Answer 1

My understanding of the canvas is that the more element ids that have been allocated, the slower it gets. It can handle tens of thousands without much problem (and maybe even 100's of thousands), but if you're creating and deleting 6000 items every 100ms, that is likely your problem. Even though you are deleting the items, it still affects performance especially when you are creating 60,000 per second.

Instead of deleting all items every 100ms, simply move the items off screen and remember them, then reuse them by using the coords method to change their coordinates for the new graph.