Python: Update Widgets from outside function using multipocessing Pool

Question

I would appreciate if you enlighten me why pressing Ok button doesn't update lineedit text field.

from PyQt4 import QtCore, QtGui

def externalFunc(arg):
    print '\n\t Accessing lineedit from outside. Result   "', jobDialog.lineEdit.text(), '"   OK'
    print "Attempting to change the lineEdit field to", arg
    jobDialog.lineEdit.setText(arg)
    print "...Completed."

class MainWindow(QtGui.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.main_layout = QtGui.QVBoxLayout()

        self.lineEdit=QtGui.QLineEdit('Initial Text')
        self.main_layout.addWidget(self.lineEdit)

        ok_button = QtGui.QPushButton("OK")
        ok_button.clicked.connect(self.OK)
        self.main_layout.addWidget(ok_button)       

        central_widget = QtGui.QWidget()
        central_widget.setLayout(self.main_layout)
        self.setCentralWidget(central_widget)

    def OK(self):
        myList=['One','Two','Three']
        from multiprocessing import Pool
        pool = Pool(processes=10)
        try: pool.map_async( externalFunc, myList)
        except Exception, e: print e

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)

    jobDialog = MainWindow()
    jobDialog.resize(480, 320)
    jobDialog.show()
    sys.exit(app.exec_())

Another example, this time with Progress Bar.

This one instead of just failing to update crashes an entire process. Same questions: why does it happen and how to fix it.

from PyQt4 import QtCore, QtGui

class PbWidget(QtGui.QProgressBar):
    def __init__(self, parent=None, total=20):
        super(PbWidget, self).__init__()
        self.setMinimum(1)
        self.setMaximum(total)        
        self._active = False  

    def update_bar(self, to_add_number):
        while True:
            time.sleep(0.01)
            value = self.value() + to_add_number            
            self.setValue(value)
            QtGui.qApp.processEvents()
            if (not self._active or value >= self.maximum()):                
                break
        self._active = False

    def closeEvent(self, event):
        self._active = False

def externalFunc(arg=None):
    print "Attempting to update Progress Bar "
    jobDialog.pb.update_bar(10)
    print "...Completed."

class MainWindow(QtGui.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.main_layout = QtGui.QVBoxLayout()

        self.pb=PbWidget(total=101)
        self.main_layout.addWidget(self.pb)

        ok_button = QtGui.QPushButton("OK")
        ok_button.clicked.connect(self.OK)
        self.main_layout.addWidget(ok_button)       

        central_widget = QtGui.QWidget()
        central_widget.setLayout(self.main_layout)
        self.setCentralWidget(central_widget)

    def OK(self):
        # externalFunc()
        myList=[1,2,3]
        from multiprocessing import Pool
        pool = Pool(processes=10)
        pool.map_async( externalFunc, myList)

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)

    jobDialog = MainWindow()
    jobDialog.resize(480, 320)
    jobDialog.show()
    sys.exit(app.exec_())

Answer 1

Here is a description of solution that worked for me.

The sub-processes started by multiprocessing's Pool methods (such as map_async()) are unable to communicate with the variables (objects) declared in a main process (a process where all the sub-processes were spawned from). It could be sufficient if all we want is to send the tasks for processing. But often we need a "live" feedback from the sub-processes (while they still work on the tasks received). In order to supply the sub-processes with a variable "visible" ("shared") among the main process and all its sub-processes we use multiprocessing Manager(). A syntax is quite simple:

from multiprocessing import Manager
manager = Manager()

After an instance of Manager() is declared (here it is called manager) we proceed with declaring a variable... so far I am aware of aka-dict and aka-list types. A following syntax is used:

myDict=manager.dict()

Aside from storing and retrieving data those variables could be used to set/read/reset True/False flags used to start/stop while loop functions (sort of Event Listeners). The idea is that we run a while loop at the background which is constantly listening (monitoring) a state of such True or False variable. Here is the revised code with Progress Bar being updated by sub-processes initiated by multiprocessing Pool() map_async() method.

from PyQt4 import QtCore, QtGui

from multiprocessing import Process, Manager, Pool
manager = Manager()
myDict=manager.dict()
myDict['state'] = True
myDict['value'] = 0


class PbWidget(QtGui.QProgressBar):
    def __init__(self, parent=None, total=20):
        super(PbWidget, self).__init__()
        self.setMinimum(1)
        self.setMaximum(total)        
        self._active = False  

    def update_bar(self, to_add_number):
        while True:
            time.sleep(0.01)
            value = self.value() + to_add_number            
            self.setValue(value)
            QtGui.qApp.processEvents()
            if (not self._active or value >= self.maximum()):                
                break
        self._active = False

    def closeEvent(self, event):
        self._active = False




def externalFunc(arg=None):
    print "\t\t Attempting to update Progress Bar by changing dictionary values"
    myDict['value']=arg
    myDict['state']=True

class MainWindow(QtGui.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.progressBarCurrentValue=0
        self.main_layout = QtGui.QVBoxLayout()

        self.pb=PbWidget(total=101)
        self.main_layout.addWidget(self.pb)

        ok_button = QtGui.QPushButton("OK")
        ok_button.clicked.connect(self.OK)
        self.main_layout.addWidget(ok_button)       

        central_widget = QtGui.QWidget()
        central_widget.setLayout(self.main_layout)
        self.setCentralWidget(central_widget)


    def OK(self):
        myList=[10,10,10,10,10,10,10,10,10,10,10,10,10]
        pool = Pool(processes=10)
        pool.map_async( externalFunc, myList)

    def EventListener(self):
        while myDict['state']:              
            value=myDict['value']
            self.pb.update_bar(value)

            self.progressBarCurrentValue=self.pb.value()
            print "...running sicne current Progress Bar value is < 99:", self.progressBarCurrentValue, myDict['state']

            if self.progressBarCurrentValue>99: 
                myDict['state'] = False
                print "...stopping"



if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)    
    jobDialog = MainWindow()
    jobDialog.resize(480, 320)
    jobDialog.show()
    jobDialog.EventListener()
    sys.exit(app.exec_())

Answer 2

It doesn't work because memory is not shared between processes (in general). Thus one process can not update widgets in another directly. Even if you did share memory, your program would regularly crash because Qt doesn't allow simultaneous access to QObjects (eg through threads or multi-processing).

What you will need to do is have your sub-processes send a message back to the main thread of your main process and from their interpret the message and update the widget.