[FIXED] Launch multiple threads from inside a pyqt thread

Issue

I currently have a script the reads a folder of files (thousands of files) into a list which is then split into 4 sub lists. I then have a thread running for each list. This is very easy to achieve in a python script.

thread1fileList, thread2fileList, thread3fileList, thread4fileList = load.sortFiles(fileList)
threads = [threading.Thread(target=load.threadLoad, args=(fileList, ))
for fileList in (thread1fileList, thread2fileList, thread3fileList, thread4fileList)]
for thread in threads:
    thread.start()
for thread in threads:
    thread.join()

However I am now moving this code to a GUI using Pyside.

I have been able to create a thread that reads the folder of files (to make sure the GUI is still responsive) but I cannot launch 4 new threads from inside the Qt.Thread to work on the file list.

This is my core code

self.unzipThread = UnzipThread()
self.unzipThread.start()
self.unzipThread.finished.connect(self.finishUp()) #finsihUp provides the final page of the wizard


class UnzipThread(QtCore.QThread):
    def __init__(self):
        QtCore.QThread.__init__(self)

    def run(self):
        for dirname, dirnames, filenames in os.walk(directorypath):
            for filename in filenames:
                if filename.endswith(".zip"):
                   fileList.append(filename)
                   count = count +1

If I try and add launching a thread from within the run function it causes an error stating that isn't allowed.

How else could I achieve this?

Thanks

EDIT##

Full example

#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
from PySide import QtGui, QtCore
import os

class Example(QtGui.QWidget):

def __init__(self):
    super(Example, self).__init__()

    self.initUI()

def initUI(self):

    runButton = QtGui.QPushButton("Run")
    runButton.clicked.connect(self.unzip)
    exitButton = QtGui.QPushButton("Exit")
    exitButton.clicked.connect(QtCore.QCoreApplication.instance().quit)
    hbox = QtGui.QHBoxLayout()
    hbox.addWidget(runButton)
    hbox.addStretch(1)
    hbox.addWidget(exitButton)
    titleBox = QtGui.QVBoxLayout()
    titleLabel = QtGui.QLabel(self.tr("Find Zip File"))

    searchBox = QtGui.QHBoxLayout()
    self.fileLabel = QtGui.QLabel(self.tr("Location: "))
    self.fileLineEdit = QtGui.QLineEdit()
    self.fileDialogBtn =  QtGui.QPushButton("Browse")
    self.fileDialogBtn.clicked.connect(self.openDirectoryDialog)
    searchBox.addWidget(self.fileLabel)
    searchBox.addWidget(self.fileLineEdit)
    searchBox.addWidget(self.fileDialogBtn)

    titleBox.addWidget(titleLabel)
    titleBox.addStretch(1)
    titleBox.addLayout(searchBox)
    titleBox.addStretch(1)
    titleBox.addLayout(hbox)

    self.setLayout(titleBox)

    self.resize(500, 300)
    self.center()
    self.setWindowTitle('Example')
    self.show()

def unzip(self):
    self.unzipThread = UnzipThread(self.filePath)
    self.unzipThread.start()
    self.unzipThread.finished.connect(self.finishUp)

def openDirectoryDialog(self):
    flags = QtGui.QFileDialog.DontResolveSymlinks | QtGui.QFileDialog.ShowDirsOnly
    directory = QtGui.QFileDialog.getExistingDirectory(self, "Open Directory", os.getcwd(),flags)
    self.fileLineEdit.setText(directory)
    self.filePath = self.fileLineEdit.text()

def center(self):
    qr = self.frameGeometry()
    cp = QtGui.QDesktopWidget().availableGeometry().center()
    qr.moveCenter(cp)
    self.move(qr.topLeft())

def closeEvent(self, event):
    reply = QtGui.QMessageBox.question(self, 'Message',
        "Are you sure to quit?", QtGui.QMessageBox.Yes |
        QtGui.QMessageBox.No, QtGui.QMessageBox.No)

    if reply == QtGui.QMessageBox.Yes:
        event.accept()
    else:
        event.ignore()

def finishUp(self):#yet to code up to GUI log windows
    print "finished"

class UnzipThread(QtCore.QThread):
def __init__(self, filepath):
    QtCore.QThread.__init__(self)

    self.filePath = filepath

def run(self):
    fileList = []
    count = 0
    for dirname, dirnames, filenames in os.walk(self.filePath):
        for filename in filenames:
            if filename.endswith(".shp"):
                fileList.append(filename)
                count += 1
    print count

    list1 = []
    list2 = []

    for shpfile in fileList:
        if "line" in shpfile:
            list1.append(shpfile)
        elif "point" in shpfile:
            list2.append(shpfile)
        else:
            pass

    self.processThread1 = ProcessThread1(list1)
    self.processThread1.start()
    self.processThread1.finished.connect(self.threadCount)

    self.processThread2 = ProcessThread2(list2)
    self.processThread2.start()
    self.processThread2.finished.connect(self.threadCount)

    return

def threadCount(self):
    print "got here"

class ProcessThread1(QtCore.QThread):
def __init__(self, filelist):
    QtCore.QThread.__init__(self)

    self.filelist = filelist

def run(self):
    count = 0
    for textfile in self.filelist:
        count +=1

    print "thread 1 count %s" %(count)

    return

class ProcessThread2(QtCore.QThread):
def __init__(self, filelist):
    QtCore.QThread.__init__(self)

    self.filelist = filelist

def run(self):
    count = 0
    for textfile in self.filelist:
        count +=1

    print "thread 2 count %s" %(count)

    return

def main():

app = QtGui.QApplication(sys.argv)
ex = Example()
sys.exit(app.exec_())


if __name__ == '__main__':
    main()

In stripping out alot of my code to provide this example it seems running a thread from inside my QT.Thread works however the initial thread believes it has finished before the threads that are running have finished.

So the results I get look like this

5635 finishedthread 1 count 2858

thread 2 count 2777here

here

So you can see the comment "finished" appears before the other code "thread count 1" etc, which proves that it thinks it has finished.

Anyway I can get them to communicate?

thanks

EDIT 2

So I have now changed my code to use QObjects and movetoThread but how do I properly determine that both sub threads have finished. This is now my code. I have had to to use a thread counter and a While loop to determine if both sub threads have finished. There must be a better way??

#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
from PySide import QtGui, QtCore
import os
from PySide.QtCore import Signal as pyqtSignal
import time

class Example(QtGui.QWidget):

    def __init__(self):
        super(Example, self).__init__()

        self.initUI()

    def initUI(self):

        runButton = QtGui.QPushButton("Run")
        runButton.clicked.connect(self.unzip)
        exitButton = QtGui.QPushButton("Exit")
        exitButton.clicked.connect(QtCore.QCoreApplication.instance().quit)
        hbox = QtGui.QHBoxLayout()
        hbox.addWidget(runButton)
        hbox.addStretch(1)
        hbox.addWidget(exitButton)
        titleBox = QtGui.QVBoxLayout()
        titleLabel = QtGui.QLabel(self.tr("Find Zip File"))

        searchBox = QtGui.QHBoxLayout()
        self.fileLabel = QtGui.QLabel(self.tr("Location: "))
        self.fileLineEdit = QtGui.QLineEdit()
        self.fileDialogBtn =  QtGui.QPushButton("Browse")
        self.fileDialogBtn.clicked.connect(self.openDirectoryDialog)
        searchBox.addWidget(self.fileLabel)
        searchBox.addWidget(self.fileLineEdit)
        searchBox.addWidget(self.fileDialogBtn)

        titleBox.addWidget(titleLabel)
        titleBox.addStretch(1)
        titleBox.addLayout(searchBox)
        titleBox.addStretch(1)
        titleBox.addLayout(hbox)

        self.setLayout(titleBox)

        self.resize(500, 300)
        self.center()
        self.setWindowTitle('Example')
        self.show()

    def unzip(self):
        thread = self.thread = QtCore.QThread()
        worker = self.worker = Worker(self.filePath)
        worker.moveToThread(thread)
        worker.finished.connect(worker.deleteLater)
        worker.finished.connect(thread.quit)
        thread.started.connect(worker.run)
        thread.finished.connect(self.finishUp)
        thread.start()

    def openDirectoryDialog(self):
        flags = QtGui.QFileDialog.DontResolveSymlinks | QtGui.QFileDialog.ShowDirsOnly
        directory = QtGui.QFileDialog.getExistingDirectory(self, "Open Directory", os.getcwd(),flags)
        self.fileLineEdit.setText(directory)
        self.filePath = self.fileLineEdit.text()

    def center(self):
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def closeEvent(self, event):
        reply = QtGui.QMessageBox.question(self, 'Message',
            "Are you sure to quit?", QtGui.QMessageBox.Yes |
            QtGui.QMessageBox.No, QtGui.QMessageBox.No)

        if reply == QtGui.QMessageBox.Yes:
            event.accept()
        else:
            event.ignore()

    def finishUp(self):#yet to code up to GUI log windows
        print "unzip thread finished"

class Worker(QtCore.QObject):
    finished = pyqtSignal()

    def __init__(self, filepath):
        QtCore.QObject.__init__(self)

        self.filePath = filepath

    def run(self):
        self.threadcount = 0
        print "finding files"
        fileList = []
        count = 0
        for dirname, dirnames, filenames in os.walk(self.filePath):
            for filename in filenames:
                if filename.endswith(".shp"):
                    fileList.append(filename)
                    count += 1
        print count

        self.list1 = []
        self.list2 = []

        for shpfile in fileList:
            if "line" in shpfile:
                self.list1.append(shpfile)
            elif "point" in shpfile:
                self.list2.append(shpfile)
            else:
                pass

        thread1 = self.thread1 = QtCore.QThread()
        worker1 = self.worker1 = Process1(self.list1)
        worker1.moveToThread(thread1)
        worker1.finished.connect(worker1.deleteLater)
        worker1.finished.connect(thread1.quit)
        thread1.started.connect(worker1.run)
        thread1.finished.connect(self.finishUp)
        thread1.start()


        thread2 = self.thread2 = QtCore.QThread()
        worker2 = self.worker2 = Process2(self.list2)
        worker2.moveToThread(thread2)
        worker2.finished.connect(worker2.deleteLater)
        worker2.finished.connect(thread2.quit)
        thread2.started.connect(worker2.run)
        thread2.finished.connect(self.finishUp)
        thread2.start()

    def finishUp(self):
        self.threadcount += 1
        while True:
            if self.threadcount != 2:
                break
            else:
                print "extra threads finished"
                self.finished.emit()
                break


class Process1(QtCore.QObject):
    finished = pyqtSignal()
    def __init__(self, filelist):
        QtCore.QObject.__init__(self)

        self.filelist = filelist

    def run(self):
        count = 0
        for textfile in self.filelist:
            count +=1

        print "thread 1 count %s" %(count)

        self.finished.emit()


class Process2(QtCore.QObject):
    finished = pyqtSignal()
    def __init__(self, filelist):
        QtCore.QObject.__init__(self)

        self.filelist = filelist

    def run(self):
        count = 0
        for textfile in self.filelist:
            count +=1

        time.sleep(15)

        print "thread 2 count %s" %(count)

        self.finished.emit()

def main():

    app = QtGui.QApplication(sys.argv)
    ex = Example()
    sys.exit(app.exec_())


if __name__ == '__main__':
    main()

Solution

First, the comment "finished" appears before the "thread count x" because the UnzipThread indeed finishes before the ProcessingThreads. This is as expected.

Second I can't say if there is a communication problem but be sure to read about the true usage of QThreads. Subclassing QThread is not recommended because of problems with signalling. Better to use worker objects and move them to Threads via QObject:movetoThread().

If still in doubt print QThread.currentThread() in all the possible places and check what code is actually running in which thread.

Answered By - Trilarion