Display a large csv file in PyQt Table View

Question

Im trying to display the Master File Table of an NTFS Volume ,in a PyQt Application . I have extracted the MFT and converted into a csv file , now i wish to display the data in a tabular form using PyQt Table View. The program runs perfectly without any error but displays nothing.

The CSV file has a size of 300 Mb.

right now this is what my code looks like :

try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName(_fromUtf8("MainWindow"))
        MainWindow.resize(640, 480)
        self.centralwidget = QtGui.QWidget(MainWindow)
        self.centralwidget.setObjectName(_fromUtf8("centralwidget"))
        self.verticalLayout = QtGui.QVBoxLayout(self.centralwidget)
        self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
        self.tableView = QtGui.QTableView(self.centralwidget)
        self.tableView.setObjectName(_fromUtf8("tableView"))
        self.verticalLayout.addWidget(self.tableView)
        MainWindow.setCentralWidget(self.centralwidget)

        self.actionOpen = QtGui.QAction(MainWindow)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(_fromUtf8(":/icons/open.jpg")), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionOpen.setIcon(icon)
        self.actionOpen.setObjectName(_fromUtf8("actionOpen"))


        self.retranslateUi(MainWindow)
        QtCore.QObject.connect(self.actionExit, QtCore.SIGNAL(_fromUtf8("triggered(bool)")), MainWindow.close)
        QtCore.QObject.connect(self.actionOpen, QtCore.SIGNAL("triggered()"),self.ExtractMFT)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        MainWindow.setWindowTitle(QtGui.QApplication.translate("MainWindow", "MainWindow", None, QtGui.QApplication.UnicodeUTF8))
        self.actionOpen.setText(QtGui.QApplication.translate("MainWindow", "Open", None, QtGui.QApplication.UnicodeUTF8))


    def ExtractMFT(self, root = None):
        if root == None:
            root = "\\\\.\C:"

        FileName = "MFT-EXtracted"
        CSVName = "MFT-EXtracted.csv"
        print 1
        Control=subprocess.Popen(["icat",root,"0-128-1",">",FileName],shell =True,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
        Control.wait()
        print 2
        print Control.stdout.read()
        if Control.stderr == None:
            print 3
            #Control=subprocess.Popen(["python","analyzeMFT.py","-f",FileName,"-o",CSVName],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
            #Control.wait()
            print 5
            print Control.stdout.read()
            if Control.stderr == None:
                self.loadCsv(CSVName)


    def loadCsv(self, fileName):
        # The problem persist in this function. The items being appended to the 
        # model are not being displayed by the tableView , infact the tableView 
        # is empty nothing come's up. 
        header = False
        header_data=[]
        data=[]
        print 6
        se2 = 0
        model = QtGui.QStandardItemModel()
        with open(fileName, "rb") as fileInput:
            for row in csv.reader(fileInput):
                if header == True:
                    items = [
                    QtGui.QStandardItem(field)
                    for field in row
                ]
                    model.appendRow(items)
                elif header ==False:
                    for field in row:
                        items = [
                        field
                        for field in row
                        ]
                        header_data.append(items)
                     header == True

           print 7
           self.tableView.setModel(model)

def main():
    app = QtGui.QApplication(sys.argv)
    MainWindow = QtGui.QMainWindow() # <-- Instantiate QMainWindow object.
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    app.exec_()



main()

The CSV file that i intend to load has 300,000+ rows in it, so is their an efficient way to load data into the view. So that less system resources are used.

Answer 1

Most likely you want to just load the parts of file that the user can see (plus a little bit on either side so you can determine when to load more). You should be able to apply the techniques discussed in Big Tables page. There may be other similar tutorials or discussions that you can find via google.