Clear treeview and tableview. Reuse session

ED2K.py

@@ -70,67 +70,148 @@
 #define  FT_FILERATING			 0xF7	// <uint8>
 """
 
-
-import sys
 import os
 import binascii
+from enum import Enum
 from collections import namedtuple
 
 
-METENTRY_FIELDS = ('FT_FILENAME', 'ED2KHASH', 'NUMPARTS', 'LASTCHANGED', 'PARTHASHES', 'FT_FILESIZE',
-                   'FT_ATTRANSFERRED', 'FT_ATREQUESTED', 'FT_ATACCEPTED', 'FT_ATTRANSFERREDHI', 'FT_PARTFILENAME',
-                   'FT_AICHHASHSET', 'FT_AICH_HASH', 'FT_ULPRIORITY', 'FT_KADLASTPUBLISHSRC')
-MetEntry = namedtuple('met_entry', METENTRY_FIELDS)
+# METENTRY_FIELDS = ('FT_FILENAME', 'ED2KHASH', 'NUMPARTS', 'LASTCHANGED', 'PARTHASHES', 'FT_FILESIZE',
+#                    'FT_ATTRANSFERRED', 'FT_ATREQUESTED', 'FT_ATACCEPTED', 'FT_ATTRANSFERREDHI', 'FT_PARTFILENAME',
+#                    'FT_AICHHASHSET', 'FT_AICH_HASH', 'FT_ULPRIORITY', 'FT_KADLASTPUBLISHSRC')
+# MetEntry = namedtuple('met_entry', METENTRY_FIELDS)
 
-HEADER_KNOWN_MET_FILE = b'\x0e'
-HEADER_PART_MET_FILE = b'\xe0'
+HEADER_KNOWN_MET_FILE = b'\x0E'
+HEADER_PART_MET_FILE = b'\xE0'
 
-FT_FILENAME = 0x01
-FT_FILESIZE = 0x02
-FT_PARTFILENAME = 0x12
-FT_PERMISSIONS = 0x16
-FT_ULPRIORITY = 0x19
 
-FT_KADLASTPUBLISHKEY = 0x20
-FT_KADLASTPUBLISHSRC = 0x21
-FT_FLAGS = 0x22
-FT_KADLASTPUBLISHNOTES = 0x26
-FT_AICH_HASH = 0x27
+# https://github.com/amule-project/amule/blob/master/src/include/tags/TagTypes.h
+# https://github.com/amule-project/amule/blob/master/src/include/tags/FileTags.h
 
-FT_LASTSHARED = 0x34
-FT_AICHHASHSET = 0x35
 
-FT_ATTRANSFERRED = 0x50
-FT_ATREQUESTED = 0x51
-FT_ATACCEPTED = 0x52
-FT_ATTRANSFERREDHI = 0x54
+# TYPE: \x02
+class TagString(Enum):
+    FT_FILENAME = 0x01
+    FT_FILETYPE = 0x03
+    FT_FILEFORMAT = 0x04
+    FT_PARTFILENAME = 0x12
+    FT_AICH_HASH = 0x27
+    FT_CORRUPTEDPARTS = 0x24
+    FT_MEDIA_ARTIST = 0xD0
+    FT_MEDIA_ALBUM = 0xD1
+    FT_MEDIA_TITLE = 0xD2
+    FT_MEDIA_CODEC = 0xD5
 
-FT_MEDIA_ARTIST = 0xd0
-FT_MEDIA_ALBUM = 0xd1
-FT_MEDIA_TITLE = 0xd2
-FT_MEDIA_LENGTH = 0xd3
-FT_MEDIA_BITRATE = 0xd4
-FT_MEDIA_CODEC = 0xd5
 
-FT_STRING = (
-    0x03, 0x04, 0x06, 0x0b, 0x11, 0x24, 0x28, 0xf6
-)
+# TYPE: \x03
+class TagUint32(Enum):
+    FT_FILESIZE = 0x02
+    FT_FILESIZE_HI = 0x3A
+    FT_LASTSEENCOMPLETE = 0x05
+    FT_TRANSFERRED = 0x08
+    FT_GAPSTART = 0x09
+    FT_GAPEND = 0x0A
+    FT_STATUS = 0x14
+    FT_SOURCES = 0x15
+    FT_PERMISSIONS = 0x16
+    FT_DLPRIORITY = 0x18
+    FT_ULPRIORITY = 0x19
+    FT_KADLASTPUBLISHKEY = 0x20
+    FT_KADLASTPUBLISHSRC = 0x21
+    FT_FLAGS = 0x22
+    FT_DL_ACTTVE_TIME = 0x23
+    FT_COMPLETE_SOURCES = 0x30
+    FT_PUBLISHINFO = 0x33
+    FT_LASTSHARED = 0x34
+    FT_ATTRANSFERRED = 0x50
+    FT_ATTRANSFERREDHI = 0x54
+    FT_ATREQUESTED = 0x51
+    FT_ATACCEPTED = 0x52
+    FT_CATEGORY = 0x53
+    FT_MEDIA_LENGTH = 0xD3
+    FT_MEDIA_BITRATE = 0xD4
 
-FT_INT = (
-    0x05, 0x08, 0x09, 0x0a, 0x14, 0x15, 0x18, 0x1a, 0x1b, 0x23, 0x25, 0x30, 0x33, 0x53, 0x55, 0xf7
-)
 
-def met_readstring(fd):
+# TYPE: \x09
+class TagUint8(Enum):
+    FT_FILERATING = 0xF7
+
+
+# TYPE: \x07
+class TagBlob(Enum):
+    FT_AICHHASHSET = 0x35
+
+
+class TagDeprecated(Enum):
+    FT_OLDDLPRIORITY = 0x13
+    FT_OLDULPRIORITY = 0x17
+
+
+class TagHash(Enum):
+    NA = 0x00
+
+
+TAG_TYPE = {
+    b'\x01': None,  # HASH
+    b'\x02': TagString,
+    b'\x03': TagUint32,
+    b'\x04': None,  # FLOAT32
+    b'\x05': None,  # BOOL
+    b'\x06': None,  # BOOLARRAY
+    b'\x07': TagBlob,
+    b'\x08': None,  # UINT16
+    b'\x09': TagUint8,
+    b'\x0A': None,  # BSOB
+    b'\x0B': None,  # UINT64
+
+    b'\x12': None,  # STRnn
+    b'\x13': None,  # STRnn
+    b'\x14': None,  # STRnn
+    b'\x15': None,  # STRnn
+    b'\x16': None,  # STRnn
+    b'\x17': None,  # STRnn
+    b'\x18': None,  # STRnn
+    b'\x19': None,  # STRnn
+    b'\x1A': None,  # STRnn
+    b'\x11': None,  # STRnn
+    b'\x1B': None,  # STRnn
+    b'\x1C': None,  # STRnn
+    b'\x1D': None,  # STRnn
+    b'\x1E': None,  # STRnn
+    b'\x1F': None,  # STRnn
+    b'\x20': None,  # STRnn
+    b'\x21': None,  # STRnn
+    b'\x22': None,  # STRnn
+    b'\x23': None,  # STRnn
+    b'\x24': None,  # STRnn
+    b'\x25': None,  # STRnn
+    b'\x26': None,  # STRnn
+}
+
+MetEntry = namedtuple('met_entry',
+                      ['ED2KHASH', 'NUMPARTS', 'LASTCHANGED', 'PARTHASHES','TAGCOUNT']
+                      + [x.name for x in TagString]
+                      + [x.name for x in TagUint32]
+                      + [x.name for x in TagBlob])
+MetEntry.__new__.__defaults__ = (None,) * len(MetEntry._fields)
+
+
+def met_read_string(fd):
     string_len = int.from_bytes(fd.read(2), byteorder='little')
     return fd.read(string_len).decode('utf-8', errors='replace')
 
 
-def met_readinteger(fd, size=4):
+def met_read_integer(fd, size=4):
     return int.from_bytes(fd.read(size), byteorder='little')
 
 
+def met_read_blob(fd):
+    blob_size = met_read_integer(fd)
+    return fd.read(blob_size)
+
+
 def record_last_changed(fd):
-    return met_readinteger(fd)
+    return met_read_integer(fd)
 
 
 def record_read_hash16(fd):
@@ -138,75 +219,43 @@ def record_read_hash16(fd):
     return binascii.hexlify(aichash).decode('utf-8')
 
 
-def record_tags(fd):
-    tags = {}
-    tags['TAGCOUNT'] = met_readinteger(fd)
-    # print('Tags:                {}'.format(tags['TAGCOUNT']))
+def record_read_tags(fd):
+    tags = dict()
+    tags['TAGCOUNT'] = met_read_integer(fd)
     for index in range(tags['TAGCOUNT']):
-        filetag = fd.read(4)
-        if filetag[:3] not in [b'\x02\x01\x00', b'\x03\x01\x00']:
-            print('<<< Premature end of record at offset 0x{:08X}'.format(fd.tell()))
+        tag_buf = fd.read(4)
+        tag_type = bytes([tag_buf[0]])
+        tag_id = tag_buf[-1]
+
+        print('Reading at 0x{:08X}: 0x{}'.format(fd.tell()-4, tag_buf.hex()))
+
+        # Validate TAG bytes
+        if tag_type not in TAG_TYPE and not tag_buf[1:3] == b'\x01\x00':
+            print('<<< Premature end of record at offset 0x{:08X} ({})'.format(fd.tell(), tag_buf))
             return tags
-        tagid = filetag[-1]
-        if tagid == FT_FILENAME:
-            tags['FT_FILENAME'] = met_readstring(fd)
-        elif tagid == FT_FILESIZE:
-            tags['FT_FILESIZE'] = met_readinteger(fd)
-        elif tagid == FT_ATTRANSFERRED:
-            tags['FT_ATTRANSFERRED'] = met_readinteger(fd)
-        elif tagid == FT_ATREQUESTED:
-            tags['FT_ATREQUESTED'] = met_readinteger(fd)
-        elif tagid == FT_ATACCEPTED:
-            tags['FT_ATACCEPTED'] = met_readinteger(fd)
-        elif tagid == FT_ATTRANSFERREDHI:
-            tags['FT_ATTRANSFERREDHI'] = met_readinteger(fd)
-        elif tagid == FT_ULPRIORITY:
-            tags['FT_ULPRIORITY'] = met_readinteger(fd)
-        elif tagid == FT_KADLASTPUBLISHSRC:
-            tags['FT_KADLASTPUBLISHSRC'] = met_readinteger(fd)
-        elif tagid == FT_KADLASTPUBLISHNOTES:
-            print('FT_KADLASTPUBLISHNOTES')
-            met_readinteger(fd)
-        elif tagid == FT_FLAGS:
-            print('FT_FLAGS')
-            met_readinteger(fd)
-        elif tagid == FT_PERMISSIONS:
-            print('FT_PERMISSIONS')
-            met_readinteger(fd)
-        elif tagid == FT_KADLASTPUBLISHKEY:
-            print('FT_KADLASTPUBLISHKEY')
-            met_readinteger(fd)
-        elif tagid == FT_AICH_HASH:
-            tags['FT_AICH_HASH'] = met_readstring(fd)
-        elif tagid == FT_LASTSHARED:
-            tags['FT_LASTSHARED'] =  met_readinteger(fd)
-        elif tagid == FT_AICHHASHSET:
-            tags['FT_AICHHASHSET'] = met_readinteger(fd, 1)
-        elif tagid == FT_PARTFILENAME:
-            tags['FT_PARTFILENAME'] = met_readstring(fd)
-        elif tagid == FT_MEDIA_LENGTH:
-            print('Media Length:        {}'.format(met_readinteger(fd)))
-        elif tagid == FT_MEDIA_CODEC:
-            print('Media Codec:         {}'.format(met_readstring(fd)))
-        elif tagid == FT_MEDIA_BITRATE:
-            print('Media Bitrate:       {}'.format(met_readinteger(fd)))
-        elif tagid == FT_MEDIA_TITLE:
-            print('Media Title:         {}'.format(met_readstring(fd)))
-        elif tagid == FT_MEDIA_ARTIST:
-            print('Media Artist:        {}'.format(met_readstring(fd)))
-        elif tagid == FT_MEDIA_ALBUM:
-            print('Media Album:         {}'.format(met_readstring(fd)))
-        elif tagid in FT_STRING:
-            print('... Reading data with tag: {}'.format(tagid))
-            met_readstring(fd)
-        elif tagid in FT_INT:
-            print('... Reading data with tag: {}'.format(tagid))
-            met_readstring(fd)
+
+        if TAG_TYPE[tag_type] is None:
+            print('... Unsupported {} with tag: {}'.format(tag_type, tag_id))
+            continue
+
+        tag_enum = TAG_TYPE[tag_type]
+        if tag_enum is TagString:
+            tags[tag_enum(tag_id).name] = met_read_string(fd)
+        elif tag_enum is TagBlob:
+            tags[tag_enum(tag_id).name] = met_read_blob(fd)
+        elif tag_enum is TagUint32:
+            if tag_id in [0x09, 0x0A]:
+                fd.seek(5, 1)
+                continue
+            tags[tag_enum(tag_id).name] = met_read_integer(fd)
+        elif tag_enum is TagUint8:
+            tags[tag_enum(tag_id).name] = met_read_integer(fd, size=1)
+
     return tags
 
 
 def record_num_parts(fd):
-    return met_readinteger(fd, 2)
+    return met_read_integer(fd, 2)
 
 
 def load_record(fd):
@@ -219,30 +268,9 @@ def load_record(fd):
     for i in range(met_tags['NUMPARTS']):
         partshashes.append(record_read_hash16(fd))
     met_tags['PARTHASHES'] = partshashes
-    met_tags.update(record_tags(fd))
+    met_tags.update(record_read_tags(fd))
 
-    for field in METENTRY_FIELDS:
-        if field not in met_tags:
-            met_tags[field] = None
-    # print(met_tags)
-    metentry = MetEntry(met_tags['FT_FILENAME'],
-                        met_tags['ED2KHASH'],
-                        met_tags['NUMPARTS'],
-                        met_tags['LASTCHANGED'],
-                        met_tags['PARTHASHES'],
-                        met_tags['FT_FILESIZE'],
-                        met_tags['FT_ATTRANSFERRED'],
-                        met_tags['FT_ATREQUESTED'],
-                        met_tags['FT_ATACCEPTED'],
-                        met_tags['FT_ATTRANSFERREDHI'],
-                        met_tags['FT_PARTFILENAME'],
-                        met_tags['FT_AICHHASHSET'],
-                        met_tags['FT_AICH_HASH'],
-                        met_tags['FT_ULPRIORITY'],
-                        met_tags['FT_KADLASTPUBLISHSRC']
-                       )
-    # print(metentry)
-    return metentry
+    return MetEntry(**met_tags)
 
 
 def load_knownfiles(filename):
@@ -257,4 +285,4 @@ def load_knownfiles(filename):
                 yield load_record(fd)
         elif magic == HEADER_PART_MET_FILE:
             # print('>>> [{}] NEW RECORD ***'.format(0))
-            yield load_record(fd)
+            yield load_record(fd)

HashDatabase.py

@@ -19,7 +19,7 @@ def load_hashsets(path, status_callback=None):
         if os.path.isfile(filename):
             hashset = os.path.basename(filename).split('.')[0]
             if status_callback:
-                status_callback('Reading Hashset "{}"...'.format(hashset))
+                status_callback.emit('Reading Hashset "{}"...'.format(hashset))
             hashes =  read_hashset_file(filename)
             hashsets[hashset] = set(hashes)
     return hashsets

MuleAnalyzerUI.py

@@ -11,7 +11,7 @@ from PyQt5 import QtCore, QtGui, QtWidgets
 class Ui_MainWindow(object):
     def setupUi(self, MainWindow):
         MainWindow.setObjectName("MainWindow")
-        MainWindow.resize(532, 361)
+        MainWindow.resize(827, 454)
         MainWindow.setDocumentMode(False)
         self.centralwidget = QtWidgets.QWidget(MainWindow)
         self.centralwidget.setObjectName("centralwidget")
@@ -64,7 +64,7 @@ class Ui_MainWindow(object):
         self.verticalLayout.addLayout(self.verticalLayout_4)
         MainWindow.setCentralWidget(self.centralwidget)
         self.menubar = QtWidgets.QMenuBar(MainWindow)
-        self.menubar.setGeometry(QtCore.QRect(0, 0, 532, 21))
+        self.menubar.setGeometry(QtCore.QRect(0, 0, 827, 21))
         self.menubar.setObjectName("menubar")
         self.menuFile = QtWidgets.QMenu(self.menubar)
         self.menuFile.setObjectName("menuFile")
@@ -76,7 +76,10 @@ class Ui_MainWindow(object):
         self.actionFile_open.setObjectName("actionFile_open")
         self.actionExit = QtWidgets.QAction(MainWindow)
         self.actionExit.setObjectName("actionExit")
+        self.actionExport_as_CSV = QtWidgets.QAction(MainWindow)
+        self.actionExport_as_CSV.setObjectName("actionExport_as_CSV")
         self.menuFile.addAction(self.actionFile_open)
+        self.menuFile.addAction(self.actionExport_as_CSV)
         self.menuFile.addSeparator()
         self.menuFile.addAction(self.actionExit)
         self.menubar.addAction(self.menuFile.menuAction())
@@ -94,4 +97,6 @@ class Ui_MainWindow(object):
         self.actionFile_open.setShortcut(_translate("MainWindow", "Ctrl+O"))
         self.actionExit.setText(_translate("MainWindow", "Exit"))
         self.actionExit.setShortcut(_translate("MainWindow", "Ctrl+Q"))
+        self.actionExport_as_CSV.setText(_translate("MainWindow", "Export as CSV"))
+        self.actionExport_as_CSV.setShortcut(_translate("MainWindow", "Ctrl+E"))
 

main.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>532</width>
-    <height>361</height>
+    <width>827</width>
+    <height>454</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -105,7 +105,7 @@
      <x>0</x>
      <y>0</y>
      <width>532</width>
-     <height>21</height>
+     <height>22</height>
     </rect>
    </property>
    <widget class="QMenu" name="menuFile">
@@ -113,6 +113,7 @@
      <string>File</string>
     </property>
     <addaction name="actionFile_open"/>
+    <addaction name="actionExport_as_CSV"/>
     <addaction name="separator"/>
     <addaction name="actionExit"/>
    </widget>
@@ -135,6 +136,14 @@
     <string>Ctrl+Q</string>
    </property>
   </action>
+  <action name="actionExport_as_CSV">
+   <property name="text">
+    <string>Export as CSV</string>
+   </property>
+   <property name="shortcut">
+    <string>Ctrl+E</string>
+   </property>
+  </action>
  </widget>
  <resources/>
  <connections/>

main.ui.orig

@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>MainWindow</class>
+ <widget class="QMainWindow" name="MainWindow">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>827</width>
+    <height>454</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>eMuleAnalyzer</string>
+  </property>
+  <property name="documentMode">
+   <bool>false</bool>
+  </property>
+  <widget class="QWidget" name="centralwidget">
+   <layout class="QVBoxLayout" name="verticalLayout">
+    <item>
+     <layout class="QVBoxLayout" name="verticalLayout_4">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_2">
+        <item>
+         <widget class="QTableWidget" name="tableWidget">
+          <property name="enabled">
+           <bool>true</bool>
+          </property>
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Expanding" vsizetype="Maximum">
+            <horstretch>0</horstretch>
+            <verstretch>0</verstretch>
+           </sizepolicy>
+          </property>
+          <property name="minimumSize">
+           <size>
+            <width>243</width>
+            <height>100</height>
+           </size>
+          </property>
+          <property name="maximumSize">
+           <size>
+            <width>16777215</width>
+            <height>100</height>
+           </size>
+          </property>
+          <property name="frameShape">
+           <enum>QFrame::Box</enum>
+          </property>
+          <property name="frameShadow">
+           <enum>QFrame::Plain</enum>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="lblSearch">
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+            <horstretch>0</horstretch>
+            <verstretch>0</verstretch>
+           </sizepolicy>
+          </property>
+          <property name="text">
+           <string>Filter:</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLineEdit" name="inputSearch">
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Preferred" vsizetype="Fixed">
+            <horstretch>0</horstretch>
+            <verstretch>0</verstretch>
+           </sizepolicy>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QPushButton" name="btnSearch">
+          <property name="text">
+           <string>Search</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item>
+       <widget class="QTreeView" name="treeView">
+        <property name="frameShape">
+         <enum>QFrame::Box</enum>
+        </property>
+        <property name="frameShadow">
+         <enum>QFrame::Plain</enum>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </item>
+   </layout>
+  </widget>
+  <widget class="QMenuBar" name="menubar">
+   <property name="geometry">
+    <rect>
+     <x>0</x>
+     <y>0</y>
+<<<<<<< HEAD
+     <width>827</width>
+     <height>21</height>
+=======
+     <width>532</width>
+     <height>22</height>
+>>>>>>> f7198fcf8814ec13ea0e3f1c9d9064648d4a427e
+    </rect>
+   </property>
+   <widget class="QMenu" name="menuFile">
+    <property name="title">
+     <string>File</string>
+    </property>
+    <addaction name="actionFile_open"/>
+    <addaction name="actionExport_as_CSV"/>
+    <addaction name="separator"/>
+    <addaction name="actionExit"/>
+   </widget>
+   <addaction name="menuFile"/>
+  </widget>
+  <widget class="QStatusBar" name="statusBar"/>
+  <action name="actionFile_open">
+   <property name="text">
+    <string>File open</string>
+   </property>
+   <property name="shortcut">
+    <string>Ctrl+O</string>
+   </property>
+  </action>
+  <action name="actionExit">
+   <property name="text">
+    <string>Exit</string>
+   </property>
+   <property name="shortcut">
+    <string>Ctrl+Q</string>
+   </property>
+  </action>
+  <action name="actionExport_as_CSV">
+   <property name="text">
+    <string>Export as CSV</string>
+   </property>
+   <property name="shortcut">
+    <string>Ctrl+E</string>
+   </property>
+  </action>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

pyMuleAnalyzerQT.py

@@ -3,11 +3,17 @@
 """
 
 import sys
+import csv
+import os
+import traceback
+import time
+import random
+
 import PyQt5
-from PyQt5.QtCore import Qt, QDateTime, QObject, pyqtSignal
+from PyQt5.QtCore import Qt, QDateTime, QObject, pyqtSignal, pyqtSlot, QRunnable, QThreadPool
 from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog, QAbstractItemView, \
     QTableWidgetItem, QSplashScreen, QProgressBar
-from PyQt5.QtGui import QStandardItemModel, QPixmap
+from PyQt5.QtGui import QStandardItemModel, QPixmap, QMovie, QPainter
 from MuleAnalyzerUI import Ui_MainWindow
 
 from ED2K import load_knownfiles
@@ -30,6 +36,63 @@ sys._excepthook = sys.excepthook
 sys.excepthook = exceptionHandler.handler
 
 
+class MovieSplashScreen(QSplashScreen):
+
+    def __init__(self, movie, parent=None):
+        movie.jumpToFrame(0)
+        pixmap = QPixmap(movie.frameRect().size())
+        QSplashScreen.__init__(self, pixmap, Qt.WindowStaysOnTopHint)
+        self.movie = movie
+        self.movie.frameChanged.connect(self.repaint)
+
+    def showEvent(self, event):
+        self.movie.start()
+
+    def hideEvent(self, event):
+        self.movie.stop()
+
+    def paintEvent(self, event):
+        painter = QPainter(self)
+        pixmap = self.movie.currentPixmap()
+        self.setMask(pixmap.mask())
+        painter.drawPixmap(0, 0, pixmap)
+
+    def sizeHint(self):
+        return self.movie.scaledSize()
+
+
+class WorkerSignals(QObject):
+    finished = pyqtSignal()
+    error = pyqtSignal(tuple)
+    result = pyqtSignal(object)
+    progress = pyqtSignal(str)
+
+
+class Worker(QRunnable):
+
+    def __init__(self, fn, *args, **kwargs):
+        QRunnable.__init__(self)
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+        self.signals = WorkerSignals()
+        kwargs['progress_callback'] = self.signals.progress
+
+    @pyqtSlot()
+    def run(self):
+        # Retrieve args/kwargs here; and fire processing using them
+        try:
+            result = self.fn(*self.args, **self.kwargs)
+        except:
+            traceback.print_exc()
+            exctype, value = sys.exc_info()[:2]
+            self.signals.error.emit((exctype, value, traceback.format_exc()))
+        else:
+            self.signals.result.emit(result)
+        finally:
+            self.signals.finished.emit()
+
+
 class MainWindow(QMainWindow, Ui_MainWindow):
 
     # Todo: Replace TreeHeaders
@@ -51,8 +114,7 @@ class MainWindow(QMainWindow, Ui_MainWindow):
         ('Parts Hash', 'PARTHASHES', list, '', Qt.AlignLeft),
     )
 
-    def __init__(self):
-        self.is_loading = False
+    def __init__(self, *args, **kwargs):
         super(MainWindow, self).__init__()
         self.setupUi(self)
 
@@ -66,18 +128,36 @@ class MainWindow(QMainWindow, Ui_MainWindow):
 
         self.tableWidget.setEditTriggers(QAbstractItemView.NoEditTriggers)
 
-        self.actionFile_open.setShortcut('Ctrl+O')
-        self.actionFile_open.triggered.connect(self.slot_load_met_file)
+        self.actionFile_open.triggered.connect(self.action_load_met_file)
+        self.actionExport_as_CSV.triggered.connect(self.action_export_data)
         self.actionExit.triggered.connect(self.close)
 
-        # Init functionality and other features
-        self.hashsets = load_hashsets('Hashsets', self.cb_print_msg)
-        self.is_loading = True
+        self.threadpool = QThreadPool()
+        self.hashsets = None
+        self.splash_screen = kwargs['splash_screen']
+        worker = Worker(self.load_hashsets)
+        worker.signals.result.connect(self.cb_got_hashset)
+        worker.signals.progress.connect(self.cb_progress)
+        self.threadpool.start(worker)
+
+    def cb_progress(self, msg):
+        self.splash_screen.showMessage('<h1>%s</h1>' % msg, Qt.AlignTop | Qt.AlignCenter, Qt.black)
+
+    def load_hashsets(self, progress_callback, folder='Hashsets'):
+        self.hashsets = load_hashsets(folder, progress_callback)
+        return self.hashsets
 
     def cb_print_msg(self, message):
-        PyQt5.QtWidgets.QApplication.processEvents()
         print('>>>', message)
 
+    def cb_got_hashset(self, hashes):
+        num_hashsets = len(hashes.keys())
+        num_hashes = 0
+        for key in hashes:
+            num_hashes += len(hashes[key])
+        self.statusBar.showMessage('Loaded {} hashsets with {:,} hashes.'.format(num_hashsets,
+                                                                                 num_hashes))
+
     def createED2KModel(self, parent):
         model = QStandardItemModel(0, len(self.Headers), parent)
         for i, fields in enumerate(self.Headers):
@@ -91,7 +171,7 @@ class MainWindow(QMainWindow, Ui_MainWindow):
                 if i == 0:
                     root = self.treeModel.invisibleRootItem()
                     row_count = root.rowCount()
-                    model.setData(model.index(0, i), row_count+1)
+                    model.setData(model.index(0, i), row_count)
                 if i == 3:
                     found_in = []
                     for hashset, hashes in self.hashsets.items():
@@ -103,8 +183,11 @@ class MainWindow(QMainWindow, Ui_MainWindow):
             else:
                 value = getattr(metinfo, field[1]) or field[3]
                 if field[2] is QDateTime:
-                    last_changed = QDateTime.fromSecsSinceEpoch(value, 0)
-                    model.setData(model.index(0, i), last_changed.toString('dd.MM.yyyy hh:mm:ss'))
+                    try:
+                        last_changed = QDateTime.fromSecsSinceEpoch(value, 0)
+                        model.setData(model.index(0, i), last_changed.toString('dd.MM.yyyy hh:mm:ss'))
+                    except TypeError:
+                        model.setData(model.index(0, i), '')
                 elif field[2] is int:
                     model.setData(model.index(0, i), '{0:,}'.format(value))
                 elif field[2] is list:
@@ -117,10 +200,39 @@ class MainWindow(QMainWindow, Ui_MainWindow):
         for met_entry in load_knownfiles(filename):
             self.addED2K(self.treeModel, met_entry)
 
-    def slot_load_met_file(self):
+    def action_export_data(self):
         options = QFileDialog.Options()
         options |= QFileDialog.DontUseNativeDialog
-        files, _ = QFileDialog.getOpenFileNames(self, 'Open .met files...', '', 'MET Files (*.met);;All Files (*)',
+        options |= QFileDialog.ReadOnly
+        filename, _ = QFileDialog.getSaveFileName(self, 'Export list as CSV', '',
+                                                  'CSV Files (*.csv);;All Files (*)',
+                                                  options=options)
+        if not filename.lower().endswith('.csv'):
+            filename += '.csv'
+        with open(filename, 'w+', newline='', encoding='utf-8') as csvfile:
+            metwriter = csv.DictWriter(csvfile, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL,
+                                       fieldnames=[x[0] for x in self.Headers])
+            metwriter.writeheader()
+
+            root = self.treeModel.invisibleRootItem()
+            row_count = root.rowCount()
+            for row_index in range(row_count):
+                data = {}
+                for column_index, column in enumerate(self.Headers):
+                    row = self.treeModel.item(row_index, column_index).text()
+                    data[column[0]] = row
+                metwriter.writerow(data)
+
+    def action_load_met_file(self):
+        # clear view before opening a new one
+        self.treeModel.removeRows(0, self.treeModel.rowCount())
+        self.tableWidget.setItem(0, 0, QTableWidgetItem(''))
+        self.tableWidget.setItem(0, 1, QTableWidgetItem(''))
+
+        options = QFileDialog.Options()
+        options |= QFileDialog.DontUseNativeDialog
+        files, _ = QFileDialog.getOpenFileNames(self, 'Open .met files...', '',
+                                                'MET Files (*.met);;All Files (*)',
                                                 options=options)
         for filename in files:
             self.loadMetFile(filename)
@@ -140,15 +252,20 @@ class MainWindow(QMainWindow, Ui_MainWindow):
 def main(argv):
     app = QApplication(argv)
 
-    splash_pix = QPixmap('splash-screen.jpg')
-    splash_screen = QSplashScreen(splash_pix, Qt.WindowStaysOnTopHint)
-    splash_screen.setMask(splash_pix.mask())
-
+    resources = './resources/loaders/'
+    movies = []
+    for filename in os.listdir(resources):
+        filepath = os.path.join(resources, filename)
+        if os.path.isfile(filepath):
+            movies.append(filepath)
+    movie = QMovie(random.choice(movies))
+    splash_screen = MovieSplashScreen(movie)
+    splash_screen.setEnabled(False)
     splash_screen.show()
-    splash_screen.showMessage('<h1>EY // eMuleAnalyzer</h1>', Qt.AlignTop | Qt.AlignCenter, Qt.black)
 
-    app.processEvents()
-    window = MainWindow()
+    window = MainWindow(splash_screen=splash_screen)
+    while not window.hashsets:
+        app.processEvents()
     window.show()
     splash_screen.finish(window)
 

pyMuleAnalyzerQT.py.orig

@@ -0,0 +1,275 @@
+#! /usr/bin/env python3
+"""
+"""
+
+import sys
+<<<<<<< HEAD
+import csv
+=======
+import os
+import traceback
+import time
+import random
+
+>>>>>>> f7198fcf8814ec13ea0e3f1c9d9064648d4a427e
+import PyQt5
+from PyQt5.QtCore import Qt, QDateTime, QObject, pyqtSignal, pyqtSlot, QRunnable, QThreadPool
+from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog, QAbstractItemView, \
+    QTableWidgetItem, QSplashScreen, QProgressBar
+from PyQt5.QtGui import QStandardItemModel, QPixmap, QMovie, QPainter
+from MuleAnalyzerUI import Ui_MainWindow
+
+from ED2K import load_knownfiles
+from HashDatabase import load_hashsets
+
+
+class ExceptionHandler(QObject):
+    errorSignal = pyqtSignal()
+
+    def __init__(self):
+        super(ExceptionHandler, self).__init__()
+
+    def handler(self, exctype, value, traceback):
+        self.errorSignal.emit()
+        sys._excepthook(exctype, value, traceback)
+
+
+exceptionHandler = ExceptionHandler()
+sys._excepthook = sys.excepthook
+sys.excepthook = exceptionHandler.handler
+
+
+class MovieSplashScreen(QSplashScreen):
+
+    def __init__(self, movie, parent=None):
+        movie.jumpToFrame(0)
+        pixmap = QPixmap(movie.frameRect().size())
+        QSplashScreen.__init__(self, pixmap, Qt.WindowStaysOnTopHint)
+        self.movie = movie
+        self.movie.frameChanged.connect(self.repaint)
+
+    def showEvent(self, event):
+        self.movie.start()
+
+    def hideEvent(self, event):
+        self.movie.stop()
+
+    def paintEvent(self, event):
+        painter = QPainter(self)
+        pixmap = self.movie.currentPixmap()
+        self.setMask(pixmap.mask())
+        painter.drawPixmap(0, 0, pixmap)
+
+    def sizeHint(self):
+        return self.movie.scaledSize()
+
+
+class WorkerSignals(QObject):
+    finished = pyqtSignal()
+    error = pyqtSignal(tuple)
+    result = pyqtSignal(object)
+    progress = pyqtSignal(str)
+
+
+class Worker(QRunnable):
+
+    def __init__(self, fn, *args, **kwargs):
+        QRunnable.__init__(self)
+        self.fn = fn
+        self.args = args
+        self.kwargs = kwargs
+        self.signals = WorkerSignals()
+        kwargs['progress_callback'] = self.signals.progress
+
+    @pyqtSlot()
+    def run(self):
+        # Retrieve args/kwargs here; and fire processing using them
+        try:
+            result = self.fn(*self.args, **self.kwargs)
+        except:
+            traceback.print_exc()
+            exctype, value = sys.exc_info()[:2]
+            self.signals.error.emit((exctype, value, traceback.format_exc()))
+        else:
+            self.signals.result.emit(result)
+        finally:
+            self.signals.finished.emit()
+
+
+class MainWindow(QMainWindow, Ui_MainWindow):
+
+    # Todo: Replace TreeHeaders
+    Headers = (
+        ('ID', None, None, 0, Qt.AlignRight),
+        ('Filename', 'FT_FILENAME', str, '', Qt.AlignLeft),
+        ('ED2K Hash', 'ED2KHASH', str, '', Qt.AlignLeft),
+        ('Hashset', None, None, '', Qt.AlignLeft),
+        ('Last Changed', 'LASTCHANGED', QDateTime, 'Never', Qt.AlignLeft),
+        ('Filesize', 'FT_FILESIZE', int, 0, Qt.AlignRight),
+        ('Transfered', 'FT_ATTRANSFERRED', int, 0, Qt.AlignRight),
+        ('Transfered (HI)', 'FT_ATTRANSFERREDHI', int, 0, Qt.AlignRight),
+        ('Requested', 'FT_ATREQUESTED', int, 0, Qt.AlignRight),
+        ('Accepted', 'FT_ATACCEPTED', int, 0, Qt.AlignRight),
+        ('AICHashes', 'FT_AICHHASHSET', str, '', Qt.AlignLeft),
+        ('AICHash', 'FT_AICH_HASH', str, '', Qt.AlignLeft),
+        ('Part Filename', 'FT_PARTFILENAME', str, '', Qt.AlignLeft),
+        ('Number of Parts', 'NUMPARTS', int, 0, Qt.AlignRight),
+        ('Parts Hash', 'PARTHASHES', list, '', Qt.AlignLeft),
+    )
+
+    def __init__(self, *args, **kwargs):
+        super(MainWindow, self).__init__()
+        self.setupUi(self)
+
+        self.treeModel = self.createED2KModel(self)
+        self.treeView.setModel(self.treeModel)
+        self.treeView.setRootIsDecorated(False)
+        self.treeView.setAlternatingRowColors(True)
+        self.treeView.setSortingEnabled(True)
+        self.treeView.setEditTriggers(QAbstractItemView.NoEditTriggers)
+        #self.treeView.setSelectionMode(QAbstractItemView.MultiSelection)
+
+        self.tableWidget.setEditTriggers(QAbstractItemView.NoEditTriggers)
+
+        self.actionFile_open.triggered.connect(self.action_load_met_file)
+        self.actionExport_as_CSV.triggered.connect(self.action_export_data)
+        self.actionExit.triggered.connect(self.close)
+
+        self.threadpool = QThreadPool()
+        self.hashsets = None
+        self.splash_screen = kwargs['splash_screen']
+        worker = Worker(self.load_hashsets)
+        worker.signals.result.connect(self.cb_got_hashset)
+        worker.signals.progress.connect(self.cb_progress)
+        self.threadpool.start(worker)
+
+    def cb_progress(self, msg):
+        self.splash_screen.showMessage('<h1>%s</h1>' % msg, Qt.AlignTop | Qt.AlignCenter, Qt.black)
+
+    def load_hashsets(self, progress_callback, folder='Hashsets'):
+        self.hashsets = load_hashsets(folder, progress_callback)
+        return self.hashsets
+
+    def cb_print_msg(self, message):
+        print('>>>', message)
+
+    def cb_got_hashset(self, hashes):
+        num_hashsets = len(hashes.keys())
+        num_hashes = 0
+        for key in hashes:
+            num_hashes += len(hashes[key])
+        self.statusBar.showMessage('Loaded {} hashsets with {:,} hashes.'.format(num_hashsets,
+                                                                                 num_hashes))
+
+    def createED2KModel(self, parent):
+        model = QStandardItemModel(0, len(self.Headers), parent)
+        for i, fields in enumerate(self.Headers):
+            model.setHeaderData(i, Qt.Horizontal, fields[0])
+        return model
+
+    def addED2K(self, model, metinfo):
+        model.insertRow(0)
+        for i, field in enumerate(self.Headers):
+            if field[2] == None:
+                if i == 0:
+                    root = self.treeModel.invisibleRootItem()
+                    row_count = root.rowCount()
+                    model.setData(model.index(0, i), row_count)
+                if i == 3:
+                    found_in = []
+                    for hashset, hashes in self.hashsets.items():
+                        if metinfo.ED2KHASH.upper() in hashes or metinfo.ED2KHASH.lower() in hashes:
+                            found_in.append(hashset)
+                            break
+                    # TODO Hash lookup
+                    model.setData(model.index(0, i), ', '.join(found_in))
+            else:
+                value = getattr(metinfo, field[1]) or field[3]
+                if field[2] is QDateTime:
+                    last_changed = QDateTime.fromSecsSinceEpoch(value, 0)
+                    model.setData(model.index(0, i), last_changed.toString('dd.MM.yyyy hh:mm:ss'))
+                elif field[2] is int:
+                    model.setData(model.index(0, i), '{0:,}'.format(value))
+                elif field[2] is list:
+                    model.setData(model.index(0, i), ''.join(value))
+                else:
+                    model.setData(model.index(0, i), value)
+            model.item(0, i).setTextAlignment(field[4])
+
+    def loadMetFile(self, filename):
+        for met_entry in load_knownfiles(filename):
+            self.addED2K(self.treeModel, met_entry)
+
+<<<<<<< HEAD
+    def action_export_data(self):
+        options = QFileDialog.Options()
+        options |= QFileDialog.DontUseNativeDialog
+        options |= QFileDialog.ReadOnly
+        filename, _ = QFileDialog.getSaveFileName(self, 'Export list as CSV', '', 'CSV Files (*.csv);;All Files (*)',
+                                                  options=options)
+        if not filename.lower().endswith('.csv'):
+            filename += '.csv'
+        with open(filename, 'w+', newline='', encoding='utf-8') as csvfile:
+            metwriter = csv.DictWriter(csvfile, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL,
+                                       fieldnames=[x[0] for x in self.Headers])
+            metwriter.writeheader()
+
+            root = self.treeModel.invisibleRootItem()
+            row_count = root.rowCount()
+            for row_index in range(row_count):
+                data = {}
+                for column_index, column in enumerate(self.Headers):
+                    row = self.treeModel.item(row_index, column_index).text()
+                    data[column[0]] = row
+                metwriter.writerow(data)
+
+    def action_load_met_file(self):
+=======
+    @pyqtSlot()
+    def slot_load_met_file(self):
+>>>>>>> f7198fcf8814ec13ea0e3f1c9d9064648d4a427e
+        options = QFileDialog.Options()
+        options |= QFileDialog.DontUseNativeDialog
+        files, _ = QFileDialog.getOpenFileNames(self, 'Open .met files...', '',
+                                                'MET Files (*.met);;All Files (*)',
+                                                options=options)
+        for filename in files:
+            self.loadMetFile(filename)
+
+        root = self.treeModel.invisibleRootItem()
+        row_count = root.rowCount()
+        transfered = 0
+        for i in range(row_count):
+            row = self.treeModel.item(i, column=5)
+            transfered += int(row.text().replace(',', ''))
+        self.tableWidget.setRowCount(1)
+        self.tableWidget.setColumnCount(2)
+        self.tableWidget.setItem(0, 0, QTableWidgetItem('Transfered overall:'))
+        self.tableWidget.setItem(0, 1, QTableWidgetItem(str('{0:,}'.format(transfered))))
+
+
+def main(argv):
+    app = QApplication(argv)
+
+    resources = './resources/loaders/'
+    movies = []
+    for filename in os.listdir(resources):
+        filepath = os.path.join(resources, filename)
+        if os.path.isfile(filepath):
+            movies.append(filepath)
+    movie = QMovie(random.choice(movies))
+    splash_screen = MovieSplashScreen(movie)
+    splash_screen.setEnabled(False)
+    splash_screen.show()
+
+    window = MainWindow(splash_screen=splash_screen)
+    while not window.hashsets:
+        app.processEvents()
+    window.show()
+    splash_screen.finish(window)
+
+    sys.exit(app.exec_())
+
+
+if __name__ == '__main__':
+    main(sys.argv)

tests.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+
+from ED2K import *
+
+
+knownmet = load_knownfiles('D:\EXPORTS\JELLY DONUT\Ex\Programme\eMule.de\config\known.met')
+for entry in knownmet:
+    print(entry)