#54 - changed PE_Info to File_info. Manage more filetypes

analyzers/FILE_Info/lib/File_analysis.py

@@ -0,0 +1,155 @@
+#!/usr/bin/python
+
+import sys
+import os
+import json
+import pefile
+import hashlib
+import pydeep
+import magic
+import pyexifinfo
+import re
+import pehashng
+
+
+
+
+class file:
+
+    def __init__(self, filepath):
+        self.path = filepath
+        self.stream = open(filepath, 'r').read()
+
+        if magic.Magic(mimetype=True).from_file(filepath) == 'application/x-dosexec':
+            try:
+                self.pe = pefile.PE(filepath)
+                self.pedict = self.pe.dump_dict()
+            except Exception as excp:
+                print('Failed processing %s') % filepath
+
+
+    # Magic
+    def magic(self):
+        return magic.Magic().from_file(self.path)
+
+    def mimetype(self):
+        return magic.Magic(mimetype=True).from_file(self.path)
+    # ExifTool
+    def exif(self):
+        exifreport=pyexifinfo.get_json(self.path)
+        result=dict((key,value) for key,value in exifreport[0].iteritems() if not (key.startswith("File") or key.startswith("SourceFile")))
+        return result
+
+    # File hash
+    def md5(self):
+        return hashlib.md5(self.stream).hexdigest();
+
+    def sha1(self):
+        return hashlib.sha1(self.stream).hexdigest();
+
+    def sha256(self):
+        return hashlib.sha256(self.stream).hexdigest();
+
+    def ssdeep(self):
+        return pydeep.hash_file(self.path)
+
+    # PE: impash
+    def imphash(self):
+        return self.pe.get_imphash()
+
+    # PE: pehash
+    def pehash(self):
+        if self.pe:
+            return  pehashng.pehashng(self.pe)
+
+    # Fileinfo
+
+    def filesize(self):
+        return os.path.getsize(self.path)
+
+    def info(self):
+        table=[]
+        try:
+            for fileinfo in self.pe.FileInfo:
+                if fileinfo.Key == 'StringFileInfo':
+                    for stringtable in fileinfo.StringTable:
+                        for entry in stringtable.entries.items():
+                            table.append({'Info':entry[0], 'Value':entry[1]})
+            return table
+        except Exception as excp:
+            return 'None'
+
+    # PE: type
+    def PEtype(self):
+
+        if self.pe and self.pe.is_dll():
+            return "DLL"
+        if self.pe and self.pe.is_driver():
+            return "DRIVER"
+        if self.pe and self.pe.is_exe():
+            return "EXE"
+
+    # PE:  Timestamp
+    def CompilationTimestamp(self):
+        if self.pe:
+            return self.pedict['FILE_HEADER']['TimeDateStamp']['Value']
+
+    # PE: OS Version
+    def OperatingSystem(self):
+        if self.pe:
+            return str(self.pedict['OPTIONAL_HEADER']['MajorOperatingSystemVersion']['Value']) + "." \
+               + str(self.pedict['OPTIONAL_HEADER']['MinorOperatingSystemVersion']['Value'])
+
+    # PE:Machine type
+    def Machine(self):
+        if self.pe:
+            machinetype = self.pedict['FILE_HEADER']['Machine']['Value']
+            mt = {'0x014c': 'x86', '0x0200': 'Itanium', '0x8664': 'x64'}
+            return mt[str(hex(machinetype))] if type(machinetype) is int else str(machinetype) + ' => Not x86/64 or Itanium'
+
+    # PE:Entry Point
+    def EntryPoint(self):
+        if self.pe:
+            return hex(self.pedict['OPTIONAL_HEADER']['AddressOfEntryPoint']['Value'])
+
+    # PE:IAT list of {'entryname':'name', 'symbols':[list of symbols]}
+    def iat(self):
+        if self.pe:
+            table = []
+            for entry in self.pe.DIRECTORY_ENTRY_IMPORT:
+                imp = {'entryname': '', 'symbols': []}
+                imp['entryname']=entry.dll
+                for symbol in entry.imports:
+                    imp['symbols'].append(symbol.name)
+                table.append(imp)
+            return table
+
+    # PE Resources : WORK IN PROGRESS
+    def resources(self):
+        for rsrc in self.pe.DIRECTORY_ENTRY_RESOURCE.entries:
+            for entry in rsrc.directory.entries:
+                print entry.name.__str__()
+                for i in entry.directory.entries:
+                    print i.data.lang
+                    print i.data.sublang
+
+    # PE:Sections list of {Name, Size, Entropy, MD5, SHA1, SHA256, SHA512}
+    def sections(self):
+        if self.pe:
+            table = []
+            for entry in self.pe.sections:
+                sect = {'entryname':str(entry.Name),'SizeOfRawData':hex(entry.SizeOfRawData),
+                        'Entropy':entry.get_entropy(),
+                        'MD5':entry.get_hash_md5(),
+                        'SHA1':entry.get_hash_sha1(),
+                        'SHA256':entry.get_hash_sha256(),
+                        'SHA512':entry.get_hash_sha512()}
+                table.append(sect)
+                sect = {}
+            return table
+
+
+    # PE :Return dump_dict() for debug only
+    def dump(self):
+        if self.pe:
+            return self.pedict

analyzers/PE_Info/report/success_long.html → analyzers/FILE_Info/report/success_long.html

@@ -1,6 +1,6 @@
-<div class="report-PEInfo">
+<div class="report-FILEInfo">
   <style>
-    .report-PEInfo dl {
+    .report-FILEInfo dl {
         margin-bottom:2px;
     }
 
@@ -23,26 +23,26 @@
           <dl class="dl-horizontal">
               <dt>SHA256</dt>
               <dd class="wrap">{{content.Identification['SHA256']}}</dd>
-          <dl class="dl-horizontal">
+          <dl class="dl-horizontal" ng-if={{content.Identification['impash']}}>
               <dt>impash</dt>
               <dd class="wrap">{{content.Identification['impash']|| "-"}} </dd>
           </dl>
-          <dl class="dl-horizontal">
+          <dl class="dl-horizontal" ng-if={{content.Identification['ssdeep']}}>
               <dt>ssdeep</dt>
               <dd class="wrap">{{content.Identification['ssdeep']|| "-"}}  </dd>
           </dl>
-          <dl class="dl-horizontal">
+          <dl class="dl-horizontal" ng-if={{content.Identification['pehash']}}>
               <dt>pehash</dt>
               <dd class="wrap">{{content.Identification['pehash']|| "-"}}  </dd>
           </dl>
-          <dl class="dl-horizontal">
+          <dl class="dl-horizontal" ng-if={{content.Identification['OperatingSystem']}}>
               <dt>Operating System</dt>
               <dd class="wrap">{{content.Identification['OperatingSystem']}}
               </dd>
           </dl>
           <dl class="dl-horizontal">
               <dt>PE Type</dt>
-              <dd class="wrap">{{content.Identification['Type']}}</dd>
+              <dd class="wrap">{{content.Identification['PEType']}}</dd>
           </dl>
           <dl class="dl-horizontal">
               <dt>Magic literal</dt>

analyzers/FILE_Info/run.py

@@ -0,0 +1,157 @@
+#!/usr/local/bin/python
+# encoding: utf-8
+
+# using https://github.com/erocarrera/pefile
+
+
+import sys
+import re
+import os
+import json
+import codecs
+import pefile
+import magic
+import pyexifinfo
+from  lib.File_analysis import file
+from StringIO import StringIO
+
+
+if sys.stdout.encoding != 'UTF-8':
+    if sys.version_info.major == 3:
+        sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
+    else:
+        sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict')
+if sys.stderr.encoding != 'UTF-8':
+    if sys.version_info.major == 3:
+        sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
+    else:
+        sys.stderr = codecs.getwriter('utf-8')(sys.stderr, 'strict')
+
+# load observable
+artifact = json.load(sys.stdin)
+
+def get_param(name, default=None, message=None, current=artifact):
+    if isinstance(name, str):
+        name = name.split('.')
+    if len(name) == 0:
+        return current
+    else:
+        value = current.get(name[0])
+        if value == None:
+            if message != None:
+                print(message)
+                sys.exit(1)
+            else:
+                return default
+        else:
+            return get_param(name[1:], default, message, value)
+
+filename    = get_param('attachmentName', 'noname.ext')
+filepath    = get_param('file', None, 'File is missing')
+max_tlp     = get_param('config.max_tlp', 10)
+tlp         = get_param('tlp', 2)  # amber by default
+
+
+# run  only if TLP condition is met
+if tlp > max_tlp:
+    error('Error with TLP value ; see max_tlp in config or tlp value in input data')
+
+result={}
+
+def error(message):
+    res={}
+    res['errorMessage']="{}".format(message)
+    json_data = json.dumps(res,indent=4)
+    print json_data
+    sys.exit(1)
+
+try:
+    __import__('imp').find_module('magic')
+except ImportError:
+    error('Import Error: Module magic not found')
+
+try:
+    __import__('imp').find_module('pefile')
+except ImportError:
+    error('Import Error: Module pefile not found')
+
+try:
+    __import__('imp').find_module('hashlib')
+except ImportError:
+    error('Import Error: Module hashlib not found')
+
+try:
+    __import__('imp').find_module('pydeep')
+except ImportError:
+    error('Import Error: Module pydeep not found')
+
+try:
+    __import__('imp').find_module('pyexifinfo')
+except ImportError:
+    error('Import Error: Module pyexifinfo not found')
+
+# set stderr back to original __stderr__
+sys.stderr = sys.__stderr__
+
+
+# Associate Mimetype and analyzer (every result has a result['Mimetype']
+def FileInfo():
+    f = file(filepath)
+    try:
+        result['Mimetype'] = f.mimetype()
+    except Exception as excp:
+        error(str(excp))
+    result['Exif'] = f.exif()
+    result['Magic']= f.magic()
+    result['Identification']= {'MD5': f.md5(),
+                            'SHA1':f.sha1(),
+                            'SHA256':f.sha256(),
+                            'ssdeep':f.ssdeep()}
+
+
+# PE_Info analyzer
+def PE_info():
+    f = file(filepath)
+    result['Identification']= {'impash':f.imphash(),
+                            'ssdeep':f.ssdeep(),
+                            'pehash':f.pehash(),
+                            'OperatingSystem':f.OperatingSystem(),
+                            'Type':f.PEtype()}
+
+    result['BasicInformation'] = {'FileInfo':f.info(),
+                              'FileSize': f.filesize(),
+                              'TargetMachine' : f.Machine(),
+                              'CompilationTimestamp' : f.CompilationTimestamp(),
+                              'EntryPoint':f.EntryPoint()}
+
+    result['Sections'] = f.sections()
+    result['ImportAdressTable'] = f.iat()
+
+
+
+def SpecificInfo():
+    if result['Mimetype'] in ['application/x-dosexec']:
+        PE_info()
+
+
+
+def dump_result(res):
+    if type(res['Mimetype']):
+        # if res['Mimetype'] == 'application/x-dosexec':
+        #     json.dump(res, sys.stdout, sort_keys=True,ensure_ascii=False, indent=4)
+        json.dump(res, sys.stdout, sort_keys=True,ensure_ascii=False, indent=4)
+
+    else:
+        error(res)
+
+
+
+
+
+def main():
+    FileInfo()
+    SpecificInfo()
+    dump_result(result)
+
+if __name__ == '__main__':
+    main()

analyzers/PE_Info_1.0.json → analyzers/FILE_Info_1.0.json

@@ -6,5 +6,5 @@
 	"dataTypeList": ["file"],
 	"baseConfig" : "PE_Info",
 	"config": {"max_tlp":"3"},
-	"command": "PE_Info/PE_Info.py"
+	"command": "FILE_Info/run.py"
 }