Merge pull request #915 from TheHive-Project/feature/fileinfo_update

Update Fileinfo

analyzers/FileInfo/Dockerfile

@@ -1,21 +1,24 @@
-FROM python:3
+FROM python:3.7
 
 WORKDIR /worker
 COPY . FileInfo
 RUN apt update
-RUN apt install -y -q libboost-regex-dev                           \
- libboost-program-options-dev                                      \
- libboost-system-dev libboost-filesystem-dev libssl-dev            \
- build-essential cmake                                             \
- git 
-RUN git clone https://github.com/JusticeRage/Manalyze.git       && \
-cd Manalyze                                                     && \
-cmake .                                                         && \
-make -j5                                                        && \
-cd bin/yara_rules                                               && \ 
-python2 update_clamav_signatures.py
+RUN apt install -y -q libboost-regex-dev                              \
+    libboost-program-options-dev                                      \
+    libboost-system-dev libboost-filesystem-dev libssl-dev            \
+    build-essential cmake                                             \
+    git 
+RUN git clone https://github.com/JusticeRage/Manalyze.git           && \
+    cd Manalyze                                                     && \
+    cmake .                                                         && \
+    make -j5                                                        && \
+    cd bin/yara_rules                                               && \ 
+    python2 update_clamav_signatures.py
 RUN apt update                                                  && \
     apt install -y -q libfuzzy-dev libimage-exiftool-perl       && \
     rm -rf /var/lib/apt/lists/*                                 && \
     pip install --no-cache-dir -r FileInfo/requirements.txt
+
+RUN curl -SL https://github.com/fireeye/flare-floss/releases/download/v1.7.0/floss-v1.7.0-linux.zip  --output floss.zip && \
+    unzip floss.zip -d /usr/bin
 ENTRYPOINT FileInfo/fileinfo_analyzer.py

analyzers/FileInfo/FileInfo.json

@@ -1,6 +1,6 @@
 {
   "name": "FileInfo",
-  "version": "7.0",
+  "version": "8.0",
   "author": "TheHive-Project",
   "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
   "license": "AGPL-V3",
@@ -40,7 +40,30 @@
       "required": false,
       "multi": false,
       "defaultValue": "/worker/Manalyze/bin/manalyze"
+    },
+    {
+      "name": "floss_enable",
+      "description": "Enable the use of FireEye FLARE FLOSS",
+      "type": "boolean",
+      "required": false,
+      "multi": false,
+      "default": false
+    },
+    {
+      "name": "floss_binary_path",
+      "description": "Path to the FLOSS binary.",
+      "type": "string",
+      "required": false,
+      "multi": false,
+      "default": "/usr/bin/floss"
+    },
+    {
+      "name": "floss_minimal_string_length",
+      "description": "Length of strings must be in order to be considered.",
+      "type": "number",
+      "required": false,
+      "multi": false,
+      "default": 4
     }
   ]
 }
-

analyzers/FileInfo/README.txt

@@ -0,0 +1,7 @@
+For local install:
+
+for ssdeep:
+`apt install libffi-dev libfuzzy-dev libfuzzy2`
+
+for pyexifinfo:
+`apt install exiftool`

analyzers/FileInfo/fileinfo_analyzer.py

@@ -7,70 +7,122 @@
 from submodules import available_submodules
 from submodules.submodule_metadata import MetadataSubmodule
 from submodules.submodule_manalyze import ManalyzeSubmodule
+from submodules.submodule_floss import FlossSubmodule
 
 
 class FileInfoAnalyzer(Analyzer):
     def __init__(self):
         Analyzer.__init__(self)
-        self.filepath = self.get_param('file', None, 'File parameter is missing.')
-        self.filename = self.get_param('filename', None, 'Filename is missing.')
+        self.filepath = self.get_param("file", None, "File parameter is missing.")
+        self.filename = self.get_param("filename", None, "Filename is missing.")
         self.filetype = pyexifinfo.fileType(self.filepath)
         self.mimetype = magic.Magic(mime=True).from_file(self.filepath)
+        self.auto_extract_artifacts = self.get_param("auto_extract_artifacts", None)
+        self.observables = []
 
         # Check if manalyze submodule is enabled
-        if self.get_param('config.manalyze_enable', False, 'Parameter manalyze_enable not given.'
-                                                           'Please enable or disable manalyze submodule explicitly.'):
-            binary_path = self.get_param('config.manalyze_binary_path',
-                                         '/opt/Cortex-Analyzers/utils/manalyze/bin/manalyze')
-            if self.get_param('config.manalyze_enable_docker', False):
+        if self.get_param(
+            "config.manalyze_enable",
+            False,
+            "Parameter manalyze_enable not given."
+            "Please enable or disable manalyze submodule explicitly.",
+        ):
+            binary_path = self.get_param(
+                "config.manalyze_binary_path",
+                "/opt/Cortex-Analyzers/utils/manalyze/bin/manalyze",
+            )
+            if self.get_param("config.manalyze_enable_docker", False):
+                available_submodules.append(ManalyzeSubmodule(use_docker=True))
+            elif self.get_param(
+                "config.manalyze_enable_binary", False
+            ) and os.path.isfile(binary_path):
                 available_submodules.append(
-                    ManalyzeSubmodule(
-                        use_docker=True
-                    )
-                )
-            elif self.get_param('config.manalyze_enable_binary', False) \
-                    and os.path.isfile(binary_path):
-                available_submodules.append(
-                    ManalyzeSubmodule(
-                        use_binary=True,
-                        binary_path=binary_path
-                    )
+                    ManalyzeSubmodule(use_binary=True, binary_path=binary_path)
                 )
             else:
-                self.error('Manalyze submodule is enabled, but either there is no method allowed (docker or binary)'
-                           'or the path to binary is not correct.')
+                self.error(
+                    "Manalyze submodule is enabled, but either there is no method allowed (docker or binary)"
+                    "or the path to binary is not correct."
+                )
+
+        # Check if floss submodule is enabled
+        if self.get_param("config.floss_enable", False):
+            binary_path = self.get_param(
+                "config.floss_binary_path", False, "Floss binary path not given."
+            )
+            available_submodules.append(
+                FlossSubmodule(
+                    binary_path=binary_path,
+                    string_length=self.get_param(
+                        "config.floss_minimal_string_length", 4
+                    ),
+                )
+            )
 
     def summary(self, raw):
         taxonomies = []
-        for submodule in raw['results']:
-            taxonomies += submodule['summary']['taxonomies']
-        return {'taxonomies': taxonomies}
+        for submodule in raw["results"]:
+            taxonomies += submodule["summary"]["taxonomies"]
+        return {"taxonomies": taxonomies}
 
     def run(self):
         results = []
 
         # Add metadata to result directly as it's mandatory
         m = MetadataSubmodule()
-        metadata_results = m.analyze_file(self.filepath)
-        results.append({
-            'submodule_name': m.name,
-            'results': metadata_results,
-            'summary': m.module_summary()
-        })
+        metadata_results, _ = m.analyze_file(self.filepath)
+        results.append(
+            {
+                "submodule_name": m.name,
+                "results": metadata_results,
+                "summary": m.module_summary(),
+            }
+        )
 
         for module in available_submodules:
-            if module.check_file(file=self.filepath, filetype=self.filetype, filename=self.filename,
-                                 mimetype=self.mimetype):
-                module_results = module.analyze_file(self.filepath)
-                module_summaries = module.module_summary()
-                results.append({
-                    'submodule_name': module.name,
-                    'results': module_results,
-                    'summary': module_summaries
-                })
+            if module.check_file(
+                file=self.filepath,
+                filetype=self.filetype,
+                filename=self.filename,
+                mimetype=self.mimetype,
+            ):
+                try:
+                    module_results, module_observables = module.analyze_file(
+                        self.filepath
+                    )
+                    if module_observables:
+                        for path in module_observables:
+                            self.observables.append((path, module.name))
+                    module_summaries = module.module_summary()
+                    results.append(
+                        {
+                            "submodule_name": module.name,
+                            "results": module_results,
+                            "summary": module_summaries,
+                        }
+                    )
+                except Exception as excp:
+                    results.append(
+                        {
+                            "submodule_name": module.name,
+                            "results": "{}".format(excp),
+                            "summary": None,
+                        }
+                    )
 
-        self.report({'results': results})
+        self.report({"results": results})
+
+    def artifacts(self, raw):
+        artifacts = []
+        if self.observables and self.self.auto_extract_artifacts:
+            for path, module in self.observables:
+                artifacts.append(
+                    self.build_artifact(
+                        "file", path, tags=["from_fileinfo:{}".format(module)]
+                    )
+                )
+        return artifacts
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     FileInfoAnalyzer().run()

analyzers/FileInfo/requirements.txt

@@ -9,3 +9,4 @@ oletools>=0.52
 extract-msg
 IMAPClient
 git+https://github.com/SteveClement/ioc_parser.git
+git+https://github.com/fireeye/[email protected]#egg=stringsifter

analyzers/FileInfo/submodules/submodule_base.py

@@ -1,8 +1,8 @@
 class SubmoduleBaseclass(object):
     def __init__(self):
-        self.name = 'This is where the module name should go.'
+        self.name = "This is where the module name should go."
         self.results = []
-        self.summary = {'taxonomies': []}
+        self.summary = {"taxonomies": []}
 
     def get_name(self):
         """
@@ -22,11 +22,11 @@ def build_taxonomy(self, level, namespace, predicate, value):
         :return: dict
         """
         return {
-                'level': level,
-                'namespace': namespace,
-                'predicate': predicate,
-                'value': value
-                }
+            "level": level,
+            "namespace": namespace,
+            "predicate": predicate,
+            "value": value,
+        }
 
     def check_file(self, **kwargs):
         """
@@ -52,10 +52,9 @@ def analyze_file(self, path):
 
         :param path: path to file
         :return:
-        :rtype: dict
+        :rtype: dict and list of observables or None
         """
-        return {}
-
+        return {}, None
 
     def module_summary(self):
         """
@@ -75,7 +74,9 @@ def add_result_subsection(self, subsection_header, results):
         :param summary: result dictionary
         :return:
         """
-        self.results.append({
-            "submodule_section_header": subsection_header,
-            "submodule_section_content": results
-        })
+        self.results.append(
+            {
+                "submodule_section_header": subsection_header,
+                "submodule_section_content": results,
+            }
+        )