From f0b99b5f50b9c2fd80e4d43bb3fa4d8784fef581 Mon Sep 17 00:00:00 2001 From: Nils Kuhnert <3c7@users.noreply.github.com> Date: Wed, 27 Dec 2017 09:24:11 +0100 Subject: [PATCH] Added MISP warning lists analyzer (#129) * Added MISP warning lists analyzer * Analyzer diet, no external dependencies, requires manual repo clone * Fixed typo in __lastcommit() * Added commit hash check for warninglists repo * Changed templates, added summary for outdated repo --- .../MISPWarningLists/MISPWarningLists.json | 12 +++ .../MISPWarningLists/mispwarninglists.py | 96 +++++++++++++++++++ analyzers/MISPWarningLists/requirements.txt | 2 + .../MISPWarningLists_1_0/long.html | 42 ++++++++ .../MISPWarningLists_1_0/short.html | 3 + 5 files changed, 155 insertions(+) create mode 100644 analyzers/MISPWarningLists/MISPWarningLists.json create mode 100755 analyzers/MISPWarningLists/mispwarninglists.py create mode 100644 analyzers/MISPWarningLists/requirements.txt create mode 100644 thehive-templates/MISPWarningLists_1_0/long.html create mode 100644 thehive-templates/MISPWarningLists_1_0/short.html diff --git a/analyzers/MISPWarningLists/MISPWarningLists.json b/analyzers/MISPWarningLists/MISPWarningLists.json new file mode 100644 index 000000000..df6767f9c --- /dev/null +++ b/analyzers/MISPWarningLists/MISPWarningLists.json @@ -0,0 +1,12 @@ +{ + "name": "MISPWarningLists", + "author": "Nils Kuhnert, CERT-Bund", + "license": "AGPL-V3", + "url": "https://github.com/BSI-CERT-Bund/misp-warninglists-analyzer", + "version": "1.0", + "baseConfig": "MISPWarningLists", + "config": {}, + "description": "Check IoCs/Observables against MISP Warninglists to filter false positives.", + "dataTypeList": ["ip", "hash", "domain", "fqdn", "url"], + "command": "MISPWarningLists/mispwarninglists.py" +} \ No newline at end of file diff --git a/analyzers/MISPWarningLists/mispwarninglists.py b/analyzers/MISPWarningLists/mispwarninglists.py new file mode 100755 index 000000000..d0865203c --- /dev/null +++ b/analyzers/MISPWarningLists/mispwarninglists.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +import io +import json +import requests + +from cortexutils.analyzer import Analyzer +from cortexutils.extractor import Extractor +from glob import glob +from os.path import exists + + +class MISPWarninglistsAnalyzer(Analyzer): + """ + This analyzer compares given data to the MISP warning lists obtainable via + https://github.com/MISP/misp-warninglists. + Configuration options are: + + ``` + MISPWarningLists { + path = "/path/to/misp-warninglists/repository" # Default: "misp-warninglists" + } + ``` + """ + def __init__(self): + Analyzer.__init__(self) + + self.data = self.get_data() + self.path = self.get_param('config.path', 'misp-warninglists') + if not exists(self.path): + self.error('Path to misp-warninglists does not exist.') + self.warninglists = self.readwarninglists() + + def readwarninglists(self): + files = glob('{}/lists/*/*.json'.format(self.path)) + listcontent = [] + for file in files: + with io.open(file, 'r') as fh: + content = json.loads(fh.read()) + values = Extractor().check_iterable(content.get('list', [])) + obj = { + "name": content.get('name', 'Unknown'), + "values": [value['value'] for value in values], + "dataTypes": [value['type'] for value in values] + } + listcontent.append(obj) + return listcontent + + def lastlocalcommit(self): + try: + with io.open('{}/.git/refs/heads/master'.format(self.path), 'r') as fh: + return fh.read().strip('\n') + except Exception as e: + return 'Error: could not get local commit hash ({}).'.format(e) + + @staticmethod + def lastremotecommit(): + url = 'https://api.github.com/repos/misp/misp-warninglists/branches/master' + try: + result_dict = requests.get(url).json() + return result_dict['commit']['sha'] + except Exception as e: + return 'Error: could not get remote commit hash ({}).'.format(e) + + def run(self): + results = [] + for list in self.warninglists: + if self.data_type not in list.get('dataTypes'): + continue + + if self.data in list.get('values', []): + results.append({ + "name": list.get('name') + }) + + self.report({ + "results": results, + "is_uptodate": self.lastlocalcommit() == self.lastremotecommit() + }) + + def summary(self, raw): + taxonomies = [] + if len(raw['results']) > 0: + taxonomies.append(self.build_taxonomy('suspicious', 'MISP', 'Warninglists', 'Potential fp')) + else: + taxonomies.append(self.build_taxonomy('info', 'MISP', 'Warninglists', 'No hits')) + + if not raw.get('is_uptodate', False): + taxonomies.append(self.build_taxonomy('info', 'MISP', 'Warninglists', 'Outdated')) + + return { + "taxonomies": taxonomies + } + + +if __name__ == '__main__': + MISPWarninglistsAnalyzer().run() diff --git a/analyzers/MISPWarningLists/requirements.txt b/analyzers/MISPWarningLists/requirements.txt new file mode 100644 index 000000000..6aabc3cfa --- /dev/null +++ b/analyzers/MISPWarningLists/requirements.txt @@ -0,0 +1,2 @@ +cortexutils +requests diff --git a/thehive-templates/MISPWarningLists_1_0/long.html b/thehive-templates/MISPWarningLists_1_0/long.html new file mode 100644 index 000000000..21fb4b8b9 --- /dev/null +++ b/thehive-templates/MISPWarningLists_1_0/long.html @@ -0,0 +1,42 @@ +
+
+ MISP warning lists information for {{artifact.data}} +
+
+
+
Results:
+
+

+ Observable was found in following MISP warning lists: +

+
    +
  • {{list.name}}
    • +
+
+
+

+ Observable was not found in warning lists. +

+
+
Warning lists status:
+
+ + Up-to-date + + + Outdated + +
+
+
+
+ + +
+
+ {{(artifact.data || artifact.attachment.name) | fang}} +
+
+ {{content.errorMessage}} +
+
diff --git a/thehive-templates/MISPWarningLists_1_0/short.html b/thehive-templates/MISPWarningLists_1_0/short.html new file mode 100644 index 000000000..3baac6a5a --- /dev/null +++ b/thehive-templates/MISPWarningLists_1_0/short.html @@ -0,0 +1,3 @@ + + {{t.namespace}}:{{t.predicate}}={{t.value}} + \ No newline at end of file