From e2dcd838ea3d6882ef49bd41d8c1b55b67dee5b4 Mon Sep 17 00:00:00 2001 From: garanews Date: Fri, 12 Jan 2018 09:22:35 +0100 Subject: [PATCH 1/3] added Malpedia Analyzer added Malpedia Analyzer --- analyzers/Malpedia/Malpedia.json | 12 +++ analyzers/Malpedia/malpedia_analyzer.py | 111 ++++++++++++++++++++++ analyzers/Malpedia/requirements.txt | 2 + thehive-templates/Malpedia_1_0/long.html | 41 ++++++++ thehive-templates/Malpedia_1_0/short.html | 3 + 5 files changed, 169 insertions(+) create mode 100644 analyzers/Malpedia/Malpedia.json create mode 100644 analyzers/Malpedia/malpedia_analyzer.py create mode 100644 analyzers/Malpedia/requirements.txt create mode 100644 thehive-templates/Malpedia_1_0/long.html create mode 100644 thehive-templates/Malpedia_1_0/short.html diff --git a/analyzers/Malpedia/Malpedia.json b/analyzers/Malpedia/Malpedia.json new file mode 100644 index 000000000..e56a197b6 --- /dev/null +++ b/analyzers/Malpedia/Malpedia.json @@ -0,0 +1,12 @@ +{ + "name": "Malpedia", + "author": "Davide Arcuri, Andrea Garavaglia - LDO-CERT", + "license": "AGPL-V3", + "url": "https://github.com/LDO-CERT/cortex-analyzers", + "version": "1.0", + "baseConfig": "Malpedia", + "config": {}, + "description": "Check files against Malpedia YARA rules.", + "dataTypeList": ["file"], + "command": "Malpedia/malpedia_analyzer.py" +} \ No newline at end of file diff --git a/analyzers/Malpedia/malpedia_analyzer.py b/analyzers/Malpedia/malpedia_analyzer.py new file mode 100644 index 000000000..8a1a2d7c8 --- /dev/null +++ b/analyzers/Malpedia/malpedia_analyzer.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# encoding: utf-8 + +from cortexutils.analyzer import Analyzer + +import os, sys +import json +import yara +import zipfile +import requests +import datetime +from requests.auth import HTTPBasicAuth +from stat import S_ISREG, ST_CTIME, ST_MODE, ST_MTIME + + +class MalpediaAnalyzer(Analyzer): + """Checking binaries through yara rules. This analyzer requires a list of yara rule paths in the cortex + configuration. If a path is given, an index file is expected.""" + def __init__(self): + Analyzer.__init__(self) + + self.baseurl = "https://malpedia.caad.fkie.fraunhofer.de/api/get" + + self.rulepaths = str(self.getParam('config.rules', None)) + self.user = self.getParam('config.user', None) + self.pwd = self.getParam('config.pwd', None) + self.update_hours = int(self.getParam('config.update_hours', 10)) + + if not os.path.exists(self.rulepaths): + os.makedirs(self.rulepaths) + + try: + newest = max(datetime.datetime.fromtimestamp(os.stat(path)[ST_MTIME]) for path in (os.path.join(self.rulepaths, fn) for fn in os.listdir(self.rulepaths) if os.path.isfile(os.path.join(self.rulepaths, fn)) and os.path.join(self.rulepaths, fn).endswith('.yar') )) + hours = (datetime.datetime.now() - newest).seconds / 3600 + except ValueError: + hours = self.update_hours + 1 + + + if hours > self.update_hours: + try: + req = requests.get("%s/yara/after/2010-01-01?format=json" % self.baseurl, auth=HTTPBasicAuth(self.user, self.pwd)) + if req.status_code == requests.codes.ok: + rules_json = json.loads(req.text) + for color, color_data in rules_json.items(): + for rule_name, rule_text in color_data.items(): + with open('%s' % os.path.join(self.rulepaths, rule_name), 'w') as f: + f.write(rule_text.encode('utf-8').strip()) + except Exception: + e = sys.exc_info()[1] + with open('%s' % os.path.join(self.rulepaths, "error.txt"), 'w') as f: + f.write(e.args[0]) + + def check(self, file): + """ + Checks a given file against all available yara rules + :param file: Path to file + :type file:str + :returns: Python list with matched rules info + :rtype: list + """ + result = [] + all_matches = [] + for filerules in os.listdir(self.rulepaths): + try: + rule = yara.compile(os.path.join(self.rulepaths, filerules)) + except yara.SyntaxError: + continue + matches = rule.match(file) + if len(matches) > 0: + for rulem in matches: + rule_family = "_".join([x for x in rulem.rule.replace("_", ".", 1).split("_")[:-1]]) + if rule_family not in all_matches: + all_matches.append(rule_family) + for rule_family in all_matches: + rules_info_txt = requests.get("%s/family/%s" % (self.baseurl, rule_family), auth=HTTPBasicAuth(self.user, self.pwd)) + rules_info_json = json.loads(rules_info_txt.text) + result.append({ + 'family':rule_family, + 'common_name': rules_info_json['common_name'], + 'description': rules_info_json['description'], + 'attribution': rules_info_json['attribution'], + 'alt_names': rules_info_json['alt_names'], + 'urls': rules_info_json['urls'] + }) + + return result + + def summary(self, raw): + taxonomies = [] + level = "info" + namespace = "Malpedia" + predicate = "Match" + + value = "\"{} rule(s)\"".format(len(raw["results"])) + if len(raw["results"]) == 0: + level = "safe" + else: + level = "malicious" + + taxonomies.append(self.build_taxonomy(level, namespace, predicate, value)) + return {"taxonomies": taxonomies} + + def run(self): + if self.data_type == 'file': + self.report({'results': self.check(self.getParam('file'))}) + else: + self.error('Wrong data type.') + +if __name__ == '__main__': + MalpediaAnalyzer().run() + diff --git a/analyzers/Malpedia/requirements.txt b/analyzers/Malpedia/requirements.txt new file mode 100644 index 000000000..41addc946 --- /dev/null +++ b/analyzers/Malpedia/requirements.txt @@ -0,0 +1,2 @@ +yara-python +cortexutils \ No newline at end of file diff --git a/thehive-templates/Malpedia_1_0/long.html b/thehive-templates/Malpedia_1_0/long.html new file mode 100644 index 000000000..96ab465ad --- /dev/null +++ b/thehive-templates/Malpedia_1_0/long.html @@ -0,0 +1,41 @@ +
+
+ Malpedia Report +
+
+
+

{{m.common_name}} [{{m.family}}]

+
+
Description
+
{{m.description}}
+
Attributions
+

{{x}}

+
Alternative Names
+

{{x}}

+
Urls
+

{{x}}

+
+
+
+
+
+
+ Malpedia Report +
+
+ No matches. +
+
+ + +
+
+ {{(artifact.data || artifact.attachment.name) | fang}} +
+
+
+
Yara:
+
{{content.errorMessage}}
+
+
+
\ No newline at end of file diff --git a/thehive-templates/Malpedia_1_0/short.html b/thehive-templates/Malpedia_1_0/short.html new file mode 100644 index 000000000..3baac6a5a --- /dev/null +++ b/thehive-templates/Malpedia_1_0/short.html @@ -0,0 +1,3 @@ + + {{t.namespace}}:{{t.predicate}}={{t.value}} + \ No newline at end of file From ed8c9703457e48e1d127f3ad383a3cd45e117ed9 Mon Sep 17 00:00:00 2001 From: Nils Kuhnert Date: Fri, 12 Jan 2018 15:00:27 +0100 Subject: [PATCH 2/3] PIP, python3, readability --- analyzers/Malpedia/Malpedia.json | 2 +- analyzers/Malpedia/malpedia_analyzer.py | 69 ++++++++++++++----------- analyzers/Malpedia/requirements.txt | 3 +- 3 files changed, 41 insertions(+), 33 deletions(-) mode change 100644 => 100755 analyzers/Malpedia/malpedia_analyzer.py diff --git a/analyzers/Malpedia/Malpedia.json b/analyzers/Malpedia/Malpedia.json index e56a197b6..9603171b6 100644 --- a/analyzers/Malpedia/Malpedia.json +++ b/analyzers/Malpedia/Malpedia.json @@ -9,4 +9,4 @@ "description": "Check files against Malpedia YARA rules.", "dataTypeList": ["file"], "command": "Malpedia/malpedia_analyzer.py" -} \ No newline at end of file +} diff --git a/analyzers/Malpedia/malpedia_analyzer.py b/analyzers/Malpedia/malpedia_analyzer.py old mode 100644 new mode 100755 index 8a1a2d7c8..888e8b002 --- a/analyzers/Malpedia/malpedia_analyzer.py +++ b/analyzers/Malpedia/malpedia_analyzer.py @@ -1,53 +1,60 @@ #!/usr/bin/env python -# encoding: utf-8 -from cortexutils.analyzer import Analyzer - -import os, sys +import os +import io +import sys import json import yara -import zipfile import requests import datetime + +from cortexutils.analyzer import Analyzer from requests.auth import HTTPBasicAuth -from stat import S_ISREG, ST_CTIME, ST_MODE, ST_MTIME +from stat import ST_MTIME class MalpediaAnalyzer(Analyzer): """Checking binaries through yara rules. This analyzer requires a list of yara rule paths in the cortex configuration. If a path is given, an index file is expected.""" + def __init__(self): Analyzer.__init__(self) self.baseurl = "https://malpedia.caad.fkie.fraunhofer.de/api/get" - - self.rulepaths = str(self.getParam('config.rules', None)) - self.user = self.getParam('config.user', None) - self.pwd = self.getParam('config.pwd', None) - self.update_hours = int(self.getParam('config.update_hours', 10)) + self.rulepaths = str(self.get_param('config.rules', None, 'No rulepath provided.')) + self.user = self.get_param('config.username', None, 'No username provided.') + self.pwd = self.get_param('config.password', None, 'No password provided.') + self.update_hours = int(self.get_param('config.update_hours', 10)) if not os.path.exists(self.rulepaths): os.makedirs(self.rulepaths) + timestamps = [] try: - newest = max(datetime.datetime.fromtimestamp(os.stat(path)[ST_MTIME]) for path in (os.path.join(self.rulepaths, fn) for fn in os.listdir(self.rulepaths) if os.path.isfile(os.path.join(self.rulepaths, fn)) and os.path.join(self.rulepaths, fn).endswith('.yar') )) + for fn in os.listdir(self.rulepaths): + for path in os.path.join(self.rulepaths, fn): + if os.path.isfile(path) and path.endswith('.yar'): + timestamps.append(datetime.datetime.fromtimestamp(os.stat(path)[ST_MTIME])) + newest = max(timestamps) hours = (datetime.datetime.now() - newest).seconds / 3600 except ValueError: hours = self.update_hours + 1 - - if hours > self.update_hours: + if hours > self.update_hours or len(timestamps) == 0: try: - req = requests.get("%s/yara/after/2010-01-01?format=json" % self.baseurl, auth=HTTPBasicAuth(self.user, self.pwd)) + req = requests.get('{}/yara/after/2010-01-01?format=json'.format(self.baseurl), + auth=HTTPBasicAuth(self.user, self.pwd)) if req.status_code == requests.codes.ok: rules_json = json.loads(req.text) for color, color_data in rules_json.items(): for rule_name, rule_text in color_data.items(): - with open('%s' % os.path.join(self.rulepaths, rule_name), 'w') as f: - f.write(rule_text.encode('utf-8').strip()) + with io.open(os.path.join(self.rulepaths, rule_name), 'w') as f: + f.write(rule_text) + else: + self.error('Could not download new rules due tue HTTP {}: {}'.format(req.status_code, req.text)) except Exception: e = sys.exc_info()[1] - with open('%s' % os.path.join(self.rulepaths, "error.txt"), 'w') as f: + with io.open('%s' % os.path.join(self.rulepaths, "error.txt"), 'w') as f: f.write(e.args[0]) def check(self, file): @@ -60,7 +67,7 @@ def check(self, file): """ result = [] all_matches = [] - for filerules in os.listdir(self.rulepaths): + for filerules in os.listdir(self.rulepaths): try: rule = yara.compile(os.path.join(self.rulepaths, filerules)) except yara.SyntaxError: @@ -71,23 +78,23 @@ def check(self, file): rule_family = "_".join([x for x in rulem.rule.replace("_", ".", 1).split("_")[:-1]]) if rule_family not in all_matches: all_matches.append(rule_family) - for rule_family in all_matches: - rules_info_txt = requests.get("%s/family/%s" % (self.baseurl, rule_family), auth=HTTPBasicAuth(self.user, self.pwd)) - rules_info_json = json.loads(rules_info_txt.text) + for rule_family in all_matches: + rules_info_txt = requests.get('{}/family/{}'.format(self.baseurl, rule_family), + auth=HTTPBasicAuth(self.user, self.pwd)) + rules_info_json = json.loads(rules_info_txt.text) result.append({ - 'family':rule_family, - 'common_name': rules_info_json['common_name'], - 'description': rules_info_json['description'], - 'attribution': rules_info_json['attribution'], - 'alt_names': rules_info_json['alt_names'], + 'family': rule_family, + 'common_name': rules_info_json['common_name'], + 'description': rules_info_json['description'], + 'attribution': rules_info_json['attribution'], + 'alt_names': rules_info_json['alt_names'], 'urls': rules_info_json['urls'] }) - + return result def summary(self, raw): taxonomies = [] - level = "info" namespace = "Malpedia" predicate = "Match" @@ -102,10 +109,10 @@ def summary(self, raw): def run(self): if self.data_type == 'file': - self.report({'results': self.check(self.getParam('file'))}) + self.report({'results': self.check(self.get_param('file', None, 'No file given.'))}) else: self.error('Wrong data type.') + if __name__ == '__main__': MalpediaAnalyzer().run() - diff --git a/analyzers/Malpedia/requirements.txt b/analyzers/Malpedia/requirements.txt index 41addc946..fc7ae2d16 100644 --- a/analyzers/Malpedia/requirements.txt +++ b/analyzers/Malpedia/requirements.txt @@ -1,2 +1,3 @@ yara-python -cortexutils \ No newline at end of file +cortexutils +requests From 1cfc8910563aac9d3c5915f6061b39bf34bfb8bf Mon Sep 17 00:00:00 2001 From: Nils Kuhnert Date: Thu, 18 Jan 2018 10:54:14 +0100 Subject: [PATCH 3/3] Small changes, encoding, template changes --- analyzers/Malpedia/malpedia_analyzer.py | 12 +++++------- thehive-templates/Malpedia_1_0/long.html | 25 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/analyzers/Malpedia/malpedia_analyzer.py b/analyzers/Malpedia/malpedia_analyzer.py index 888e8b002..c449eee15 100755 --- a/analyzers/Malpedia/malpedia_analyzer.py +++ b/analyzers/Malpedia/malpedia_analyzer.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python - +#!/usr/bin/env python3 import os import io import sys @@ -21,7 +20,7 @@ def __init__(self): Analyzer.__init__(self) self.baseurl = "https://malpedia.caad.fkie.fraunhofer.de/api/get" - self.rulepaths = str(self.get_param('config.rules', None, 'No rulepath provided.')) + self.rulepaths = self.get_param('config.path', None, 'No rulepath provided.') self.user = self.get_param('config.username', None, 'No username provided.') self.pwd = self.get_param('config.password', None, 'No password provided.') self.update_hours = int(self.get_param('config.update_hours', 10)) @@ -48,14 +47,13 @@ def __init__(self): rules_json = json.loads(req.text) for color, color_data in rules_json.items(): for rule_name, rule_text in color_data.items(): - with io.open(os.path.join(self.rulepaths, rule_name), 'w') as f: + with io.open(os.path.join(self.rulepaths, rule_name), 'w', encoding='utf-8') as f: f.write(rule_text) else: self.error('Could not download new rules due tue HTTP {}: {}'.format(req.status_code, req.text)) - except Exception: - e = sys.exc_info()[1] + except Exception as e: with io.open('%s' % os.path.join(self.rulepaths, "error.txt"), 'w') as f: - f.write(e.args[0]) + f.write('Error: {}\n'.format(e)) def check(self, file): """ diff --git a/thehive-templates/Malpedia_1_0/long.html b/thehive-templates/Malpedia_1_0/long.html index 96ab465ad..a12b308be 100644 --- a/thehive-templates/Malpedia_1_0/long.html +++ b/thehive-templates/Malpedia_1_0/long.html @@ -7,13 +7,28 @@

{{m.common_name}} [{{m.family}}]

Description
-
{{m.description}}
+
{{m.description || "No description given."}}
Attributions
-

{{x}}

+
+ No attributions given. +
    +
  • {{x}}
    • +
+
Alternative Names
-

{{x}}

+
+ No alternative names given. +
    +
  • {{x}}
    • +
+
Urls
-

{{x}}

+
+ No Urls given. + +
@@ -38,4 +53,4 @@

{{m.common_name}} [{{m.family}}]

{{content.errorMessage}}
- \ No newline at end of file +