Merge pull request #1 from TheHive-Project/master

Pull in latest release

analyzers/Cyberprotect/CyberprotectAnalyzer.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+import requests
+from cortexutils.analyzer import Analyzer
+
+class CyberprotectAnalyzer(Analyzer):
+
+    URI = "https://threatscore.cyberprotect.fr/api/score/"
+
+    def __init__(self):
+        Analyzer.__init__(self)
+        self.service = self.get_param('config.service', None, 'Service parameter is missing')
+
+    def summary(self, raw):
+        taxonomies = []
+        namespace = "Cyberprotect"
+        if self.service == 'ThreatScore':
+            level = 'info'
+            value = 'not in database'
+            if raw.get('data') and raw.get('scores') and len(raw.get('scores')) > 0:
+                value = 'not analyzed yet'
+                if raw['scores'][0].get('score'):
+                    level = 'safe'
+                    value = raw['scores'][0]['score']
+                    if value >= 0.5:
+                        level = 'malicious'
+                    elif value >= 0.25 and value < 0.5:
+                        level = 'suspicious'
+            taxonomies.append(self.build_taxonomy(level, namespace, self.service, value))
+        return {"taxonomies": taxonomies}
+
+    def run(self):
+        Analyzer.run(self)
+        if self.service == 'ThreatScore' and (self.data_type == 'domain' or self.data_type == 'ip'):
+            try:
+                response = requests.get("{}{}".format(self.URI, self.get_data()))
+                result = response.json()
+                self.report(result if len(result) > 0 else {})
+            except Exception as e:
+                self.unexpectedError(e)
+        else:
+            self.notSupported()
+
+if __name__ == '__main__':
+    CyberprotectAnalyzer().run()

analyzers/Cyberprotect/Cyberprotect_ThreatScore.json

@@ -0,0 +1,15 @@
+{
+    "name": "Cyberprotect_ThreatScore",
+    "author": "Rémi Allain, Cyberprotect",
+    "license": "AGPL-V3",
+    "url": "https://github.com/Cyberprotect/Cortex-Analyzers",
+    "version": "1.0",
+    "description": "ThreatScore is a cyber threat scoring system provided by Cyberprotect",
+    "dataTypeList": ["domain", "ip"],
+    "command": "Cyberprotect/CyberprotectAnalyzer.py",
+    "baseConfig": "Cyberprotect",
+    "config": {
+        "service": "ThreatScore",
+        "check_tlp": true
+    }
+}

analyzers/Cyberprotect/requirements.txt

@@ -0,0 +1,2 @@
+cortexutils
+requests

analyzers/DNSDB/dnsdb.py

@@ -19,13 +19,17 @@ def __init__(self):
 
     def execute_dnsdb_service(self, client):
         data = self.get_data()
+        rrtype = self.get_param('parameters.rrtype', None, None)
+        bailiwick = self.get_param('parameters.bailiwick', None, None)
+        before = self.get_param('parameters.before', None, None)
+        after = self.get_param('parameters.after', None, None)
 
         if self.service == 'domain_name' and self.data_type in ['domain', 'fqdn']:
-            return client.query_rrset(data)
+            return client.query_rrset(data, rrtype=rrtype, bailiwick=bailiwick, before=before, after=after)
         elif self.service == 'ip_history' and self.data_type == 'ip':
-            return client.query_rdata_ip(data)
+            return client.query_rdata_ip(data, before=before, after=after)
         elif self.service == 'name_history' and self.data_type in ['domain', 'fqdn']:
-            return client.query_rdata_name(data)
+            return client.query_rdata_name(data, rrtype=rrtype, before=before, after=after)
         else:
             self.error('Unknown DNSDB service or invalid data type')
 

analyzers/DomainTools/DomainTools_HostingHistory.json

@@ -0,0 +1,30 @@
+{
+  "name": "DomainTools_HostingHistory",
+  "version": "2.0",
+  "author": "ANSSI",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Use DomainTools to get a list of historical registrant, name servers and IP addresses for a domain name.",
+  "dataTypeList": ["domain"],
+  "command": "DomainTools/domaintools_analyzer.py",
+  "baseConfig": "DomainTools",
+  "config": {
+      "service": "hosting-history"
+  },
+  "configurationItems": [
+    {
+      "name": "username",
+      "description": "DomainTools API credentials",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "key",
+      "description": "DomainTools API credentials",
+      "type": "string",
+      "multi": false,
+      "required": true
+    }
+  ]
+}

analyzers/DomainTools/DomainTools_Reputation.json

@@ -1,17 +1,17 @@
 {
-    "name": "DomainTools_Reputation",
-    "version": "2.0",
-    "author": "CERT-BDF",
-    "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
-    "license": "AGPL-V3",
-    "description": "Use DomainTools to get a reputation score on a domain or fqdn",
-    "dataTypeList": ["domain","fqdn"],
-    "command": "DomainTools/domaintools_analyzer.py",
-    "baseConfig": "DomainTools",
-    "config": {
-        "service": "reputation"
-    },
-    "configurationItems": [
+  "name": "DomainTools_Reputation",
+  "version": "2.0",
+  "author": "CERT-BDF",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Use DomainTools to get a reputation score on a domain or fqdn",
+  "dataTypeList": ["domain","fqdn"],
+  "command": "DomainTools/domaintools_analyzer.py",
+  "baseConfig": "DomainTools",
+  "config": {
+      "service": "reputation"
+  },
+  "configurationItems": [
     {
       "name": "username",
       "description": "DomainTools API credentials",

analyzers/DomainTools/DomainTools_ReverseIP.json

@@ -1,17 +1,17 @@
 {
-    "name": "DomainTools_ReverseIP",
-    "version": "2.0",
-    "author": "CERT-BDF",
-    "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
-    "license": "AGPL-V3",
-    "description": "Use DomainTools to get a list of domain names sharing the same IP address.",
-    "dataTypeList": ["ip", "domain","fqdn"],
-    "command": "DomainTools/domaintools_analyzer.py",
-    "baseConfig": "DomainTools",
-    "config": {
-        "service": "reverse-ip"
-    },
-    "configurationItems": [
+  "name": "DomainTools_ReverseIP",
+  "version": "2.0",
+  "author": "CERT-BDF",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Use DomainTools to get a list of domain names sharing the same IP address.",
+  "dataTypeList": ["ip", "domain","fqdn"],
+  "command": "DomainTools/domaintools_analyzer.py",
+  "baseConfig": "DomainTools",
+  "config": {
+      "service": "reverse-ip"
+  },
+  "configurationItems": [
     {
       "name": "username",
       "description": "DomainTools API credentials",

analyzers/DomainTools/DomainTools_ReverseIPWhois.json

@@ -0,0 +1,30 @@
+{
+  "name": "DomainTools_ReverseIPWhois",
+  "version": "2.0",
+  "author": "ANSSI",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Use DomainTools to get a list of IP addresses which share the same registrant information.",
+  "dataTypeList": ["mail", "ip", "domain", "other"],
+  "command": "DomainTools/domaintools_analyzer.py",
+  "baseConfig": "DomainTools",
+  "config": {
+      "service": "reverse-ip-whois"
+  },
+  "configurationItems": [
+    {
+      "name": "username",
+      "description": "DomainTools API credentials",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "key",
+      "description": "DomainTools API credentials",
+      "type": "string",
+      "multi": false,
+      "required": true
+    }
+  ]
+}

analyzers/DomainTools/DomainTools_Risk.json

@@ -1,17 +1,17 @@
 {
-    "name": "DomainTools_Risk",
-    "version": "2.0",
-    "author": "CERT-BDF",
-    "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
-    "license": "AGPL-V3",
-    "description": "Use DomainTools to get a risk score and evidence details on a domain or fqdn",
-    "dataTypeList": ["domain","fqdn"],
-    "command": "DomainTools/domaintools_analyzer.py",
-    "baseConfig": "DomainTools",
-    "config": {
-        "service": "risk_evidence"
-    },
-    "configurationItems": [
+  "name": "DomainTools_Risk",
+  "version": "2.0",
+  "author": "CERT-BDF",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Use DomainTools to get a risk score and evidence details on a domain or fqdn",
+  "dataTypeList": ["domain","fqdn"],
+  "command": "DomainTools/domaintools_analyzer.py",
+  "baseConfig": "DomainTools",
+  "config": {
+      "service": "risk_evidence"
+  },
+  "configurationItems": [
     {
       "name": "username",
       "description": "DomainTools API credentials",

analyzers/DomainTools/DomainTools_WhoisLookup.json

@@ -4,8 +4,8 @@
   "author": "CERT-BDF",
   "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
   "license": "AGPL-V3",
-  "description": "Use DomainTools to get the ownership record for a domain with basic registration details.",
-  "dataTypeList": ["domain"],
+  "description": "Use DomainTools to get the ownership record for a domain or an IP address with basic registration details parsed.",
+  "dataTypeList": ["domain", "ip"],
   "command": "DomainTools/domaintools_analyzer.py",
   "baseConfig": "DomainTools",
   "config": {

analyzers/DomainTools/DomainTools_WhoisLookupIP.json → analyzers/DomainTools/DomainTools_WhoisLookupUnparsed.json

@@ -1,11 +1,11 @@
 {
-  "name": "DomainTools_WhoisLookup_IP",
+  "name": "DomainTools_WhoisLookupUnparsed",
   "version": "2.0",
   "author": "CERT-BDF",
   "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
   "license": "AGPL-V3",
-  "description": "Use DomainTools to get the ownership record for an IP address with basic registration details.",
-  "dataTypeList": ["ip"],
+  "description": "Use DomainTools to get the ownership record for an IP address or a domain without parsing.",
+  "dataTypeList": ["ip", "domain"],
   "command": "DomainTools/domaintools_analyzer.py",
   "baseConfig": "DomainTools",
   "config": {

analyzers/DomainTools/domaintools_analyzer.py

@@ -42,30 +42,38 @@ def domaintools(self, data):
         elif self.service == 'whois/history' and self.data_type == 'domain':
             response = api.whois_history(data).response()
 
-        elif self.service == 'whois/parsed' and self.data_type == 'domain':
+        elif self.service == 'whois/parsed' and self.data_type in ['domain','ip']:
             response = api.parsed_whois(data).response()
 
+        elif self.service == 'hosting-history' and self.data_type == 'domain':
+            response = api.hosting_history(data).response()
+
         elif self.service == 'risk_evidence' and self.data_type in ['domain', 'fqdn']:
             response = api.risk_evidence(data).response()
 
         elif self.service == 'reputation' and self.data_type in ['domain', 'fqdn']:
             response = api.reputation(data, include_reasons=True).response()
 
         elif self.service == 'reverse-whois':
-            response = api.reverse_whois(data, mode='purchase').response()
+            scope = self.getParam('parameters.scope', 'current', None)
+            response = api.reverse_whois(data, mode='purchase', scope=scope).response()
 
-        elif self.service == 'whois' and self.data_type == 'ip':
+        elif self.service == 'reverse-ip-whois':
+            response = api.reverse_ip_whois(data).response()
+
+        elif self.service == 'whois' and self.data_type in ['domain', 'ip']:
             response = api.whois(data).response()
 
         return response
 
 
     def summary(self, raw):
+
         r = {
             "service": self.service,
             "dataType": self.data_type
         }
-
+        
         if "ip_addresses" in raw:
             if type(raw["ip_addresses"]) == dict:
                 r["ip"] = {
@@ -87,14 +95,23 @@ def summary(self, raw):
                 "historic": raw["domain_count"]["historic"]
             }
 
+        if "registrar_history" in raw:
+            r["registrar_history"] = len(raw["registrar_history"])
+        if "ip_history" in raw:
+            r["ip_history"] = len(raw["ip_history"])
+        if "nameserver_history" in raw:
+            r["ns_history"] = len(raw["nameserver_history"])
+
+        if "record_count" in raw:
+            r["record_count"] = raw["record_count"]
+
         if "registrant" in raw:
             r["registrant"] = raw["registrant"]
         elif "response" in raw and "registrant" in raw["response"]:
             r["registrant"] = raw["response"]["registrant"]
 
         if "parsed_whois" in raw:
             r["registrar"] = raw["parsed_whois"]["registrar"]["name"]
-            #
 
         if "name_server" in raw:
             r["name_server"] = raw["name_server"]["hostname"]
@@ -123,6 +140,16 @@ def summary(self, raw):
                                                                                          r["domain_count"][
                                                                                              "historic"])))
 
+        if r["service"] == "reverse-ip-whois":
+            taxonomies.append(self.build_taxonomy("info", "DT", "Reverse_IP_Whois",
+                                                  "records:{}".format(r["record_count"])))
+
+        if r["service"] == "hosting-history":
+            taxonomies.append(self.build_taxonomy("info", "DT", "Hosting_History",
+                                                  "registrars:{} / ips:{} / ns:{}".format(r["registrar_history"],
+                                                                                              r["ip_history"],
+                                                                                                  r["ns_history"])))
+
         if r["service"] == "whois/history":
             taxonomies.append(self.build_taxonomy("info", "DT", "Whois_History",
                                                   "{} {}".format(r["record_count"], "records" if r["record_count"] > 1 else "record")))

analyzers/EmlParser/parse.py

@@ -67,7 +67,7 @@ def parseEml(filepath):
     #cause eml_parser does not provide raw headers (as far as I know)
     hParser = email.parser.HeaderParser()
     h = hParser.parsestr(raw_eml)
-    result['headers'] = (str(h).split('\n\n')[0])
+    result['headers'] = dict(h)
 
     parsed_eml = eml_parser.eml_parser.decode_email(filepath, include_raw_body=True, include_attachment_data=True)
     #parsed_eml['header'].keys() gives:

analyzers/FileInfo/requirements.txt

@@ -6,5 +6,5 @@ pefile
 git+https://github.com/AnyMaster/pehashng
 git+https://github.com/Rafiot/pdfid.git
 oletools>=0.52
-git+https://github.com/mattgwwalker/msg-extractor.git
+git+https://github.com/mattgwwalker/msg-extractor.git@v0.19
 IMAPClient

analyzers/HIBP/HIBP_Query.json

@@ -0,0 +1,25 @@
+{
+    "name": "HIBP_Query",
+    "version": "1.0",
+    "author": "Matt Erasmus",
+    "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+    "license": "AGPL-V3",
+    "description": "Query haveibeenpwned.com for a compromised email address",
+    "dataTypeList": ["mail"],
+    "baseConfig": "HIBP",
+    "config": {
+      "service": "query",
+      "url": "https://haveibeenpwned.com/api/v2/breachedaccount/"
+    },
+    "command": "HIBP_Query/hibpquery_analyzer.py",
+    "configurationItems": [
+        {
+          "name": "unverified",
+          "description": "Include unverified breaches",
+          "type": "boolean",
+          "multi": false,
+          "required": true,
+          "defaultValue": true
+        }
+    ]
+}