Merge tag '2.6.0' into develop

2.6.0

analyzers/AbuseIPDB/abuseipdb.py

@@ -7,7 +7,7 @@
 
 class AbuseIPDBAnalyzer(Analyzer):
     """
-    AbuseIPDB API docs: https://www.abuseipdb.com/api
+    AbuseIPDB APIv2 docs: https://docs.abuseipdb.com/
     """
 
     @staticmethod
@@ -43,21 +43,33 @@ def run(self):
         try:
             if self.data_type == "ip":
                 api_key = self.get_param('config.key', None, 'Missing AbuseIPDB API key')
+
                 days_to_check = self.get_param('config.days', 30)
                 ip = self.get_data()
-                url = 'https://www.abuseipdb.com/check/{}/json?days={}'.format(ip, days_to_check)
-                response = requests.post(url, data = {'key': api_key})
+
+                url = 'https://api.abuseipdb.com/api/v2/check'
+                headers = {'Accept': 'application/json', 'Content-Type': 'application/x-www-form-urlencoded', 'Key': '%s' % api_key }
+                params = {'maxAgeInDays': days_to_check, 'verbose': 'True', 'ipAddress': ip}
+                response = requests.get(url, headers = headers, params = params)
+
                 if not (200 <= response.status_code < 300):
                     self.error('Unable to query AbuseIPDB API\n{}'.format(response.text))
+
                 json_response = response.json()
                 # this is because in case there's only one result, the api gives back a list instead of a dict
                 response_list = json_response if isinstance(json_response, list) else [json_response]
-                for found in response_list:
-                    if 'category' in found:
+                for response in response_list:
+                    if 'reports' in response["data"]:
                         categories_strings = []
-                        for category in found['category']:
-                            categories_strings.append(self.extract_abuse_ipdb_category(category))
-                        found['categories_strings'] = categories_strings
+                        for item in response["data"]["reports"]:
+                            item['categories_strings'] = []
+                            for category in item["categories"]:
+                                category_as_str = self.extract_abuse_ipdb_category(category)
+                                item['categories_strings'].append(category_as_str)
+                                if category_as_str not in categories_strings:
+                                    categories_strings.append(category_as_str)
+                        response['categories_strings'] = categories_strings
+
                 self.report({'values': response_list})
             else:
                 self.notSupported()
@@ -67,8 +79,8 @@ def run(self):
     def summary(self, raw):
         taxonomies = []
 
-        if raw and 'values' in raw and len(raw['values']) > 0 :
-            taxonomies.append(self.build_taxonomy('malicious', 'AbuseIPDB', 'Records', len(raw['values'])))
+        if raw and 'values' in raw and raw['values'][0]['data']['totalReports'] > 0 :
+            taxonomies.append(self.build_taxonomy('malicious', 'AbuseIPDB', 'Records', raw['values'][0]['data']['totalReports']))
         else:
             taxonomies.append(self.build_taxonomy('safe', 'AbuseIPDB', 'Records', 0))
 

analyzers/Crtsh/crtshquery.py

@@ -48,7 +48,7 @@ def search(self, domain, wildcard=True):
         if wildcard:
             url2 = base_url.format("%25{}.".format(domain))
             req2 = requests.get(url2, headers={'User-Agent': ua})
-            if req2.ok:
+            if req2.ok and not req2.headers['content-type'].startswith('text/html'):
                 try:
                     content2 = req2.content.decode('utf-8')
                     data2 = json.loads(content2.replace('}{', '},{'))
@@ -58,17 +58,19 @@ def search(self, domain, wildcard=True):
                     return None
 
         for c in data:
-            det_url = 'https://crt.sh/?q={}&output=json'.format(c['min_cert_id'])
-            try:
-                det_req = requests.get(det_url, headers={'User-Agent': ua})
-                if det_req.status_code == requests.codes.ok:
-                    det_con = det_req.content.decode('utf-8')
-                    sha1 = re.findall(rex, det_con)[0]
-                    c['sha1'] = sha1
-                else:
+            if c.get('min_cert_id'):
+                det_url = 'https://crt.sh/?q={}&output=json'.format(c['min_cert_id'])
+                try:
+                    det_req = requests.get(det_url, headers={'User-Agent': ua})
+                    if det_req.status_code == requests.codes.ok:
+                        det_con = det_req.content.decode('utf-8')
+                        sha1 = re.findall(rex, det_con)[0]
+                        c['sha1'] = sha1
+                    else:
+                        c['sha1'] = ''
+                except:
                     c['sha1'] = ''
-            except:
-                c['sha1'] = ''
+
         return data
 
     def __init__(self):

analyzers/DomainToolsIris/DomainToolsIris_Investigate.json

@@ -0,0 +1,38 @@
+{
+  "name": "DomainToolsIris_Investigate",
+  "version": "1.0",
+  "author": "DomainTools",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Use DomainTools Iris API to investigate a domain.",
+  "dataTypeList": ["domain"],
+  "command": "DomainToolsIris/domaintoolsiris_analyzer.py",
+  "baseConfig": "DomainToolsIris",
+  "config": {
+      "service": "investigate-domain"
+  },
+  "configurationItems": [
+    {
+      "name": "username",
+      "description": "DomainTools Iris API credentials",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "key",
+      "description": "DomainTools Iris API credentials",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "pivot_count_threshold",
+      "description": "Pivot count threshold.",
+      "type": "number",
+      "multi": false,
+      "required": false,
+      "defaultValue": 500
+    }
+  ]
+}

analyzers/DomainToolsIris/DomainToolsIris_Pivot.json

@@ -25,14 +25,6 @@
       "type": "string",
       "multi": false,
       "required": true
-    },
-    {
-      "name": "pivot_count_threshold",
-      "description": "Pivot count threshold.",
-      "type": "number",
-      "multi": false,
-      "required": false,
-      "defaultValue": 500
     }
   ]
 }

analyzers/DomainToolsIris/domaintoolsiris_analyzer.py

@@ -153,12 +153,12 @@ def format_single_domain(self, domain_data):
                 ] = DomainToolsAnalyzer.get_threat_level_class(
                     domain_risk["tpm"]["value"]
                 )
-            threat_profile_phshing_data = DomainToolsAnalyzer.get_threat_component(
+            threat_profile_phishing_data = DomainToolsAnalyzer.get_threat_component(
                 risk_components, "threat_profile_phishing"
             )
-            if threat_profile_phshing_data:
+            if threat_profile_phishing_data:
                 domain_risk["tpp"] = {}
-                domain_risk["tpp"]["value"] = threat_profile_malware_data.get(
+                domain_risk["tpp"]["value"] = threat_profile_phishing_data.get(
                     "risk_score", 0
                 )
                 domain_risk["tpp"][
@@ -171,7 +171,7 @@ def format_single_domain(self, domain_data):
             )
             if threat_profile_spam_data:
                 domain_risk["tps"] = {}
-                domain_risk["tps"]["value"] = threat_profile_malware_data.get(
+                domain_risk["tps"]["value"] = threat_profile_spam_data.get(
                     "risk_score", 0
                 )
                 domain_risk["tps"][

analyzers/FileInfo/submodules/submodule_oletools.py

@@ -31,7 +31,8 @@ def check_file(self, **kwargs):
             ] or (kwargs.get('mimetype').startswith("application/vnd.openxmlformats-officedocument") or
                  kwargs.get('mimetype').startswith("application/encrypted") or
                  kwargs.get('mimetype').startswith("application/vnd.ms-") or
-                 kwargs.get('mimetype').startswith("application/msword")
+                 kwargs.get('mimetype').startswith("application/msword") or
+                 kwargs.get('mimetype').startswith("Composite Document File V2 Document")
             ):
                 if kwargs.get('mimetype').startswith("application/encrypted") and not is_encrypted(kwargs.get('file')):
                     return False

analyzers/IntezerCommunity/IntezerCommunity.json

@@ -0,0 +1,25 @@
+{
+    "name": "IntezerCommunity",
+    "version": "1.0",
+    "author": "Matteo Lodi",
+    "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+    "license": "AGPL-v3",
+    "description": "Analyze a possible malicious file with Intezer Analyzer",
+    "dataTypeList": ["file"],
+    "baseConfig": "IntezerCommunity",
+    "command": "IntezerCommunity/intezer_community.py",
+    "configurationItems": [
+      {
+        "name": "key",
+        "description": "API key for Intezer",
+        "type": "string",
+        "multi": false,
+        "required": true
+      }
+    ],
+    "config": {
+        "check_tlp": true,
+        "max_tlp": 2,
+        "auto_extract": false
+    }
+}

analyzers/IntezerCommunity/intezer_community.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+import requests
+import time
+import os
+
+from cortexutils.analyzer import Analyzer
+
+
+class IntezerCommunityAnalyzer(Analyzer):
+    """
+    Intezer Community APIs: https://analyze.intezer.com/api/docs/documentation
+    """
+
+    def run(self):
+
+        try:
+
+            if self.data_type == 'file':
+                api_key = self.get_param('config.key', None, 'Missing Intezer API key')
+                filepath = self.get_param('file', None, 'File is missing')
+                filename = self.get_param('filename', os.path.basename(filepath))
+
+                base_url = 'https://analyze.intezer.com/api/v2-0'
+                # this should be done just once in a day, but we cannot do that with Cortex Analyzers
+                response = requests.post(base_url + '/get-access-token', json={'api_key': api_key})
+                response.raise_for_status()
+                session = requests.session()
+                session.headers['Authorization'] = session.headers['Authorization'] = 'Bearer %s' % response.json()[
+                    'result']
+
+                with open(filepath, 'rb') as file_to_upload:
+                    files = {'file': (filename, file_to_upload)}
+                    response = session.post(base_url + '/analyze', files=files)
+                    if response.status_code != 201:
+                        self.error('Error sending file to Intezer Analyzer\n{}'.format(response.text))
+
+                while response.status_code != 200:
+                    time.sleep(3)
+                    result_url = response.json()['result_url']
+                    response = session.get(base_url + result_url)
+                    response.raise_for_status()
+
+                report = response.json()
+                self.report(report)
+
+            else:
+                self.notSupported()
+
+        except requests.HTTPError as e:
+            self.error(e)
+        except Exception as e:
+            self.unexpectedError(e)
+
+    def summary(self, raw):
+        taxonomies = []
+        namespace = 'IntezerCommunity'
+
+        if 'status' in raw and raw['status'] == 'succeeded':
+            predicate = 'Analysis succeeded'
+        else:
+            predicate = 'Analysis failed'
+
+        level = 'info'
+        value = 'no family'
+        if 'result' in raw:
+            if 'verdict' in raw['result']:
+                level = raw['result']['verdict']
+                if level == 'trusted':
+                    level = 'safe'
+                if level not in ['info', 'safe', 'suspicious', 'malicious']:
+                    level = 'info'
+            if 'family_name' in raw['result']:
+                value = raw['result']['family_name']
+
+        taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))
+
+        return {'taxonomies': taxonomies}
+
+
+if __name__ == '__main__':
+    IntezerCommunityAnalyzer().run()

analyzers/IntezerCommunity/requirements.txt

@@ -0,0 +1,2 @@
+requests
+cortexutils

analyzers/MISPWarningLists/mispwarninglists.py

@@ -40,8 +40,8 @@ def readwarninglists(self):
                 values = Extractor().check_iterable(content.get('list', []))
                 obj = {
                     "name": content.get('name', 'Unknown'),
-                    "values": [value['value'] for value in values],
-                    "dataTypes": [value['type'] for value in values]
+                    "values": [value['data'] for value in values],
+                    "dataTypes": [value['dataType'] for value in values]
                 }
                 listcontent.append(obj)
         return listcontent

analyzers/NSRL/NSRL.json

@@ -0,0 +1,37 @@
+{
+  "name": "NSRL",
+  "author": "Andrea Garavaglia, Davide Arcuri - LDO-CERT",
+  "license": "AGPL-V3",
+  "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+  "version": "1.0",
+  "description": "Query NSRL",
+  "dataTypeList": [
+    "hash",
+    "filename"
+  ],
+  "baseConfig": "NSRL",
+  "command": "NSRL/nsrl.py",
+  "configurationItems": [
+    {
+      "name": "conn",
+      "description": "sqlalchemy connection string",
+      "multi": false,
+      "required": false,
+      "type": "string"
+    },
+    {
+      "name": "grep_path",
+      "description": "path of grep",
+      "type": "string",
+      "multi": false,
+      "required": false
+    },
+    {
+      "name": "nsrl_folder",
+      "description": "path of NSRL folder",
+      "type": "string",
+      "multi": false,
+      "required": false
+    }
+  ]
+}

analyzers/NSRL/create_db.py

@@ -0,0 +1,43 @@
+import os
+import sqlalchemy as db
+from glob import glob
+
+conn_string = "<insert postgres connection string >"
+NSRL_folder_path = "/path/to/NSRLFolder/*"
+
+engine = db.create_engine(conn_string)
+metadata = db.MetaData()
+
+nsrl = db.Table(
+    "nsrl",
+    metadata,
+    db.Column("id", db.Integer, primary_key=True, autoincrement=True),
+    db.Column("sha1", db.String),
+    db.Column("md5", db.String),
+    db.Column("crc32", db.String),
+    db.Column("filename", db.String),
+    db.Column("filesize", db.String),
+    db.Column("productcode", db.String),
+    db.Column("opsystemcode", db.String),
+    db.Column("specialcode", db.String),
+    db.Column("dbname", db.String),
+    db.Column("release", db.String),
+    db.Index("idx_sha1", "sha1"),
+    db.Index("idx_md5", "md5"),
+)
+metadata.create_all(engine)
+
+conn = engine.raw_connection()
+cursor = conn.cursor()
+for NSRL_file_path in glob(NSRL_folder_path):
+    dbname, release = NSRL_file_path.split("/")[-1].replace(".txt","").split("_")
+    print(dbname, release)
+    with open(NSRL_file_path, "r", encoding="latin-1") as f:
+        cmd = 'COPY nsrl("sha1", "md5", "crc32", "filename", "filesize", "productcode", "opsystemcode", "specialcode") FROM STDIN WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE)'
+        cursor.copy_expert(cmd, f)
+        conn.commit()
+        engine.execute("update nsrl set dbname='%s', release='%s' where dbname is null" % (dbname, release))
+        conn.commit()
+cursor.close()
+conn.close()
+engine.dispose()