Merge pull request #285 from ninoseki/feature/urlhaus-analyzer

Feature/urlhaus analyzer
TheHive-Project · Jul 2, 2018 · ba2a8fe · ba2a8fe
2 parents cc8acc0 + 10d130a
commit ba2a8fe
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 9 deletions.
diff --git a/analyzers/URLhaus/URLhaus.json b/analyzers/URLhaus/URLhaus.json
@@ -14,7 +14,7 @@
       "type": "number",
       "multi": false,
       "required": true,
-      "defaultValue": 3600
+      "defaultValue": 300
     },
     {
       "name": "cache.root",

diff --git a/analyzers/URLhaus/URLhaus.py b/analyzers/URLhaus/URLhaus.py
@@ -1,5 +1,5 @@
+from bs4 import BeautifulSoup
 from diskcache import Cache
-from requests_html import HTML
 import requests
 
 
@@ -16,7 +16,7 @@ class URLhaus:
 
     def __init__(self,
                  query,
-                 cache_duration=3600,
+                 cache_duration=300,
                  cache_root="/tmp/cortex/URLhaus"):
         self.URL = "https://urlhaus.abuse.ch/browse.php"
         self.query = query
@@ -47,15 +47,15 @@ def fetch(self):
 
     def parse(self, doc):
         results = []
-        html = HTML(html=doc)
-        table = html.find("table.table", first=True)
-        rows = table.find("tr")[1:]
+        soup = BeautifulSoup(doc, "html.parser")
+        table = soup.find("table", class_="table")
+        rows = table.find_all("tr")[1:]
         for row in rows:
-            cols = row.find("td")
+            cols = row.find_all("td")
             results.append({
                 "dateadded": cols[0].text,
                 "malware_url": cols[1].text,
-                "link": cols[1].find("a", first=True).attrs.get("href"),
+                "link": cols[1].find("a").attrs.get("href"),
                 "status": cols[2].text,
                 "tags": cols[3].text.split(),
                 "gsb": cols[4].text,

diff --git a/analyzers/URLhaus/requirements.txt b/analyzers/URLhaus/requirements.txt
@@ -1,4 +1,4 @@
+beautifulsoup4
 cortexutils
 diskcache
 requests
-requests-html
diff --git a/thehive-templates/URLhaus_0_1_0/long.html b/thehive-templates/URLhaus_0_1_0/long.html
@@ -0,0 +1,51 @@
+<div class="panel panel-info" ng-if="success">
+  <div class="panel-heading">
+    URLhaus search resutls for
+    <strong>{{artifact.data}}</strong>
+  </div>
+  <div class="panel-body">
+    <p ng-if="content.results.length == 0">
+      No result found.
+    </p>
+    <table class="table" ng-if="content.results">
+      <thead>
+        <th>Dateadded (UTC)</th>
+        <th>Malware URL</th>
+        <th>Status</th>
+        <th>Tags</th>
+        <th>GSB</th>
+        <th>Reporter</th>
+      </thead>
+      <tbody ng-repeat="r in content.results">
+        <tr>
+          <td>{{r.dateadded}}</td>
+          <td>
+            <a href="https://urlhaus.abuse.ch{{r.link}}" target=”_blank”>
+              {{r.malware_url}}
+            </a>
+          </td>
+          <td>{{r.status}}</td>
+          <td>
+            <span ng-repeat="tag in r.tags"> {{tag}} </span>
+          </td>
+          <td>{{r.gsb}}</td>
+          <td>{{r.reporter}}</td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+</div>
+
+<!-- General error  -->
+<div class="panel panel-danger" ng-if="!success">
+  <div class="panel-heading">
+    <strong>{{artifact.data | fang}}</strong>
+  </div>
+  <div class="panel-body">
+    <dl class="dl-horizontal" ng-if="content.errorMessage">
+      <dt>
+        <i class="fa fa-warning"></i> urlscan.io: </dt>
+      <dd class="wrap">{{content.errorMessage}}</dd>
+    </dl>
+  </div>
+</div>
diff --git a/thehive-templates/URLhaus_0_1_0/short.html b/thehive-templates/URLhaus_0_1_0/short.html
@@ -0,0 +1,3 @@
+<span class="label" ng-repeat="t in content.taxonomies" ng-class="{'info': 'label-info', 'safe': 'label-success', 'suspicious': 'label-warning', 'malicious':'label-danger'}[t.level]">
+  {{t.namespace}}:{{t.predicate}}={{t.value}}
+</span>