Merge pull request #709 from TheHive-Project/feature/Urlscan_scan

Added url scan feature

analyzers/Urlscan.io/Urlscan_Scan.json

@@ -0,0 +1,23 @@
+{
+  "name": "Urlscan.io_Scan",
+  "author": "ninoseki, Kyle Parrish (@arnydo)",
+  "license": "MIT",
+  "url": "https://github.com/arnydo/Cortex-Analyzers",
+  "version": "0.1.0",
+  "description": "Scan URLs on urlscan.io",
+  "dataTypeList": ["url", "domain", "fqdn"],
+  "command": "Urlscan.io/urlscan_analyzer.py",
+  "baseConfig": "Urlscan.io",
+  "config": {
+    "service":"scan"
+  },
+  "configurationItems": [
+    {
+      "name": "key",
+      "description": "API key for Urlscan.io",
+      "type": "string",
+      "multi": false,
+      "required": true
+    }
+  ]
+}

analyzers/Urlscan.io/Urlscan_Search.json

@@ -1,11 +1,14 @@
 {
   "name": "Urlscan.io_Search",
-  "author": "ninoseki",
+  "author": "ninoseki, Kyle Parrish (@arnydo)",
   "license": "MIT",
-  "url": "https://github.com/ninoseki/cortex_urlscan_analyzer",
-  "version": "0.1.0",
-  "baseConfig": "Urlscan",
+  "url": "https://github.com/arnydo/Cortex-Analyzers",
+  "version": "0.1.1",
   "description": "Search IPs, domains, hashes or URLs on urlscan.io",
-  "dataTypeList": ["ip", "domain", "hash", "url"],
-  "command": "Urlscan.io/urlscan_analyzer.py"
+  "dataTypeList": ["ip", "domain", "hash", "fqdn", "url"],
+  "command": "Urlscan.io/urlscan_analyzer.py",
+  "baseConfig": "Urlscan.io",
+  "config": {
+    "service": "get"
+  }
 }

analyzers/Urlscan.io/urlscan.py

@@ -1,5 +1,6 @@
 import requests
 import json
+import time
 
 
 class UrlscanException(Exception):
@@ -18,3 +19,35 @@ def search(self):
             return r.json()
         else:
             raise UrlscanException("urlscan.io returns %s" % r.status_code)
+
+    def scan(self, api_key):
+        headers = {
+            "Content-Type": "application/json",
+            "API-Key": api_key,
+        }
+        data = '{"url": %s, "public": "on"}' % self.query
+        r = requests.post(
+            "https://urlscan.io/api/v1/scan/", headers=headers, data=data, verify=False
+        )
+        if r.status_code == 200:
+            submission_url = r.json()["api"]
+
+            finished = False
+            tries = 0
+            while tries <= 15:        
+                submission_req = requests.get(submission_url)
+                if submission_req.status_code == 200:
+                    return submission_req.json()
+                tries += 1
+                time.sleep(20)
+
+            raise UrlscanException(
+                "urlscan.io returns {0} and data was {1} on url {2}".format(
+                    submission_req.status_code, data, submission_url
+                )
+            )
+
+        else:
+            raise UrlscanException(
+                "urlscan.io returns {0} and data was {1}".format(r.status_code, data)
+            )

analyzers/Urlscan.io/urlscan_analyzer.py

@@ -6,6 +6,9 @@
 class UrlscanAnalyzer(Analyzer):
     def __init__(self):
         Analyzer.__init__(self)
+        self.service = self.get_param('config.service', None, 'Service parameter is missing')
+        if self.service == 'scan':
+            self.api_key = self.get_param('config.key', None, 'Missing URLScan API key')
 
     def search(self, indicator):
         """
@@ -17,40 +20,80 @@ def search(self, indicator):
         res = Urlscan(indicator).search()
         return res
 
+    def scan(self, indicator):
+        """
+        Scans a website for indicators
+        :param indicator: url
+        :type indicator: str
+        :return: dict
+        """
+        res = Urlscan(indicator).scan(self.api_key)
+        return res
+
     def run(self):
-        targets = ['ip', 'domain', 'hash', 'url']
-        if self.data_type == 'url':
-            query = '"{}"'.format(self.get_data())
+        if self.service == 'scan':
+            if self.data_type in ['domain', 'url', 'fqdn']:
+                query = '"{}"'.format(self.get_data())
+                try:
+                    self.report({
+                        'type': self.data_type,
+                        'query': query,
+                        'service': self.service,
+                        'indicator': self.scan(query)
+                    })
+                except UrlscanException as err:
+                    self.error(str(err))
+            else:
+                self.error('Invalid data type. URL expected')
+        elif self.service == 'get':
+            targets = ['ip', 'domain', 'fqdn', 'hash', 'url']
+            if self.data_type == 'url':
+                query = '"{}"'.format(self.get_data())
+            else:
+                query = self.get_data()
+
+            try:
+                if self.data_type in targets:
+                    self.report({
+                        'type': self.data_type,
+                        'query': query,
+                        'service': self.service,
+                        'indicator': self.search(query)
+                    })
+            except UrlscanException as err:
+                self.error(str(err))
         else:
-            query = self.get_data()
-
-        try:
-            if self.data_type in targets:
-                self.report({
-                    'type': self.data_type,
-                    'query': query,
-                    'indicator': self.search(query)
-                })
-        except UrlscanException as err:
-            self.error(str(err))
+            self.error('Invalid service')
+
 
     def summary(self, raw):
         taxonomies = []
         level = "info"
         namespace = "urlscan.io"
-        predicate = "Search"
-
-        total = raw["indicator"]["total"]
-        if total <= 1:
-            level = 'suspicious' if total == 1 else 'info'
-            value = "{} result".format(total)
-            taxonomies.append(self.build_taxonomy(
-                level, namespace, predicate, value))
+        predicate = "Search" if raw["service"] == 'get' else "Scan"
+
+        if predicate == "Search":
+            total = raw["indicator"]["total"]
+            if total <= 1:
+                level = 'suspicious' if total == 1 else 'info'
+                value = "{} result".format(total)
+                taxonomies.append(self.build_taxonomy(
+                    level, namespace, predicate, value))
+            else:
+                level = 'suspicious'
+                value = "{} results".format(total)
+                taxonomies.append(self.build_taxonomy(
+                    level, namespace, predicate, value))
         else:
-            level = 'suspicious'
-            value = "{} results".format(total)
+            score = raw["indicator"]["verdicts"]["overall"]["score"]
+            value = "Overall Score:{}".format(score)
+            malicious = raw["indicator"]["verdicts"]["overall"]["malicious"]
+            if malicious:
+                level = 'malicious'
+            elif score > 0:
+                level = 'suspicious'
             taxonomies.append(self.build_taxonomy(
-                level, namespace, predicate, value))
+                    level, namespace, predicate, value))
 
         return {"taxonomies": taxonomies}
 

thehive-templates/Urlscan_io_Scan_0_1_0/long.html

@@ -0,0 +1,117 @@
+<div class="panel panel-info" ng-if="::content.indicator.page">
+    <div class="panel-heading">
+        Urlscan.io Scan - General info
+    </div>
+    <div class="panel-body">
+        <dl class="dl-horizontal">
+            <dt>Url</dt>
+            <dd class="wrap">{{content.indicator.page.url}}</dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>Domain</dt>
+            <dd class="wrap">{{content.indicator.page.domain}}</dd>
+        </dl>        
+        <dl class="dl-horizontal">
+            <dt>Report</dt>
+            <dd class="wrap">
+                <a href="{{content.indicator.task.reportURL}}" target="_blank">
+                    {{content.indicator.task.reportURL}}
+                </a>
+            </dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>Ip</dt>
+            <dd class="wrap">{{content.indicator.page.ip}}</dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>Country</dt>
+            <dd class="wrap">{{content.indicator.page.country}} - {{content.indicator.page.city}}</dd>
+        </dl>
+    </div>
+</div>
+
+<div class="panel panel-info" ng-if="::content.indicator.verdicts">
+    <div class="panel-heading">
+        Verdicts
+    </div>
+    <div class="panel-body">
+        <dl class="dl-horizontal">
+            <dt>Community score:</dt>
+            <dd class="wrap">
+                {{content.indicator.verdicts.community.score}}
+                [Malicious votes: {{content.indicator.verdicts.community.votesMalicious}}/{{content.indicator.verdicts.community.votesTotal}}]
+            </dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>Engines score:</dt>
+            <dd class="wrap">
+                {{content.indicator.verdicts.engines.score}}
+                [Malicious engines: {{content.indicator.verdicts.engines.maliciousTotal}}/{{content.indicator.verdicts.engines.enginesTotal}}]
+            </dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>Urlscan score:</dt>
+            <dd class="wrap">
+                {{content.indicator.verdicts.urlscan.score}} 
+                <span class="label label-danger" ng-if="content.indicator.verdicts.urlscan.malicious">
+                    [malicious]
+                </span>
+            </dd>
+        </dl>
+        <dl class="dl-horizontal">
+            <dt>Overall score:</dt>
+            <dd class="wrap">
+                {{content.indicator.verdicts.overall.score}} 
+                <span class="label label-danger" ng-if="content.indicator.verdicts.urlscan.malicious">
+                    [malicious]
+                </span>
+            </dd>
+        </dl>
+    </div>
+</div>
+
+<div class="panel panel-info" ng-if="::content.indicator.lists">
+    <div class="panel-heading">
+        Items found
+    </div>
+    <div class="panel-body">
+        <table class="table">
+            <thead>
+                <tr>
+                    <th>IP</th>
+                    <th>asns</th>
+                    <th>Countries</th>
+                    <th>Hashes</th>
+                    <th>Servers</th>
+                    <th>Urls</th>
+                    <th>Domains</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>{{content.indicator.lists.ips.length}}</td>
+                    <td>{{content.indicator.lists.asns.length}}</td>
+                    <td>{{content.indicator.lists.countries.length}}</td>
+                    <td>{{content.indicator.lists.hashes.length}}</td>
+                    <td>{{content.indicator.lists.servers.length}}</td>
+                    <td>{{content.indicator.lists.urls.length}}</td>
+                    <td>{{content.indicator.lists.domains.length}}</td>
+                </tr>
+            </tbody>         
+        </table>                         
+    </div>
+</div>
+
+<!-- General error  -->
+<div class="panel panel-danger" ng-if="!success">
+    <div class="panel-heading">
+        <strong>{{artifact.data | fang}}</strong>
+    </div>
+    <div class="panel-body">
+        <dl class="dl-horizontal" ng-if="content.errorMessage">
+            <dt>
+                <i class="fa fa-warning"></i> urlscan.io: </dt>
+            <dd class="wrap">{{content.errorMessage}}</dd>
+        </dl>
+    </div>
+</div>

thehive-templates/Urlscan_io_Search_0_1_1/short.html

@@ -0,0 +1,3 @@
+<span class="label" ng-repeat="t in content.taxonomies" ng-class="{'info': 'label-info', 'safe': 'label-success', 'suspicious': 'label-warning', 'malicious':'label-danger'}[t.level]">
+    {{t.namespace}}:{{t.predicate}}="{{t.value}}"
+</span>