From b7bcd924c780c2f2b2419e4ec72bea1f3be8e48b Mon Sep 17 00:00:00 2001
From: Eric Capuano <ec19051@ec19051-csd-mbp.local>
Date: Mon, 21 Nov 2016 23:44:33 -0600
Subject: [PATCH] New analyzer to check URL categories

---
 .../URLCategory/report/success_long.html      | 18 ++++
 .../URLCategory/report/success_short.html     |  4 +
 analyzers/URLCategory/urlcategory.py          | 85 +++++++++++++++++++
 analyzers/URLCategory_1.0.json                | 13 +++
 4 files changed, 120 insertions(+)
 create mode 100644 analyzers/URLCategory/report/success_long.html
 create mode 100644 analyzers/URLCategory/report/success_short.html
 create mode 100644 analyzers/URLCategory/urlcategory.py
 create mode 100644 analyzers/URLCategory_1.0.json

diff --git a/analyzers/URLCategory/report/success_long.html b/analyzers/URLCategory/report/success_long.html
new file mode 100644
index 0000000000..a48ebbb6da
--- /dev/null
+++ b/analyzers/URLCategory/report/success_long.html
@@ -0,0 +1,18 @@
+<div class="panel panel-info">
+    <div class="panel-heading">
+        URL Categories of <strong>{{artifact.data}}</strong>
+    </div>
+    <div class="panel-body">
+	<dl class="dl-horizontal" ng-if="content.fortinet_category">
+                <dt>Fortinet URL Category: </dt>
+                <dd class="wrap">{{content.fortinet_category}}&nbsp; 
+			<i class="fa fa-search"></i>
+                        <a ng-href="http://www.fortiguard.com/iprep?data={{artifact.data}}&lookup=Lookup" target="_blank">
+                                View Full Report</a>
+			<i class="fa fa-recycle"></i>    
+			<a ng-href="http://www.fortiguard.com/iprep_form?data={{artifact.data}}" target="_blank">
+				Request Recategorization</a> 
+		</dd>
+        </dl>
+    </div>
+</div>
diff --git a/analyzers/URLCategory/report/success_short.html b/analyzers/URLCategory/report/success_short.html
new file mode 100644
index 0000000000..bc42a51644
--- /dev/null
+++ b/analyzers/URLCategory/report/success_short.html
@@ -0,0 +1,4 @@
+<span class="label label-info">
+	URLCat:
+	<span ng-if="content.fortinet_category">{{content.fortinet_category}}&nbsp;</span>
+</span>
diff --git a/analyzers/URLCategory/urlcategory.py b/analyzers/URLCategory/urlcategory.py
new file mode 100644
index 0000000000..5d8fd4463a
--- /dev/null
+++ b/analyzers/URLCategory/urlcategory.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# encoding: utf-8
+import sys
+import os
+import json
+import codecs
+import time
+import re
+import requests
+
+if sys.stdout.encoding != 'UTF-8':
+	if sys.version_info.major == 3:
+		sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
+	else:
+		sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict')
+if sys.stderr.encoding != 'UTF-8':
+	if sys.version_info.major == 3:
+		sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
+	else:
+		sys.stderr = codecs.getwriter('utf-8')(sys.stderr, 'strict')
+
+# load artifact
+artifact = json.load(sys.stdin)
+
+def error(message):
+	print('{{"errorMessage":"{}"}}'.format(message))
+	sys.exit(1)
+
+def get_param(name, default=None, message=None, current=artifact):
+	if isinstance(name, str):
+		name = name.split('.')
+	if len(name) == 0:
+		return current
+	else:
+		value = current.get(name[0])
+		if value == None:
+			if message != None:
+				error(message)
+			else:
+				return default
+		else:
+			return get_param(name[1:], default, message, value)
+
+def debug(msg):
+	#print >> sys.stderr, msg
+	pass
+
+def fortinet_category(data):
+	debug('>> fortinet_category ' + str(data))
+	pattern = re.compile("(?:Category: )([\w\s]+)")
+	baseurl = 'http://www.fortiguard.com/iprep?data='
+	tailurl = '&lookup=Lookup'
+	url = baseurl + data + tailurl
+	r = requests.get(url)
+	category_match = re.search(pattern, r.content, flags=0)
+	return category_match.group(1)
+
+http_proxy     = get_param('config.proxy.http')
+https_proxy    = get_param('config.proxy.https')
+max_tlp        = get_param('config.max_tlp', 1)
+tlp            = get_param('tlp', 2) # amber by default
+data_type      = get_param('dataType', None, 'Missing dataType field')
+service        = get_param('config.service', None, 'Service parameter is missing')
+
+# run  only if TLP condition is met
+if tlp > max_tlp:
+	error('Error with TLP value ; see max_tlp in config or tlp value in input data')
+
+# setup proxy
+if http_proxy != None:
+	os.environ['http_proxy'] = http_proxy
+if https_proxy != None:
+	os.environ['https_proxy'] = https_proxy
+
+if service == 'query':
+	if data_type == 'url' or data_type == 'domain':
+		data = get_param('data', None, 'Data is missing')
+	        json.dump({
+                        'fortinet_category': fortinet_category(data)
+                	}, sys.stdout, ensure_ascii=False)
+	else:
+		error('Invalid data type')
+else:
+	error('Invalid service')
+
diff --git a/analyzers/URLCategory_1.0.json b/analyzers/URLCategory_1.0.json
new file mode 100644
index 0000000000..a98d46127a
--- /dev/null
+++ b/analyzers/URLCategory_1.0.json
@@ -0,0 +1,13 @@
+{
+	"name": "URLCategory",
+	"version": "1.0",
+	"report": "URLCategory/report",
+	"description": "URL Category query: checks the category of a specific URL or domain",
+	"dataTypeList": ["url", "domain"],
+	"baseConfig" : "URLCategory",
+	"config": {
+		"service": "query",
+		"max_tlp": 10
+	},
+	"command": "URLCategory/urlcategory.py"
+}