Splunk Analyzer

analyzers/Splunk/splunk.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+import splunklib.client as client
+from time import sleep
+from cortexutils.analyzer import Analyzer
+import splunklib.results as results
+import urllib
+import re
+
+
+class Splunk(Analyzer):
+
+    def __init__(self):
+        Analyzer.__init__(self)
+        self.service = self.getParam('config.service', None, 'Service parameter is missing')
+        self.HOST = self.getParam('config.host', None, 'Host parameter is missing')
+        self.PORT = self.getParam('config.port', None, 'Port parameter is missing')
+        self.USERNAME = self.getParam('config.username', None, 'Username parameter is missing')
+        self.PASSWORD = self.getParam('config.password', None, 'Password parameter is missing')
+        self.DAYS = self.getParam('config.num_of_days', None, 'Number of days parameter is missing')
+        self.FILTERQUERY = self.getParam('config.filter_expression', None, 'Splunk Search query')
+
+        if self.getParam('config.sourcetype', None) is not None:
+            self.SOURCE = self.getParam('config.sourcetype', None)
+            self.source_or_index = "sourcetype"
+        elif self.getParam('config.index', None) is not None:
+            self.SOURCE = self.getParam('config.index', None)
+            self.source_or_index = "index"
+        else:
+            self.error("You must specify either a sourcetype or an index to search on. You may also specify '*' for all sourcetypes/indexes.")
+
+        self.base_search_query = 'search {0} {1}={2}'
+
+
+    # Create a Service instance and log in
+    def SplunkConnect(self):
+        try:
+            self.service = client.connect(
+                host=self.HOST,
+                port=self.PORT,
+                username=self.USERNAME,
+                password=self.PASSWORD)
+        except Exception, e:
+            self.unexpectedError(e)
+
+    def SplunkURLSearch(self, url):
+        try:
+            regex = re.compile(r"(?::\/\/)([^\/|\?|\&|\$|\+|\,|\:|\;|\=|\@|\#]+)")
+            match = regex.search(url)
+            domain = match.group(1)
+        except Exception as e:
+            self.error('Malformed URL. Could not extract FQDN from URL.' + str(e))
+        searchquery_normal = self.base_search_query.format(domain, self.source_or_index ,self.SOURCE) + self.FILTERQUERY
+        kwargs_normalsearch = {"exec_mode": "normal", "earliest_time": "-{0}d".format(self.DAYS), "latest":"now" ,"output_mode": "xml"}
+        job = self.service.jobs.create(searchquery_normal, **kwargs_normalsearch)
+        # # A normal search returns the job's SID right away, so we need to poll for completion
+        while True:
+            while not job.is_ready():
+                pass
+            stats = {"isDone": job["isDone"]}
+            if stats["isDone"] == "1":
+                break
+            sleep(2)
+        # Get the results and display them
+        finalResult = {}
+        index = 0
+        for result in results.ResultsReader(job.results()):
+            finalResult[index] = result
+            index += 1
+        finalResult["length"] = index
+
+        searchquery_formatted = searchquery_normal.replace("\/\"", "")
+        finalResult["search_query"] = urllib.quote_plus(searchquery_formatted, safe=';/?:@&=+$,"$#@=?%^Q^$')
+
+        job.cancel()
+
+        self.report(finalResult)
+
+
+    def SplunkDomainSearch(self, domain):
+        searchquery_normal = self.base_search_query.format(domain, self.source_or_index ,self.SOURCE) + self.FILTERQUERY
+        kwargs_normalsearch = {"exec_mode": "normal", "earliest_time": "-{0}d".format(self.DAYS), "latest":"now" ,"output_mode": "xml"}
+        job = self.service.jobs.create(searchquery_normal, **kwargs_normalsearch)
+        # # A normal search returns the job's SID right away, so we need to poll for completion
+        while True:
+            while not job.is_ready():
+                pass
+            stats = {"isDone": job["isDone"]}
+            if stats["isDone"] == "1":
+                break
+            sleep(2)
+        # Get the results and display them
+        finalResult = {}
+        index = 0
+        for result in results.ResultsReader(job.results()):
+            finalResult[index] = result
+            index += 1
+        finalResult["length"] = index
+
+        searchquery_formatted = searchquery_normal.replace("\/\"", "")
+        finalResult["search_query"] = urllib.quote_plus(searchquery_formatted, safe=';/?:@&=+$,"$#@=?%^Q^$')
+
+        job.cancel()
+
+        self.report(finalResult)
+
+    def SplunkIPSearch(self, ipaddr):
+        searchquery_normal = self.base_search_query.format(ipaddr, self.source_or_index ,self.SOURCE) + self.FILTERQUERY
+        kwargs_normalsearch = {"exec_mode": "normal", "earliest_time": "-{0}d".format(self.DAYS), "latest":"now" ,"output_mode": "xml"}
+        job = self.service.jobs.create(searchquery_normal, **kwargs_normalsearch)
+        # # A normal search returns the job's SID right away, so we need to poll for completion
+        while True:
+            while not job.is_ready():
+                pass
+            stats = {"isDone": job["isDone"]}
+
+            if stats["isDone"] == "1":
+                break
+            sleep(2)
+        # Get the results and display them
+        finalResult = {}
+        index = 0
+        for result in results.ResultsReader(job.results()):
+            finalResult[index] = result
+            index += 1
+        finalResult["length"] = index
+
+        searchquery_formatted = searchquery_normal.replace("\/\"", "")
+        finalResult["search_query"] = urllib.quote_plus(searchquery_formatted, safe=';/?:@&=+$,"$#@=?%^Q^$')
+
+        job.cancel()
+
+        self.report(finalResult)
+
+    def SplunkGenericSearch(self, searchparam):
+        searchquery_normal = self.base_search_query.format(searchparam, self.source_or_index ,self.SOURCE) + self.FILTERQUERY
+        kwargs_normalsearch = {"exec_mode": "normal", "earliest_time": "-{0}d".format(self.DAYS), "latest": "now",
+                               "output_mode": "xml"}
+        job = self.service.jobs.create(searchquery_normal, **kwargs_normalsearch)
+        # # A normal search returns the job's SID right away, so we need to poll for completion
+        while True:
+            while not job.is_ready():
+                pass
+            stats = {"isDone": job["isDone"]}
+
+            if stats["isDone"] == "1":
+                break
+            sleep(2)
+        # Get the results and display them
+        finalResult = {}
+        index = 0
+        for result in results.ResultsReader(job.results()):
+            finalResult[index] = result
+            index += 1
+        finalResult["length"] = index
+
+        searchquery_formatted = searchquery_normal.replace("\/\"", "")
+        finalResult["search_query"] = urllib.quote_plus(searchquery_formatted, safe=';/?:@&=+$,"$#@=?%^Q^$')
+
+        job.cancel()
+
+        self.report(finalResult)
+
+    def summary(self, raw):
+        taxonomies = []
+        predicate = "Hits"
+        value = "\"0\""
+        result = {
+            "has_result": True
+        }
+
+        if self.data_type == "domain" or self.data_type == "url":
+            namespace = "Splunk_Web_Proxy_Logs_{0}_days".format(self.DAYS)
+            result["length"] = raw["length"]
+            if result["length"] > 0:
+                level = "suspicious"
+                value = "\"{}\"".format(result["length"])
+            else:
+                level = "safe"
+        else:
+            namespace = "Splunk_{0}".format(self.SOURCE)
+            result["length"] = raw["length"]
+            if result["length"] > 0:
+                level = "info"
+                value = "\"{}\"".format(result["length"])
+            else:
+                level = "safe"
+
+        taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))
+        return {"taxonomies": taxonomies}
+
+    def run(self):
+        Analyzer.run(self)
+        if self.service == 'search':
+            if self.data_type == 'url':
+                data = self.getParam('data', None, 'Data is missing')
+                self.SplunkConnect()
+                self.SplunkURLSearch(data)
+            elif self.data_type == 'domain':
+                data = self.getParam('data', None, 'Data is missing')
+                self.SplunkConnect()
+                self.SplunkDomainSearch(data)
+            elif self.data_type == 'ip':
+                data = self.getParam('data', None, 'Data is missing')
+                self.SplunkConnect()
+                self.SplunkIPSearch(data)
+            elif self.data_type != 'file':
+                data = self.getParam('data', None, 'Data is missing')
+                self.SplunkConnect()
+                self.SplunkGenericSearch(data)
+            else:
+                self.error('Invalid Datatype')
+        else:
+            self.error('Invalid service')
+
+if __name__ == '__main__':
+    Splunk().run()

analyzers/Splunk/splunk_30_days.json

@@ -0,0 +1,19 @@
+{
+    "name": "Splunk_Search_30_days",
+    "version": "2.0",
+    "url": "",
+    "author": "Unit777",
+    "license": "AGPL-V3",
+    "dataTypeList": ["url", "domain"],
+    "description": "Searches for hits in Splunk web proxy logs for past 30 days",
+    "baseConfig": "Splunk",
+    "config": {
+        "check_tlp": false,
+        "max_tlp": 4,
+        "service": "search",
+        "num_of_days": 30,
+        "sourcetype":"proxylogs",
+        "filter_expression": "| eval time=strftime(_time, \"%y-%m-%d\") | eval action=if(isnull(action), \"-\",action) | eval url=if(isnull(url), \"-\",url) | eval http_referrer=if(isnull(http_referrer), \"-\",http_referrer) | eval dest_domain_full=if(isnull(dest_domain_full), \"-\",dest_domain_full) | eval http_method=if(isnull(http_method), \"-\",http_method) | eval Event=action.\"^\".url.\"^\".http_referrer.\"^\".dest_domain_full.\"^\".http_method.\"^\".time |top Event limit=5 by user |stats count, values(Event) as Events by user | table *"
+    },
+    "command": "Splunk/splunk.py"
+}

analyzers/Splunk/splunk_3_months.json

@@ -0,0 +1,19 @@
+{
+    "name": "Splunk_Search_3_months",
+    "version": "2.0",
+    "url": "",
+    "author": "Unit777",
+    "license": "AGPL-V3",
+    "dataTypeList": ["url", "domain"],
+    "description": "Searches for hits in Splunk web proxy logs for past 3 months",
+    "baseConfig": "Splunk",
+    "config": {
+        "check_tlp": false,
+        "max_tlp": 4,
+        "service": "search",
+        "num_of_days": 90,
+        "sourcetype":"proxylogs",
+        "filter_expression": "| eval time=strftime(_time, \"%y-%m-%d\") | eval action=if(isnull(action), \"-\",action) | eval url=if(isnull(url), \"-\",url) | eval http_referrer=if(isnull(http_referrer), \"-\",http_referrer) | eval dest_domain_full=if(isnull(dest_domain_full), \"-\",dest_domain_full) | eval http_method=if(isnull(http_method), \"-\",http_method) | eval Event=action.\"^\".url.\"^\".http_referrer.\"^\".dest_domain_full.\"^\".http_method.\"^\".time |top Event limit=5 by user |stats count, values(Event) as Events by user | table *"
+    },
+    "command": "Splunk/splunk.py"
+}

analyzers/Splunk/splunk_7_days.json

@@ -0,0 +1,19 @@
+{
+    "name": "Splunk_Search_7_days",
+    "version": "2.0",
+    "url": "",
+    "author": "Unit777",
+    "license": "AGPL-V3",
+    "dataTypeList": ["url", "domain"],
+    "description": "Searches for hits in Splunk web proxy logs for past 7 days",
+    "baseConfig": "Splunk",
+    "config": {
+        "check_tlp": false,
+        "max_tlp": 4,
+        "service": "search",
+        "num_of_days": 7,
+        "sourcetype":"proxylogs",
+        "filter_expression": "| eval time=strftime(_time, \"%y-%m-%d\") | eval action=if(isnull(action), \"-\",action) | eval url=if(isnull(url), \"-\",url) | eval http_referrer=if(isnull(http_referrer), \"-\",http_referrer) | eval dest_domain_full=if(isnull(dest_domain_full), \"-\",dest_domain_full) | eval http_method=if(isnull(http_method), \"-\",http_method) | eval Event=action.\"^\".url.\"^\".http_referrer.\"^\".dest_domain_full.\"^\".http_method.\"^\".time |top Event limit=5 by user |stats count, values(Event) as Events by user | table *"
+    },
+    "command": "Splunk/splunk.py"
+}

analyzers/Splunk/splunk_generic_example.json

@@ -0,0 +1,19 @@
+{
+    "name": "Splunk_Generic_Search",
+    "version": "2.0",
+    "url": "",
+    "author": "Unit777",
+    "license": "AGPL-V3",
+    "dataTypeList": ["domain", "url", "hash", "mail"],
+    "description": "Generic Splunk Search",
+    "baseConfig": "Splunk",
+    "config": {
+        "check_tlp": false,
+        "max_tlp": 4,
+        "service": "search",
+        "num_of_days": 30,
+        "sourcetype":"genericlogs",
+        "filter_query": "| table *"
+    },
+    "command": "Splunk/splunk.py"
+}

analyzers/Splunk/splunk_ip_internet_tier.json

@@ -0,0 +1,19 @@
+{
+    "name": "Splunk_Search_IP_Logs",
+    "version": "2.0",
+    "url": "",
+    "author": "Unit777",
+    "license": "AGPL-V3",
+    "dataTypeList": ["ip"],
+    "description": "Searches for hits in Splunk IP logs for past 30 days",
+    "baseConfig": "Splunk",
+    "config": {
+        "check_tlp": false,
+        "max_tlp": 4,
+        "service": "search",
+        "num_of_days": 30,
+        "sourcetype":"iplogs",
+        "filter_expression": "| eval time=strftime(_time, \"%y-%m-%d\") | eval src_ip=if(isnull(src_ip), \"-\",src_ip) | eval src=if(isnull(src), \"-\",src) | eval Event=src_ip.\"^\".src.\"^\".time |dedup src |stats count, values(Event) as Events by src_ip | table *"
+    },
+    "command": "Splunk/splunk.py"
+}

analyzers/Splunk/splunklib/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Python library for Splunk."""
+
+__version_info__ = (1, 6, 2)
+__version__ = ".".join(map(str, __version_info__))
+