Support Shadowserver Vulnerable SMTP server feed, fix #1984

docs/user/bots.rst

@@ -1664,6 +1664,7 @@ These are the supported feed name and their corresponding file name for automati
    Vulnerable-Exchange-Server `*`            `scan_exchange`
    Vulnerable-ISAKMP                         `scan_isakmp`
    Vulnerable-HTTP                           `scan_http`
+   Vulnerable-SMTP                           `scan_smtp_vulnerable`
   =======================================   =========================
 
 `*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected.

intelmq/bots/parsers/shadowserver/_config.py

@@ -2787,6 +2787,32 @@ def scan_exchange_identifier(field):
     }
 }
 
+# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/
+vulnerable_smtp = {
+    'required_fields': [
+        ('time.source', 'timestamp', add_UTC_to_timestamp),
+        ('source.ip', 'ip', validate_ip),
+        ('source.port', 'port'),
+    ],
+    'optional_fields': [
+        ('source.reverse_dns', 'hostname'),
+        ('extra.', 'tag'),
+        ('source.asn', 'asn', invalidate_zero),
+        ('source.geolocation.cc', 'geo'),
+        ('source.geolocation.region', 'region'),
+        ('source.geolocation.city', 'city'),
+        ('extra.', 'naics', invalidate_zero),
+        ('extra.', 'sic', invalidate_zero),
+        ('extra.', 'banner', validate_to_none),
+    ],
+    'constant_fields': {
+        'classification.identifier': 'vulnerable-smtp',
+        'classification.taxonomy': 'vulnerable',
+        'classification.type': 'vulnerable-system',
+        'protocol.application': 'smtp',
+    }
+}
+
 mapping = (
     # feed name, file name, function
     ('Accessible-ADB', 'scan_adb', accessible_adb),
@@ -2862,8 +2888,9 @@ def scan_exchange_identifier(field):
     ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event46_sinkhole_http_referer),
     ('Spam-URL', 'spam_url', spam_url),
     ('Vulnerable-ISAKMP', 'scan_isakmp', vulnerable_isakmp),
-    ('Vulnerable-HTTP', 'scan_http', accessible_vulnerable_http),
+    ('Vulnerable-HTTP', 'scan_http_vulnerable', accessible_vulnerable_http),
     ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange),
+    ('Vulnerable-SMTP', 'scan_smtp_vulnerable', vulnerable_smtp),
 )
 
 feedname_mapping = {feedname: function for feedname, filename, function in mapping}

intelmq/tests/bots/parsers/shadowserver/test_broken.py

@@ -53,7 +53,7 @@ def test_broken(self):
         self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.",
                               levelname="DEBUG")
         self.assertLogMatches(pattern="Failed to parse line.")
-        self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Vulnerable-HTTP'. Possible change in data format or misconfiguration.")
+        self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.")
         self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.",
                               levelname="INFO")
 

intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py

@@ -0,0 +1,97 @@
+# SPDX-FileCopyrightText: 2021 Mikk Margus Möll <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
+
+with open(os.path.join(os.path.dirname(__file__),
+                       'testdata/scan_http_vulnerable.csv')) as handle:
+    EXAMPLE_FILE = handle.read()
+EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
+
+EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP',
+                  "raw": utils.base64_encode(EXAMPLE_FILE),
+                  "__type": "Report",
+                  "time.observation": "2021-08-01T09:00:00+00:00",
+                  "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv",
+                  }
+EVENTS = [{'__type': 'Event',
+           'feed.name': 'Vulnerable HTTP',
+           'classification.taxonomy': 'other',
+           'classification.type': 'other',
+           'classification.identifier': 'accessible-http',
+           'extra.http': 'HTTP/1.1',
+           'extra.http_code': 401,
+           'extra.http_reason': 'Unauthorized',
+           'extra.content_type': 'text/html; charset=%s',
+           'extra.connection': 'close',
+           'extra.server': 'mini_httpd/1.28 04Feb2018',
+           'extra.http_date': '2000-11-25T20:21:50+00:00',
+           'protocol.transport': 'tcp',
+           'extra.tag': 'basic-auth,http',
+           'extra.www_authenticate': 'Basic realm="Managed Switch"',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[1]])),
+           'source.asn': 1234,
+           'source.geolocation.cc': 'EE',
+           'source.geolocation.city': 'TARTU',
+           'source.geolocation.region': 'TARTUMAA',
+           'source.ip': '210.181.42.1',
+           'source.port': 8000,
+           'time.observation': '2021-08-01T09:00:00+00:00"',
+           'time.source': '2021-08-01T07:00:24+00:00'},
+          {'__type': 'Event',
+           'feed.name': 'Vulnerable HTTP',
+           'classification.taxonomy': 'other',
+           'classification.type': 'other',
+           'classification.identifier': 'accessible-http',
+           'extra.http': 'HTTP/1.1',
+           'extra.http_code': 401,
+           'extra.http_reason': 'Unauthorized',
+           'extra.server': 'Web Server',
+           'extra.content_type': 'text/html; charset=ISO-8859-1',
+           'extra.connection': 'close',
+           'extra.http_date': '2021-08-01T10:02:39+00:00',
+           'extra.tag': 'basic-auth,http',
+           'extra.www_authenticate': 'Basic realm="streaming_server"',
+           'protocol.transport': 'tcp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[2]])),
+           'source.reverse_dns': 'host.invalid',
+           'source.asn': 5678,
+           'source.geolocation.cc': 'EE',
+           'source.geolocation.city': 'TALLINN',
+           'source.geolocation.region': 'HARJUMAA',
+           'source.ip': '22.23.35.23',
+           'source.port': 8000,
+           'time.observation': '2021-08-01T09:00:00+00:00"',
+           'time.source': '2021-08-01T07:02:32+00:00'},
+          ]
+
+
+class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for a ShadowserverParserBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ShadowserverParserBot
+        cls.default_input_message = EXAMPLE_REPORT
+
+    def test_event(self):
+        """ Test if correct Event has been produced. """
+        self.run_bot()
+        for i, EVENT in enumerate(EVENTS):
+            self.assertMessageEqual(i, EVENT)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()

intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: 2021 Mikk Margus Möll <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
+
+with open(os.path.join(os.path.dirname(__file__),
+                       'testdata/scan_smtp_vulnerable.csv')) as handle:
+    EXAMPLE_FILE = handle.read()
+EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
+
+EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP',
+                  "raw": utils.base64_encode(EXAMPLE_FILE),
+                  "__type": "Report",
+                  "time.observation": "2021-07-08T00:00:00+00:00",
+                  "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv",
+                  }
+
+EVENTS = [{'__type': 'Event',
+           'feed.name': 'Vulnerable SMTP',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'extra.banner': '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 '
+                           '10:00:00 +0300|',
+           'extra.protocol': 'tcp',
+           'classification.identifier': 'vulnerable-smtp',
+           'extra.tag': 'smtp;21nails',
+           'protocol.application': 'smtp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[1]])),
+           'source.reverse_dns': 'smtp-server.invalid',
+           'source.asn': 12345,
+           'source.geolocation.cc': 'EE',
+           'source.geolocation.city': 'TALLINN',
+           'source.geolocation.region': 'HARJUMAA',
+           'source.ip': '1.2.3.4',
+           'source.port': 25,
+           'time.observation': '2021-07-08T00:00:00+00:00',
+           'time.source': '2021-07-08T11:58:42+00:00'},
+          {'__type': 'Event',
+           'feed.name': 'Vulnerable SMTP',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'extra.banner': '220 smtp-out.invalid, ESMTP EXIM 4.86_2|',
+           'extra.protocol': 'tcp',
+           'classification.identifier': 'vulnerable-smtp',
+           'extra.tag': 'smtp;21nails',
+           'protocol.application': 'smtp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[2]])),
+           'source.reverse_dns': 'smtp-out.invalid',
+           'source.asn': 23456,
+           'source.geolocation.cc': 'EE',
+           'source.geolocation.city': 'TALLINN',
+           'source.geolocation.region': 'HARJUMAA',
+           'source.ip': '5.6.7.8',
+           'source.port': 25,
+           'time.observation': '2021-07-08T00:00:00+00:00',
+           'time.source': '2021-07-08T11:58:44+00:00'},
+          ]
+
+
+class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for a ShadowserverParserBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ShadowserverParserBot
+        cls.default_input_message = EXAMPLE_REPORT
+
+    def test_event(self):
+        """ Test if correct Event has been produced. """
+        self.run_bot()
+        for i, EVENT in enumerate(EVENTS):
+            self.assertMessageEqual(i, EVENT)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()

intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv

@@ -0,0 +1,3 @@
+"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date"
+"2021-08-01 07:00:24","210.181.42.1","tcp",8000,,"basic-auth,http",1234,"EE","TARTUMAA","TARTU",,,"HTTP/1.1",401,"Unauthorized","text/html; charset=%s","close","Basic realm=""Managed Switch""",,"mini_httpd/1.28 04Feb2018",,,"Sat, 25 Nov 2000 20:21:50 GMT"
+"2021-08-01 07:02:32","22.23.35.23","tcp",8000,"host.invalid","basic-auth,http",5678,"EE","HARJUMAA","TALLINN",,,"HTTP/1.1",401,"Unauthorized","text/html; charset=ISO-8859-1","close","Basic realm=""streaming_server""",,"Web Server",,,"Thu, 01 Aug 2021 10:02:39 GMT"

intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2021 Mikk Margus Möll <[email protected]>
+SPDX-License-Identifier: AGPL-3.0-or-later

intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv

@@ -0,0 +1,3 @@
+"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner"
+"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|"
+"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|"

intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2021 Mikk Margus Möll <[email protected]>
+SPDX-License-Identifier: AGPL-3.0-or-later