Add JSON parser function to Jinja2 templates

docs/user/bots.rst

@@ -4226,6 +4226,17 @@ Templates are in Jinja2 format with the event provided in the variable "event".
 
 See the Jinja2 documentation at https://jinja.palletsprojects.com/ .
 
+As an extension to the Jinja2 environment, the function "from_json" is
+available for parsing JSON strings into Python structures. This is
+useful if you want to handle complicated structures in the "output"
+field of an event. In that case, you would start your template with a
+line like::
+
+   {%- set output = from_json(event['output']) %}
+
+and can then use "output" as a regular Python object in the rest of
+the template.
+
 Attachments are template strings, especially useful for sending
 structured data. E.g. to send a JSON document including "malware.name"
 and all other fields starting with "source."::

intelmq/bots/outputs/templated_smtp/output.py

@@ -14,6 +14,17 @@
 
 See the Jinja2 documentation at https://jinja.palletsprojects.com/ .
 
+As an extension to the Jinja2 environment, the function "from_json" is
+available for parsing JSON strings into Python structures. This is
+useful if you want to handle complicated structures in the "output"
+field of an event. In that case, you would start your template with a
+line like:
+
+   {%- set output = from_json(event['output']) %}
+
+and can then use "output" as a regular Python object in the rest of
+the template.
+
 Attachments are template strings, especially useful for sending
 structured data. E.g. to send a JSON document including "malware.name"
 and all other fields starting with "source.":
@@ -79,7 +90,7 @@
 
 """
 
-import io
+import json
 import smtplib
 import ssl
 from typing import List, Optional
@@ -138,8 +149,11 @@ def init(self):
             "from": Template(self.mail_from),
             "to": Template(self.mail_to),
             "body": Template(self.body),
-            "attachments": []
         }
+        for tmpl in self.templates.values():
+            tmpl.globals.update({"from_json": json.loads})
+
+        self.templates["attachments"] = []
         for att in self.attachments:
             if "content-type" not in att:
                 self.logger.error("Attachment does not have a content-type, ignoring: %s.", att)
@@ -155,6 +169,8 @@ def init(self):
                 "text": Template(att["text"]),
                 "name": Template(att["name"])
             }
+            for tmpl in attachment_template.values():
+                tmpl.globals.update({"from_json": json.loads})
             self.templates["attachments"].append(attachment_template)
 
     def process(self):

intelmq/tests/bots/outputs/templated_smtp/test_output.py

@@ -131,6 +131,57 @@ def test_multiple_recipients_event(self):
         self.assertEqual({"from_addr": "myself", "to_addrs": ["[email protected]", "[email protected]"]},
                          SENT_MESSAGE[1])
 
+    def test_json_parsing(self):
+        event = EVENT.copy()
+        event["output"] = json.dumps(
+            {
+                "title": "Test title",
+                "lines": [
+                    {
+                        "time": "2021-10-11 12:13Z",
+                        "log": "An event occurred"
+                    },
+                    {
+                        "time": "2021-10-14 15:16Z",
+                        "log": "Another event occurred"
+                    }
+                ]
+            }
+        )
+        self.input_message = event
+        with unittest.mock.patch("smtplib.SMTP.send_message", new=send_message), \
+             unittest.mock.patch("smtplib.SMTP.connect", return_value=(220, "Mock server")), \
+             unittest.mock.patch("smtplib.SMTP.close"):
+            self.run_bot(parameters={"body": """
+{%- set output = from_json(event['output']) %}
+{{ output['title'] }}
+
+{% for line in output['lines'] -%}
+{{ line['time'] }} : {{ line['log'] }}
+{% endfor -%}
+"""})
+        self.assertEqual(SENT_MESSAGE[0].get_payload()[0].get_payload(), """
+Test title
+
+2021-10-11 12:13Z : An event occurred
+2021-10-14 15:16Z : Another event occurred
+""")
+
+    def test_malformed_json(self):
+        event = EVENT.copy()
+        event["output"] = "{ malformed"
+        self.input_message = event
+        with unittest.mock.patch("smtplib.SMTP.send_message", new=send_message), \
+             unittest.mock.patch("smtplib.SMTP.connect", return_value=(220, "Mock server")), \
+             unittest.mock.patch("smtplib.SMTP.close"):
+            self.run_bot(parameters={"body": """
+{%- set output = from_json(event['output']) %}
+A{{ output }}B
+"""})
+        self.assertEqual(SENT_MESSAGE[0].get_payload()[0].get_payload(), """
+A{ malformedB
+""")
+
 
 @test.skip_exotic()
 class TestDefaultTemplatedSMTPOutputBot(test.BotTestCase, unittest.TestCase):