Merge pull request #170 from thalesgroup-cert/test

v2.1.1

.env

@@ -48,6 +48,7 @@ THE_HIVE_KEY=
 # Ensure the custom field referenced here is CREATED IN THEHIVE. Otherwise, Alert exports to TheHive will be impacted
 THE_HIVE_CUSTOM_FIELD=watcher-id
 THE_HIVE_EMAIL_SENDER=[email protected]
+THE_HIVE_TAGS=Watcher,Impersonation,Malicious Domain,Typosquatting
 
 # MISP Setup
 MISP_URL=

Watcher/Watcher/common/utils/send_thehive_alerts.py

@@ -43,16 +43,18 @@ def post_to_thehive(url, data, headers, proxies):
         return None
 
 
-def send_thehive_alert(title, description, severity, tags, app_name, domain_name, observables=None, customFields=None, thehive_url=None, api_key=None):
+def send_thehive_alert(title, description, severity, tlp, pap, tags, app_name, domain_name, observables=None, customFields=None, thehive_url=None, api_key=None):
     from common.core import generate_ref
     """
     Send or update an alert in TheHive based on the application and ticket_id.
 
     :param title: The title of the alert.
     :param description: The description of the alert.
     :param severity: The severity level of the alert (integer).
+    :param tlp: The Traffic Light Protocol (TLP) level of the alert (integer).
+    :param pap: The Permissible Action Protocol (PAP) level of the alert (integer).
     :param tags: A list of tags associated with the alert.
-    :param app_name: The application triggering the alert (e.g., 'website_monitoring').
+    :param app_name: The application triggering the alert.
     :param domain_name: The domain name related to the alert (used for ticket_id lookup).
     :param observables: Any observables (default is None).
     :param customFields: Custom fields for the alert (default is None).
@@ -95,6 +97,8 @@ def send_thehive_alert(title, description, severity, tags, app_name, domain_name
             title=title,
             description=description,
             severity=severity,
+            tlp=tlp,
+            pap=pap,
             tags=tags,
             app_name=app_name,
             observables=observables,
@@ -114,6 +118,8 @@ def send_thehive_alert(title, description, severity, tags, app_name, domain_name
             title=title,
             description=description,
             severity=severity,
+            tlp=tlp,
+            pap=pap,
             tags=tags,
             app_name=app_name,
             observables=observables,

Watcher/Watcher/common/utils/update_thehive.py

@@ -106,15 +106,31 @@ def create_observables(observables):
 
     observables_data = []
     for obs in observables:
+        tag_info = []
+        if 'tags' in obs:
+            for tag in obs['tags']:
+                tag_parts = tag.split(':')
+                if len(tag_parts) == 2:
+                    tag_info.append(f"*{tag_parts[0]}:* {tag_parts[1]}")
+
+        tags_message = "\n".join(tag_info) if tag_info else "No tags"
+        message = f"**More information(s)**:\n{tags_message}"
+
         observable_data = {
             "dataType": obs['dataType'],
             "data": obs['data'],
-            "message": f"An observable was added on {current_date} at {current_time}.",
+            "message": message,
             "ioc": True,
             "sighted": True,
-            "tlp": 2
+            "tlp": 1,
+            "pap": 1
         }
+
+        if 'tags' in obs:
+            observable_data['tags'] = obs['tags']
+
         observables_data.append(observable_data)
+
     return observables_data
 
 
@@ -140,7 +156,7 @@ def update_existing_alert_case(item_type, existing_item, observables, comment, t
         add_comment_to_item(item_type, item_id, comment, thehive_url, api_key)
 
 
-def create_new_alert(ticket_id, title, description, severity, tags, app_name, observables, customFields, comment, thehive_url, api_key):
+def create_new_alert(ticket_id, title, description, severity, tlp, pap, tags, app_name, observables, customFields, comment, thehive_url, api_key):
     from common.core import generate_ref
     """
     Create a new alert in TheHive with the provided details.
@@ -149,6 +165,8 @@ def create_new_alert(ticket_id, title, description, severity, tags, app_name, ob
     :param title: The title of the alert.
     :param description: The description of the alert.
     :param severity: The severity level of the alert (integer).
+    :param tlp: The Traffic Light Protocol (TLP) level of the alert (integer).
+    :param pap: The Permissible Action Protocol (PAP) level of the alert (integer).
     :param tags: A list of tags associated with the alert.
     :param app_name: The application triggering the alert.
     :param observables: A list of observables to associate with the alert.
@@ -166,6 +184,8 @@ def create_new_alert(ticket_id, title, description, severity, tags, app_name, ob
         "title": title,
         "description": description,
         "severity": severity,
+        "tlp": tlp,
+        "pap": pap,
         "tags": tags,
         "type": app_name,
         "source": "watcher",
@@ -201,7 +221,7 @@ def create_new_alert(ticket_id, title, description, severity, tags, app_name, ob
         return None
 
 
-def handle_alert_or_case(ticket_id, observables, comment, title, description, severity, tags, app_name, customFields, thehive_url, api_key):
+def handle_alert_or_case(ticket_id, observables, comment, title, description, severity, tlp, pap, tags, app_name, customFields, thehive_url, api_key):
     """
     Handle the creation or updating of alerts and cases in TheHive.
 
@@ -211,6 +231,8 @@ def handle_alert_or_case(ticket_id, observables, comment, title, description, se
     :param title: The title for the alert.
     :param description: The description for the alert.
     :param severity: The severity of the alert (integer).
+    :param tlp: The Traffic Light Protocol (TLP) level of the alert (integer).
+    :param pap: The Permissible Action Protocol (PAP) level of the alert (integer).
     :param tags: A list of tags for the alert.
     :param app_name: The name of the application triggering the alert.
     :param customFields: Custom fields to be included in the alert.
@@ -235,6 +257,6 @@ def handle_alert_or_case(ticket_id, observables, comment, title, description, se
     else:
         create_new_alert(
             ticket_id=ticket_id, title=title, description=description, severity=severity, 
-            tags=tags, app_name=app_name, observables=observables, 
+            tlp=tlp, pap=pap, tags=tags, app_name=app_name, observables=observables, 
             customFields=customFields, comment=comment, thehive_url=thehive_url, api_key=api_key
         )

Watcher/Watcher/watcher/settings.py

@@ -126,6 +126,7 @@
 THE_HIVE_KEY = os.environ.get('THE_HIVE_KEY', '')
 THE_HIVE_CUSTOM_FIELD = os.environ.get('THE_HIVE_CUSTOM_FIELD', 'watcher-id')
 THE_HIVE_EMAIL_SENDER = os.environ.get('THE_HIVE_EMAIL_SENDER', '[email protected]')
+THE_HIVE_TAGS = os.environ.get('THE_HIVE_TAGS', "Watcher,Impersonation,Malicious Domain,Typosquatting").split(",")
 
 # MISP Setup
 MISP_URL = os.environ.get('MISP_URL', 'https://127.0.0.1')

Watcher/docs/_build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: d2d6539df681e8aa87a57ee80d749b7a
+config: a65999e63ea13b7ea219f9784a0a743b
 tags: 645f666f9bcd5a90fca523b33c5a78b7

Watcher/docs/_build/html/.buildinfo.bak

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
-# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: e4eed274a380d35a6ece8892666d1ca9
+# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: d2d6539df681e8aa87a57ee80d749b7a
 tags: 645f666f9bcd5a90fca523b33c5a78b7

Watcher/docs/_build/html/README.html

@@ -6,15 +6,15 @@
   <meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Install Watcher &mdash; Watcher 2.1 documentation</title>
+  <title>Install Watcher &mdash; Watcher 2.1.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
       <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
 
 
     <link rel="shortcut icon" href="_static/Watcher-favicon.ico"/>
       <script src="_static/jquery.js?v=5d32c60e"></script>
       <script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="_static/documentation_options.js?v=c4a36953"></script>
+      <script src="_static/documentation_options.js?v=841abef3"></script>
       <script src="_static/doctools.js?v=9bcbadda"></script>
       <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/js/theme.js"></script>
@@ -369,7 +369,7 @@ <h3>Configure your Email notifications<a class="headerlink" href="#configure-you
 <li><p>Choose your email provider (example: Gmail, Outlook…).</p></li>
 <li><p>Go to the email provider’s settings and generate the <strong>SMTP configuration</strong>:</p>
 <ul class="simple">
-<li><p>For <strong>Gmail</strong>, detailed instructions can be found in <a class="reference external" href="https://support.google.com/a/answer/176600?hl=en">Google SMTP documentation</a>.</p></li>
+<li><p>For <strong>Gmail</strong>, detailed instructions can be found in <a class="reference external" href="https://support.google.com/a/answer/176600?hl=en">Google’s SMTP documentation</a>.</p></li>
 <li><p>For <strong>Outlook</strong>, you can refer to the <a class="reference external" href="https://support.microsoft.com/en-us/office/pop-imap-and-smtp-settings-for-outlook-com-d088b986-291d-42b8-9564-9c414e2aa040">Outlook SMTP documentation</a> for more information.</p></li>
 </ul>
 </li>
@@ -461,6 +461,7 @@ <h3>Configure your Citadel notifications<a class="headerlink" href="#configure-y
 </div>
 <p>Follow these steps to get the required information:</p>
 <ol class="arabic simple">
+<li><p>If you don’t have an account, go to this link to create one: Citadel Team Documentation.</p></li>
 <li><p>Create a <strong>New Room</strong>.</p></li>
 <li><p>Retrieve the <code class="docutils literal notranslate"><span class="pre">CITADEL_ROOM_ID</span></code> from the room’s settings. Copy the room’s link, then extract the ID after <code class="docutils literal notranslate"><span class="pre">/#/room/</span></code> and add it to your .env file.</p></li>
 <li><p>Next, visit this link: <a class="reference external" href="https://cds.thalesgroup.com/en/ercom/citadel">Citadel Team Website</a> to request your <code class="docutils literal notranslate"><span class="pre">CITADEL_API_TOKEN</span></code>. This token will allow you to send automatic notifications.</p></li>

Watcher/docs/_build/html/_sources/README.md.txt

@@ -203,7 +203,7 @@ Follow these steps to get the required information:
 
 1. Choose your email provider (example: Gmail, Outlook...).
 2. Go to the email provider’s settings and generate the **SMTP configuration**:
-   - For **Gmail**, detailed instructions can be found in [Google SMTP documentation](https://support.google.com/a/answer/176600?hl=en).
+   - For **Gmail**, detailed instructions can be found in [Google's SMTP documentation](https://support.google.com/a/answer/176600?hl=en).
    - For **Outlook**, you can refer to the [Outlook SMTP documentation](https://support.microsoft.com/en-us/office/pop-imap-and-smtp-settings-for-outlook-com-d088b986-291d-42b8-9564-9c414e2aa040) for more information.
 3. Follow the instructions to retrieve the SMTP server, email port, and other necessary credentials.
 4. Save these values in your `.env` file.
@@ -288,10 +288,11 @@ To configure Citadel, you need the following variables, in the `.env` file:
 
 Follow these steps to get the required information:
 
-1. Create a **New Room**.
-2. Retrieve the `CITADEL_ROOM_ID` from the room's settings. Copy the room's link, then extract the ID after `/#/room/` and add it to your .env file.
-3. Next, visit this link: [Citadel Team Website](https://cds.thalesgroup.com/en/ercom/citadel) to request your `CITADEL_API_TOKEN`. This token will allow you to send automatic notifications.
-4. For the `CITADEL_URL` variable, if you're using a public instance, the URL should be: [https://join.citadel.team/](https://join.citadel.team/). Otherwise, enter your customized instance URL.
+1. If you don't have an account, go to this link to create one: Citadel Team Documentation.
+2. Create a **New Room**.
+3. Retrieve the `CITADEL_ROOM_ID` from the room's settings. Copy the room's link, then extract the ID after `/#/room/` and add it to your .env file.
+4. Next, visit this link: [Citadel Team Website](https://cds.thalesgroup.com/en/ercom/citadel) to request your `CITADEL_API_TOKEN`. This token will allow you to send automatic notifications.
+5. For the `CITADEL_URL` variable, if you're using a public instance, the URL should be: [https://join.citadel.team/](https://join.citadel.team/). Otherwise, enter your customized instance URL.
 
 Now, you can restart your instance and the parameters will be taken into account:
 

Watcher/docs/_build/html/_static/documentation_options.js

@@ -1,5 +1,5 @@
 const DOCUMENTATION_OPTIONS = {
-    VERSION: '2.1',
+    VERSION: '2.1.1',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

Watcher/docs/_build/html/genindex.html

@@ -5,15 +5,15 @@
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Index &mdash; Watcher 2.1 documentation</title>
+  <title>Index &mdash; Watcher 2.1.1 documentation</title>
       <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
       <link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
 
 
     <link rel="shortcut icon" href="_static/Watcher-favicon.ico"/>
       <script src="_static/jquery.js?v=5d32c60e"></script>
       <script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="_static/documentation_options.js?v=c4a36953"></script>
+      <script src="_static/documentation_options.js?v=841abef3"></script>
       <script src="_static/doctools.js?v=9bcbadda"></script>
       <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
     <script src="_static/js/theme.js"></script>
@@ -84,7 +84,6 @@ <h1 id="index">Index</h1>
 <div class="genindex-jumpbox">
  <a href="#C"><strong>C</strong></a>
  | <a href="#F"><strong>F</strong></a>
- | <a href="#G"><strong>G</strong></a>
  | <a href="#I"><strong>I</strong></a>
  | <a href="#L"><strong>L</strong></a>
  | <a href="#M"><strong>M</strong></a>
@@ -110,11 +109,11 @@ <h2 id="C">C</h2>
       <li><a href="modules/site_monitoring_core.html#Watcher.site_monitoring.core.check_mail">check_mail() (in module Watcher.site_monitoring.core)</a>
 </li>
       <li><a href="modules/data_leak_core.html#Watcher.data_leak.core.check_pastebin">check_pastebin() (in module Watcher.data_leak.core)</a>
-</li>
-      <li><a href="modules/data_leak_core.html#Watcher.data_leak.core.check_searx">check_searx() (in module Watcher.data_leak.core)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="modules/data_leak_core.html#Watcher.data_leak.core.check_searx">check_searx() (in module Watcher.data_leak.core)</a>
+</li>
       <li><a href="modules/data_leak_core.html#Watcher.data_leak.core.check_urls">check_urls() (in module Watcher.data_leak.core)</a>
 </li>
       <li><a href="modules/data_leak_core.html#Watcher.data_leak.core.cleanup">cleanup() (in module Watcher.data_leak.core)</a>
@@ -123,8 +122,6 @@ <h2 id="C">C</h2>
         <li><a href="modules/threats_watcher_core.html#Watcher.threats_watcher.core.cleanup">(in module Watcher.threats_watcher.core)</a>
 </li>
       </ul></li>
-      <li><a href="modules/common_core.html#Watcher.common.core.collect_observables">collect_observables() (in module Watcher.common.core)</a>
-</li>
       <li><a href="modules/site_monitoring_core.html#Watcher.site_monitoring.core.create_alert">create_alert() (in module Watcher.site_monitoring.core)</a>
 </li>
       <li><a href="modules/site_monitoring_misp.html#Watcher.site_monitoring.misp.create_attributes">create_attributes() (in module Watcher.site_monitoring.misp)</a>
@@ -148,14 +145,6 @@ <h2 id="F">F</h2>
   </ul></td>
 </tr></table>
 
-<h2 id="G">G</h2>
-<table style="width: 100%" class="indextable genindextable"><tr>
-  <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="modules/common_core.html#Watcher.common.core.generate_ref">generate_ref() (in module Watcher.common.core)</a>
-</li>
-  </ul></td>
-</tr></table>
-
 <h2 id="I">I</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
@@ -187,8 +176,6 @@ <h2 id="M">M</h2>
     module
 
       <ul>
-        <li><a href="modules/common_core.html#module-Watcher.common.core">Watcher.common.core</a>
-</li>
         <li><a href="modules/data_leak_core.html#module-Watcher.data_leak.core">Watcher.data_leak.core</a>
 </li>
         <li><a href="modules/dns_finder_core.html#module-Watcher.dns_finder.core">Watcher.dns_finder.core</a>
@@ -225,10 +212,6 @@ <h2 id="R">R</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="modules/threats_watcher_core.html#Watcher.threats_watcher.core.remove_banned_words">remove_banned_words() (in module Watcher.threats_watcher.core)</a>
-</li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="modules/common_core.html#Watcher.common.core.remove_html_tags">remove_html_tags() (in module Watcher.common.core)</a>
 </li>
   </ul></td>
 </tr></table>
@@ -237,10 +220,6 @@ <h2 id="S">S</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="modules/site_monitoring_misp.html#Watcher.site_monitoring.misp.search_attributes">search_attributes() (in module Watcher.site_monitoring.misp)</a>
-</li>
-      <li><a href="modules/common_core.html#Watcher.common.core.send_app_specific_notifications">send_app_specific_notifications() (in module Watcher.common.core)</a>
-</li>
-      <li><a href="modules/common_core.html#Watcher.common.core.send_app_specific_notifications_group">send_app_specific_notifications_group() (in module Watcher.common.core)</a>
 </li>
       <li><a href="modules/data_leak_core.html#Watcher.data_leak.core.send_data_leak_notifications">send_data_leak_notifications() (in module Watcher.data_leak.core)</a>
 </li>
@@ -252,8 +231,6 @@ <h2 id="S">S</h2>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="modules/common_core.html#Watcher.common.core.send_only_thehive_notifications">send_only_thehive_notifications() (in module Watcher.common.core)</a>
-</li>
       <li><a href="modules/threats_watcher_core.html#Watcher.threats_watcher.core.send_threats_watcher_notifications">send_threats_watcher_notifications() (in module Watcher.threats_watcher.core)</a>
 </li>
       <li><a href="modules/site_monitoring_core.html#Watcher.site_monitoring.core.send_website_monitoring_notifications">send_website_monitoring_notifications() (in module Watcher.site_monitoring.core)</a>
@@ -295,13 +272,6 @@ <h2 id="W">W</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li>
-    Watcher.common.core
-
-      <ul>
-        <li><a href="modules/common_core.html#module-Watcher.common.core">module</a>
-</li>
-      </ul></li>
-      <li>
     Watcher.data_leak.core
 
       <ul>
@@ -315,15 +285,15 @@ <h2 id="W">W</h2>
         <li><a href="modules/dns_finder_core.html#module-Watcher.dns_finder.core">module</a>
 </li>
       </ul></li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
       <li>
     Watcher.site_monitoring.core
 
       <ul>
         <li><a href="modules/site_monitoring_core.html#module-Watcher.site_monitoring.core">module</a>
 </li>
       </ul></li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
       <li>
     Watcher.site_monitoring.misp