Implementing system-level safelist

metadefender.py

@@ -8,6 +8,7 @@
 
 from assemblyline.common.exceptions import RecoverableError
 from assemblyline.common.isotime import iso_to_local, iso_to_epoch, epoch_to_local, now, now_as_local
+from assemblyline_v4_service.common.api import ServiceAPIError
 from assemblyline_v4_service.common.base import ServiceBase
 from assemblyline_v4_service.common.request import ServiceRequest
 from assemblyline_v4_service.common.result import Result, ResultSection, Classification, BODY_FORMAT, Heuristic
@@ -69,10 +70,17 @@ def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
         self.blocklist: Optional[List[str]] = None
         self.kw_score_revision_map: Optional[Dict[str, int]] = None
         self.sig_score_revision_map: Optional[Dict[str, Any]] = None
+        self.safelist_match: List[str] = []
         api_key = self.config.get("api_key")
         if api_key:
             self.headers = {"apikey": api_key}
 
+        try:
+            safelist = self.get_api_interface().get_safelist(["av.virus_name"])
+            [self.safelist_match.extend(match_list) for _, match_list in safelist.get('match', {}).items()]
+        except ServiceAPIError as e:
+            self.log.warning(f"Couldn't retrieve safelist from service: {e}. Continuing without it..")
+
     def start(self) -> None:
         self.log.debug("MetaDefender service started")
         base_urls: List[str] = []
@@ -395,6 +403,8 @@ def parse_results(self, response: Dict[str, Any]) -> Result:
 
                 if heur_id is not None:
                     virus_name = virus_name.replace("a variant of ", "")
+                    if virus_name in self.safelist_match:
+                        continue
                     engine = self.nodes[self.current_node]['engine_map'][self._format_engine_name(majorkey)]
                     av_hit_section = AvHitSection(majorkey, virus_name, engine, heur_id,
                                                   self.sig_score_revision_map, self.kw_score_revision_map)

service_manifest.yml

@@ -37,13 +37,14 @@ config:
     # Signature names are not valid YAML keys according to the Assemblyline
     # ODM so we cannot use them in the heuristic signature_score_map. Hence why we're putting this here.
     sig_score_revision_map:
-      Ikarus.Trojan-Downloader.MSWord.Agent: 0
-      Ikarus.Trojan-Downloader.VBA.Agent: 0
-      NANOAV.Exploit.Xml.CVE-2017-0199.equmby: 0
       TACHYON.Suspicious/XOX.Obfus.Gen.2: 100
-      TACHYON.Suspicious/XOX.Obfus.Gen.3: 0
-      Vir.IT eXplorer.Office.VBA_Macro_Heur: 0
-      Vir.IT eXplorer.W97M/Downloader.AB: 0
+      # Note the following should be added to the system-level safelist since their revised scores are 0
+#      Ikarus.Trojan-Downloader.MSWord.Agent: 0
+#      Ikarus.Trojan-Downloader.VBA.Agent: 0
+#      NANOAV.Exploit.Xml.CVE-2017-0199.equmby: 0
+#      TACHYON.Suspicious/XOX.Obfus.Gen.3: 0
+#      Vir.IT eXplorer.Office.VBA_Macro_Heur: 0
+#      Vir.IT eXplorer.W97M/Downloader.AB: 0
 
 heuristics:
   - heur_id: 1