Add externals to TagCheck, fix handling them

TagCheck support now the default YARA_EXTERNALS together
with tag data. The usage of YARA_EXTERNALS was unified across
all classes. In collecting values of externals, fixed using keys
with prefix against original fields as well as attempts to
get values from __dict__, which does not contain properties

Closes CybercentreCanada/assemblyline#269

tagcheck/tagcheck.py

@@ -1,9 +1,12 @@
 from assemblyline.odm.models.tagging import Tagging
 
+from yara_.helper import YARA_EXTERNALS
 from yara_.yara_ import Yara
 
+tags_ext = list(Tagging.flat_fields().keys())
+TAGCHECK_EXTERNALS = [*tags_ext, *YARA_EXTERNALS]
+
 
 class TagCheck(Yara):
     def __init__(self, config=None):
-        externals = list(Tagging.flat_fields().keys())
-        super().__init__(config, externals=externals)
+        super().__init__(config, externals=TAGCHECK_EXTERNALS)

tagcheck/update_server.py

@@ -1,8 +1,8 @@
-from assemblyline.odm.models.tagging import Tagging
+from tagcheck.tagcheck import TAGCHECK_EXTERNALS
 from yara_.update_server import YaraUpdateServer
 
-YARA_EXTERNALS = {f'al_{x.replace(".", "_")}': '' for x in list(Tagging.flat_fields().keys())}
-
-if __name__ == '__main__':
-    with YaraUpdateServer(externals=YARA_EXTERNALS, default_pattern=".*\.rules") as server:
+if __name__ == "__main__":
+    with YaraUpdateServer(
+        externals=TAGCHECK_EXTERNALS, default_pattern=".*\.rules"
+    ) as server:
         server.serve_forever()

yara_/helper.py

@@ -10,7 +10,7 @@
 
 DEFAULT_STATUS = "DEPLOYED"
 Classification = forge.get_classification()
-YARA_EXTERNALS = {f"al_{x}": x for x in ["submitter", "mime", "file_type", "tag"]}
+YARA_EXTERNALS = ["submitter", "mime", "file_type", "tag", "file_name"]
 
 
 class YaraImporter(object):

yara_/update_server.py

@@ -79,9 +79,9 @@ def replace_include(include, dirname, processed_files: set[str], cur_logger: log
 
 
 class YaraUpdateServer(ServiceUpdater):
-    def __init__(self, *args, externals: dict[str, str], **kwargs):
+    def __init__(self, *args, externals: list[str], **kwargs):
         super().__init__(*args, **kwargs)
-        self.externals = externals
+        self.externals = {f'al_{x.replace(".", "_")}': "" for x in externals}
 
     # A sanity check to make sure we do in fact have things to send to services
     def _inventory_check(self) -> bool:

yara_/yara_.py

@@ -450,16 +450,19 @@ def execute(self, request):
 
         yara_externals = {}
         for k in self.yara_externals.keys():
+            # Externals are always prepended with al_
+            clean_key = k[3:]
+
             # Check default request.task fields
-            sval = request.task.__dict__.get(k, None)
+            sval = getattr(request.task,clean_key, None)
 
             # if not sval:
             #     # Check metadata dictionary
             #     sval = request.task.metadata.get(k, None)
 
             if not sval:
                 # Check params dictionary
-                sval = request.task.service_config.get(k, None)
+                sval = request.task.service_config.get(clean_key, None)
 
             if not sval:
                 # Check tags list
@@ -469,7 +472,7 @@ def execute(self, request):
 
             if not sval:
                 # Check temp submission data
-                sval = request.task.temp_submission_data.get(k, None)
+                sval = request.task.temp_submission_data.get(clean_key, None)
 
             # Normalize unicode with safe_str and make sure everything else is a string
             if sval: