Merge pull request #261 from CybercentreCanada/feature/whitelist

Feature/whitelist (dev)

Author: cccs-sgaron

assemblyline/cachestore/__init__.py

@@ -21,7 +21,7 @@ def __init__(self, component, config=None, datastore=None):
             config = forge.get_config()
 
         self.component = component
-        self.datastore = datastore or forge.get_datastore()
+        self.datastore = datastore or forge.get_datastore(config=config)
         self.filestore = FileStore(*config.filestore.cache)
 
     def __enter__(self):
@@ -30,15 +30,15 @@ def __enter__(self):
     def __exit__(self, ex_type, exc_val, exc_tb):
         self.filestore.close()
 
-    def save(self, cache_key, data, ttl=DEFAULT_CACHE_LEN):
+    def save(self, cache_key, data, ttl=DEFAULT_CACHE_LEN, force=False):
         if not COMPONENT_VALIDATOR.match(cache_key):
             raise ValueError("Invalid cache_key for cache item. "
                              "(Only letters, numbers, underscores and dots allowed)")
 
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
 
         self.datastore.cached_file.save(new_key, {'expiry_ts': now_as_iso(ttl), 'component': self.component})
-        self.filestore.put(new_key, data)
+        self.filestore.put(new_key, data, force=force)
 
     def get(self, cache_key):
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key

assemblyline/common/forge.py

@@ -138,31 +138,43 @@ def get_service_queue(service: str, redis=None):
     return PriorityQueue(service_queue_name(service), redis)
 
 
-def get_tag_whitelister(log=None, yml_config=None):
-    from assemblyline.common.tagging import TagWhitelister, InvalidWhitelist
+def get_tag_safelist_data(yml_config=None):
 
     if yml_config is None:
-        yml_config = "/etc/assemblyline/tag_whitelist.yml"
+        yml_config = "/etc/assemblyline/tag_safelist.yml"
 
-    tag_whitelist_data = {}
-    default_file = os.path.join(os.path.dirname(__file__), "tag_whitelist.yml")
+    tag_safelist_data = {}
+    default_file = os.path.join(os.path.dirname(__file__), "tag_safelist.yml")
     if os.path.exists(default_file):
         with open(default_file) as default_fh:
             default_yml_data = yaml.safe_load(default_fh.read())
             if default_yml_data:
-                tag_whitelist_data.update(default_yml_data)
+                tag_safelist_data.update(default_yml_data)
 
     # Load modifiers from the yaml config
     if os.path.exists(yml_config):
         with open(yml_config) as yml_fh:
             yml_data = yaml.safe_load(yml_fh.read())
             if yml_data:
-                tag_whitelist_data = recursive_update(tag_whitelist_data, yml_data)
+                tag_safelist_data = recursive_update(tag_safelist_data, yml_data)
 
-    if not tag_whitelist_data:
-        raise InvalidWhitelist('Could not find any tag_whitelist file to load.')
+    return tag_safelist_data
 
-    return TagWhitelister(tag_whitelist_data, log=log)
+
+def get_tag_safelister(log=None, yml_config=None, config=None, datastore=None):
+    from assemblyline.common.tagging import TagSafelister, InvalidSafelist
+
+    with get_cachestore('system', config=config, datastore=datastore) as cache:
+        tag_safelist_yml = cache.get('tag_safelist_yml')
+        if tag_safelist_yml:
+            tag_safelist_data = yaml.safe_load(tag_safelist_yml)
+        else:
+            tag_safelist_data = get_tag_safelist_data(yml_config=yml_config)
+
+    if not tag_safelist_data:
+        raise InvalidSafelist('Could not find any tag_safelist file to load.')
+
+    return TagSafelister(tag_safelist_data, log=log)
 
 
 class CachedObject:

assemblyline/common/tag_whitelist.yml assemblyline/common/tag_safelist.yml

@@ -1,21 +1,21 @@
-# Default tag_whitelist.yml file
+# Default tag_safelist.yml file
 #
-#    The following tags are whitelisted:
+#    The following tags are safelisted:
 #     - Domains pointing to localhost
 #     - Domain commonly found in XML files, certificates and during dynamic Analysis runs
 #     - IPs in the private network IP space
 #     - URI pointing to IPs in the private network IP space
 #     - URIs commonly found in XML files, certificates and during dynamic Analysis runs
 #
-#    Note: - You can override the default tag_whitelist.yml by putting an
-#            updated version in /etc/assemblyline/tag_whitelist.yml.
+#    Note: - You can override the default tag_safelist.yml by putting an
+#            updated version in /etc/assemblyline/tag_safelist.yml.
 #          - If you want to add values to one of the following tag types,
 #            you have to copy the default values to the new file.
 #          - You can nullify value by putting empty object or empty list
 #            in your new file
 
 # Match section contains tag types and for each tag type
-#  a list of values that should be whitelisted using a direct
+#  a list of values that should be safelisted using a direct
 #  string comparison.
 match:
   # Direct match to dynamic domains
@@ -64,18 +64,18 @@ match:
     - purl.org
 
 # Regex section contains tag types and for each tag type
-#  a list of regular expression to be run to whitelist
+#  a list of regular expression to be run to safelist
 #  the associated tags.
 regex:
-  # Regular expression to whitelist dynamic IPs (Private IPs)
+  # Regular expression to safelist dynamic IPs (Private IPs)
   #  note: Since IPs have already been validated, the regular expression in simpler
   network.dynamic.ip:
     - (?:127\.|10\.|192\.168|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[01]\.).*
-  # Regular expression to whitelist static IPs (Private IPs)
+  # Regular expression to safelist static IPs (Private IPs)
   #  note: Since IPs have already been validated, the regular expression in simpler
   network.static.ip:
     - (?:127\.|10\.|192\.168|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[01]\.).*
-  # Regular expression to whitelist dynamic URIs
+  # Regular expression to safelist dynamic URIs
   network.dynamic.uri:
     - (?:ftp|http)s?://localhost(?:$|/.*)
     - (?:ftp|http)s?://(?:(?:(?:10|127)(?:\.(?:[2](?:[0-5][0-5]|[01234][6-9])|[1][0-9][0-9]|[1-9][0-9]|[0-9])){3})|(?:172\.(?:1[6-9]|2[0-9]|3[0-1])(?:\.(?:2[0-4][0-9]|25[0-5]|[1][0-9][0-9]|[1-9][0-9]|[0-9])){2}|(?:192\.168(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){2})))(?:$|/.*)
@@ -110,7 +110,7 @@ regex:
     - https?://ns\.adobe\.com/xap/1\.0/sType/ResourceEvent#
     - https?://purl\.org/dc/elements/1\.1/
     - https?://www\.w3\.org/1999/02/22-rdf-syntax-ns#
-  # Regular expression to whitelist static URIs
+  # Regular expression to safelist static URIs
   network.static.uri:
     - (?:ftp|http)s?://localhost(?:$|/.*)
     - (?:ftp|http)s?://(?:(?:(?:10|127)(?:\.(?:[2](?:[0-5][0-5]|[01234][6-9])|[1][0-9][0-9]|[1-9][0-9]|[0-9])){3})|(?:172\.(?:1[6-9]|2[0-9]|3[0-1])(?:\.(?:2[0-4][0-9]|25[0-5]|[1][0-9][0-9]|[1-9][0-9]|[0-9])){2}|(?:192\.168(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])){2})))(?:$|/.*)
@@ -144,4 +144,4 @@ regex:
     - https?://ns\.adobe\.com/xap/1\.0/mm/
     - https?://ns\.adobe\.com/xap/1\.0/sType/ResourceEvent#
     - https?://purl\.org/dc/elements/1\.1/
-    - https?://www\.w3\.org/1999/02/22-rdf-syntax-ns#
+    - https?://www\.w3\.org/1999/02/22-rdf-syntax-ns#

assemblyline/common/tagging.py

@@ -1,8 +1,8 @@
 import re
 
-from typing import List, Dict
+from typing import List, Dict, Set
 
-from assemblyline.common.dict_utils import flatten
+from assemblyline.common.forge import CachedObject, get_datastore
 from assemblyline.odm.models.tagging import Tagging
 
 
@@ -16,22 +16,33 @@ def tag_list_to_dict(tag_list: List[Dict]) -> Dict:
     return tag_dict
 
 
-def tag_dict_to_list(tag_dict: Dict) -> List[Dict]:
+def tag_dict_to_list(tag_dict: Dict, safelisted: bool = False) -> List[Dict]:
     return [
-        {'type': k, 'value': t, 'short_type': k.rsplit(".", 1)[-1]}
-        for k, v in flatten(tag_dict).items()
+        {'safelisted': safelisted, 'type': k, 'value': t, 'short_type': k.rsplit(".", 1)[-1]}
+        for k, v in tag_dict.items()
         if v is not None
         for t in v
     ]
 
 
-class InvalidWhitelist(Exception):
+def get_safelist_key(t_type: str, t_value: str) -> str:
+    return f"{t_type}__{t_value}"
+
+
+def get_safelist(ds) -> Set:
+    return {get_safelist_key(sl['tag']['type'], sl['tag']['value']): True
+            for sl in ds.safelist.stream_search("type:tag AND enabled:true", as_obj=False)}
+
+
+class InvalidSafelist(Exception):
     pass
 
 
-class TagWhitelister(object):
+class TagSafelister(object):
     def __init__(self, data, log=None):
         valid_tags = set(Tagging.flat_fields().keys())
+        self.datastore = get_datastore()
+        self.safelist = CachedObject(get_safelist, kwargs={'ds': self.datastore}, refresh=300)
 
         self.match = data.get('match', {})
         self.regex = data.get('regex', {})
@@ -40,38 +51,62 @@ def __init__(self, data, log=None):
         # Validate matches and regex
         for section, item in {'match': self.match, 'regex': self.regex}.items():
             if not isinstance(item, dict):
-                raise InvalidWhitelist(f"Section {section} should be of type: DICT")
+                raise InvalidSafelist(f"Section {section} should be of type: DICT")
 
             for k, v in item.items():
                 if not isinstance(v, list):
-                    raise InvalidWhitelist(f"Values in the {section} section should all be of type: LIST")
+                    raise InvalidSafelist(f"Values in the {section} section should all be of type: LIST")
 
                 if k not in valid_tags:
-                    raise InvalidWhitelist(f"Key ({k}) in the {section} section is not a valid tag.")
+                    raise InvalidSafelist(f"Key ({k}) in the {section} section is not a valid tag.")
 
                 if section == 'regex':
                     self.regex[k] = [re.compile(x) for x in v]
 
-    def is_whitelisted(self, t_type, t_value):
+    def is_safelisted(self, t_type, t_value):
+        if self.safelist.get(get_safelist_key(t_type, t_value), False):
+            if self.log:
+                self.log.info(f"Tag '{t_type}' with value '{t_value}' was safelisted.")
+            return True
+
         for match in self.match.get(t_type, []):
             if t_value == match:
                 if self.log:
-                    self.log.info(f"Tag '{t_type}' with value '{t_value}' was whitelisted by match rule.")
+                    self.log.info(f"Tag '{t_type}' with value '{t_value}' was safelisted by match rule.")
                 return True
 
         for regex in self.regex.get(t_type, []):
             if regex.match(t_value):
                 if self.log:
                     self.log.info(f"Tag '{t_type}' with value '{t_value}' "
-                                  f"was whitelisted by regex '{regex.pattern}'.")
+                                  f"was safelisted by regex '{regex.pattern}'.")
                 return True
 
         return False
 
-    def whitelist_many(self, t_type, t_values):
+    def safelist_many(self, t_type, t_values):
         if not isinstance(t_values, list):
             t_values = [t_values]
-        return [x for x in t_values if not self.is_whitelisted(t_type, x)]
+
+        tags = []
+        safelisted_tags = []
+        for x in t_values:
+            if self.is_safelisted(t_type, x):
+                safelisted_tags.append(x)
+            else:
+                tags.append(x)
+
+        return tags, safelisted_tags
 
     def get_validated_tag_map(self, tag_map):
-        return {k: self.whitelist_many(k, v) for k, v in tag_map.items() if v is not None}
+        tags = {}
+        safelisted_tags = {}
+        for k, v in tag_map.items():
+            if v is not None and v != []:
+                c_tags, c_safelisted_tags = self.safelist_many(k, v)
+                if c_tags:
+                    tags[k] = c_tags
+                if c_safelisted_tags:
+                    safelisted_tags[k] = c_safelisted_tags
+
+        return tags, safelisted_tags

assemblyline/datastore/helper.py

@@ -32,6 +32,7 @@
 from assemblyline.odm.models.user import User
 from assemblyline.odm.models.user_favorites import UserFavorites
 from assemblyline.odm.models.user_settings import UserSettings
+from assemblyline.odm.models.safelist import Safelist
 from assemblyline.odm.models.workflow import Workflow
 from assemblyline.remote.datatypes.lock import Lock
 
@@ -60,6 +61,7 @@ def __init__(self, datastore_object):
         self.ds.register('user_avatar')
         self.ds.register('user_favorites', UserFavorites)
         self.ds.register('user_settings', UserSettings)
+        self.ds.register('safelist', Safelist)
         self.ds.register('workflow', Workflow)
 
     def __enter__(self):
@@ -160,6 +162,10 @@ def user_settings(self) -> Collection:
     def vm(self) -> Collection:
         return self.ds.vm
 
+    @property
+    def safelist(self) -> Collection:
+        return self.ds.safelist
+
     @property
     def workflow(self) -> Collection:
         return self.ds.workflow
@@ -735,6 +741,7 @@ def get_summary_from_keys(self, keys, cl_engine=forge.get_classification(), user
             "attack_matrix": [],
             "heuristics": {
                 "info": [],
+                "safe": [],
                 "suspicious": [],
                 "malicious": []
             },
@@ -785,7 +792,9 @@ def get_summary_from_keys(self, keys, cl_engine=forge.get_classification(), user
 
                 if section.get('heuristic', False):
                     # Get the heuristics data
-                    if section['heuristic']['score'] < 100:
+                    if section['heuristic']['score'] < 0:
+                        h_type = "safe"
+                    elif section['heuristic']['score'] < 100:
                         h_type = "info"
                     elif section['heuristic']['score'] < 1000:
                         h_type = "suspicious"
@@ -828,7 +837,25 @@ def get_summary_from_keys(self, keys, cl_engine=forge.get_classification(), user
                                     'h_type': h_type,
                                     'short_type': tag_type.rsplit(".", 1)[-1],
                                     'value': tag,
-                                    'key': key
+                                    'key': key,
+                                    'safelisted': False
+                                })
+                                done_map['tags'].add(cache_key)
+
+                # Get safelisted tag data
+                for tag_type, tags in section.get('safelisted_tags', {}).items():
+                    if tags is not None:
+                        for tag in tags:
+                            cache_key = f"{tag_type}_{tag}_{key}"
+
+                            if cache_key not in done_map['tags']:
+                                out['tags'].append({
+                                    'type': tag_type,
+                                    'h_type': h_type,
+                                    'short_type': tag_type.rsplit(".", 1)[-1],
+                                    'value': tag,
+                                    'key': key,
+                                    'safelisted': True
                                 })
                                 done_map['tags'].add(cache_key)
 
@@ -851,7 +878,18 @@ def get_tag_list_from_keys(self, keys):
                                 'type': tag_type,
                                 'short_type': tag_type.rsplit(".", 1)[-1],
                                 'value': tag,
-                                'key': key
+                                'key': key,
+                                'safelisted': False
+                            })
+                for tag_type, tags in section.get('safelisted_tags', {}).items():
+                    if tags is not None:
+                        for tag in tags:
+                            out.append({
+                                'type': tag_type,
+                                'short_type': tag_type.rsplit(".", 1)[-1],
+                                'value': tag,
+                                'key': key,
+                                'safelisted': True
                             })
 
         return out