Feature/SubmissionProfiles (master)

assemblyline/odm/models/config.py

@@ -2,8 +2,10 @@
 
 from assemblyline import odm
 from assemblyline.common.constants import PRIORITIES
+from assemblyline.common.forge import get_classification
 from assemblyline.odm.models.service import EnvironmentVariable
 from assemblyline.odm.models.service_delta import DockerConfigDelta
+from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, ServiceSelection
 
 AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group',
                       'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota',
@@ -16,6 +18,8 @@
 DEFAULT_SUBMISSION_QUOTA = 5
 DEFAULT_ASYNC_SUBMISSION_QUOTA = 0
 
+Classification = get_classification()
+
 
 @odm.model(index=False, store=False, description="Password Requirement")
 class PasswordRequirement(odm.Model):
@@ -211,11 +215,14 @@ class OAuthProvider(odm.Model):
     redirect_uri: str = odm.Optional(odm.Keyword(),
                                      description="URI to redirect to after authentication with OAuth provider")
     request_token_url: str = odm.Optional(odm.Keyword(), description="URL to request token")
-    request_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters to request token")
+    request_token_params: Dict[str, str] = odm.Optional(
+        odm.Mapping(odm.Keyword()), description="Parameters to request token")
     access_token_url: str = odm.Optional(odm.Keyword(), description="URL to get access token")
-    access_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Parameters to get access token")
+    access_token_params: Dict[str, str] = odm.Optional(odm.Mapping(
+        odm.Keyword()), description="Parameters to get access token")
     authorize_url: str = odm.Optional(odm.Keyword(), description="URL used to authorize access to a resource")
-    authorize_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters used to authorize access to a resource")
+    authorize_params: Dict[str, str] = odm.Optional(odm.Mapping(
+        odm.Keyword()), description="Parameters used to authorize access to a resource")
     api_base_url: str = odm.Optional(odm.Keyword(), description="Base URL for downloading the user's and groups info")
     client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),
                                                  description="Keyword arguments passed to the different URLs")
@@ -1675,6 +1682,7 @@ class APIProxies(odm.Model):
 DEFAULT_API_PROXIES = {}
 DOWNLOAD_ENCODINGS = ["cart", "raw", "zip"]
 
+
 @odm.model(index=False, store=False, description="UI Configuration")
 class UI(odm.Model):
     ai: AI = odm.Compound(AI, default=DEFAULT_AI, description="AI support for the UI")
@@ -1703,7 +1711,8 @@ class UI(odm.Model):
                                           description="Default API quotas values")
     discover_url: str = odm.Optional(odm.Keyword(), description="Discover URL")
     download_encoding = odm.Enum(values=DOWNLOAD_ENCODINGS, description="Which encoding will be used for downloads?")
-    default_zip_password = odm.Optional(odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files")
+    default_zip_password = odm.Optional(
+        odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files")
     email: str = odm.Optional(odm.Email(), description="Assemblyline admins email address")
     enforce_quota: bool = odm.Boolean(description="Enforce the user's quotas?")
     external_links: List[ExternalLinks] = odm.List(
@@ -1992,6 +2001,94 @@ class MetadataConfig(odm.Model):
 }
 
 
+@odm.model(index=True, store=False, description="Submission Parameters for profile")
+class SubmissionProfileParams(odm.Model):
+    classification = odm.Optional(odm.Classification(),
+                                        description="Original classification of the submission")
+    deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?")
+    generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?")
+    ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?")
+    ignore_recursion_prevention = odm.Optional(odm.Boolean(),
+                                               description="Should we ignore recursion prevention?")
+    ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?")
+    ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?")
+    max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files")
+    max_supplementary = odm.Optional(odm.Integer(), description="Max number of supplementary files")
+    priority = odm.Optional(odm.Integer(), description="Priority of the scan")
+    services = odm.Optional(odm.Compound(ServiceSelection), description="Service selection")
+    service_spec = odm.Optional(odm.Mapping(odm.Mapping(odm.Any())), index=False, store=False,
+                                description="Service-specific parameters")
+    auto_archive = odm.Optional(odm.Boolean(),
+                                description="Does the submission automatically goes into the archive when completed?")
+    delete_after_archive = odm.Optional(odm.Boolean(),
+                                        description="When the submission is archived, should we delete it from hot storage right away?")
+    ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission")
+    type = odm.Optional(odm.Keyword(), description="Type of submission")
+    use_archive_alternate_dtl = odm.Optional(odm.Boolean(),
+                                             description="Should we use the alternate dtl while archiving?")
+
+
+DEFAULT_EDITABLE_PARAMS = {
+    # Default editable params that are used in all profiles
+    "submission": ["classification", "deep_scan", "ignore_cache", "generate_alert", "ignore_filtering", "priority", "type", "ttl"],
+    "CAPA": ["renderer"],
+    "CAPE": ["password", "analysis_timeout_in_seconds"],
+    "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"],
+    "Extract": ["password"],
+    "Intezer": ["dynamic_submit"],
+    "URLDownloader": ["proxy"],
+    "URLCreator": ["minimum_maliciousness"],
+    "XLMMacroDeobfuscator": ["password"]
+}
+
+@odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users")
+class SubmissionProfile(odm.Model):
+    name = odm.Text(description="Submission profile name")
+    display_name = odm.Text(description="Submission profile display name")
+    classification = odm.ClassificationString(default=Classification.UNRESTRICTED,
+                                              description="Submission profile classification")
+    params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile")
+    editable_params = odm.Mapping(odm.List(odm.Text()), default=DEFAULT_EDITABLE_PARAMS,
+                                  description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.")
+    description = odm.Optional(odm.Text(), description="A description of what the profile does")
+
+
+DEFAULT_SUBMISSION_PROFILES = [
+    {
+        # Only perform static analysis
+        "name": "static",
+        "display_name": "Static Analysis",
+        "params": {
+            "services": {
+                "selected": DEFAULT_SRV_SEL
+            }
+        },
+        "description": "Analyze files using static analysis techniques and extract information from the file without executing it, such as metadata, strings, and structural information."
+    },
+    {
+        # Perform static analysis along with dynamic analysis
+        "name": "static_with_dynamic",
+        "display_name": "Static + Dynamic Analysis",
+        "params": {
+            "services": {
+                "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"]
+            }
+        },
+        "description": "Analyze files using static analysis techniques along with executing them in a controlled environment to observe their behavior and capture runtime activities, interactions with the system, network communications, and any malicious behavior exhibited by the file during execution."
+    },
+    {
+        # Perform static analysis along with internet connected services
+        "name": "static_with_internet",
+        "display_name": "Internet-Connected Static Analysis",
+        "params": {
+            "services": {
+                "selected": DEFAULT_SRV_SEL + ["Internet Connected"]
+            },
+        },
+        "description": "Combine traditional static analysis techniques with internet-connected services to gather additional information and context about the file being analyzed."
+    },
+]
+
 TEMPORARY_KEY_TYPE = [
     # Keep this key as submission wide list merging equal items
     'union',
@@ -2033,6 +2130,8 @@ class Submission(odm.Model):
     temporary_keys: dict[str, str] = odm.mapping(odm.enum(TEMPORARY_KEY_TYPE),
                                                  description="Set the operation that will be used to update values "
                                                              "using this key in the temporary submission data.")
+    profiles = odm.List(odm.Compound(SubmissionProfile),
+                        description="Submission profiles with preset submission parameters")
 
 
 DEFAULT_TEMPORARY_KEYS = {
@@ -2057,6 +2156,7 @@ class Submission(odm.Model):
     'verdicts': DEFAULT_VERDICTS,
     'default_temporary_keys': DEFAULT_TEMPORARY_KEYS,
     'temporary_keys': {},
+    'profiles': DEFAULT_SUBMISSION_PROFILES
 }
 
 

assemblyline/odm/models/submission.py

@@ -26,8 +26,6 @@ class ServiceSelection(odm.Model):
         description="List of services to rescan when initial run scores as malicious")
     resubmit = odm.List(odm.Keyword(), default=DEFAULT_RESUBMIT,
                         description="Add to service selection when resubmitting")
-    runtime_excluded = odm.List(odm.Keyword(), default=[], description="List of runtime excluded services")
-
 
 # Fields in the parameters used to calculate hashes used for result caching
 _KEY_HASHED_FIELDS = {
@@ -67,7 +65,6 @@ class SubmissionParams(odm.Model):
     max_extracted = odm.Integer(default=500, description="Max number of extracted files")
     max_supplementary = odm.Integer(default=500, description="Max number of supplementary files")
     priority = odm.Integer(default=1000, description="Priority of the scan", min=1, max=constants.MAX_PRIORITY)
-    profile = odm.Boolean(default=False, description="Should the submission do extra profiling?")
     psid = odm.Optional(odm.UUID(), description="Parent submission ID")
     quota_item = odm.Boolean(default=False, description="Does this submission count against quota?")
     services = odm.Compound(ServiceSelection, default={}, description="Service selection")

assemblyline/odm/models/user.py

@@ -12,7 +12,7 @@
     ("signature_importer", 3),
     ("viewer", 4),
     ("submitter", 5),
-    ("custom", 6)
+    ("custom", 6),
 ])
 
 ROLES = StringTable('ROLES', [
@@ -52,6 +52,7 @@
     ("badlist_manage", 32),
     ("archive_comment", 33),
     ("assistant_use", 34),
+    ("submission_customize", 35)
 ])
 
 
@@ -97,6 +98,7 @@
     ROLES.retrohunt_run,       # Run yara searches
     ROLES.badlist_view,        # View badlist items
     ROLES.badlist_manage,      # Manage (add/delete) badlist items
+    ROLES.submission_customize # Allowed to customize submission properties
 }
 
 USER_ROLES = USER_ROLES_BASIC.union({
@@ -173,6 +175,7 @@
         ROLES.submission_create,
         ROLES.submission_delete,
         ROLES.submission_manage,
+        ROLES.submission_customize,
         ROLES.retrohunt_run,
     ],
     "E": [

assemblyline/odm/models/user_settings.py

@@ -1,5 +1,6 @@
 from assemblyline import odm
 from assemblyline.common import forge, constants
+from assemblyline.odm.models.config import SubmissionProfileParams
 from assemblyline.odm.models.submission import ServiceSelection
 
 Classification = forge.get_classification()
@@ -11,9 +12,10 @@
 @odm.model(index=False, store=False, description="Model of User Settings")
 class UserSettings(odm.Model):
     classification = odm.Classification(default=Classification.UNRESTRICTED,
-                                        description="Default submission classification")
-    deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?")
-    description = odm.Keyword(default="", description="Default description")
+                                        description="Default submission classification",
+                                        deprecation="This will be moved to the \"default\" submission profile")
+    deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?",
+                            deprecation="This will be moved to the \"default\" submission profile")
     download_encoding = odm.Enum(values=ENCODINGS, default="cart",
                                  description="Default download encoding when downloading files")
     default_external_sources = odm.List(odm.Keyword(), default=[],
@@ -24,18 +26,49 @@ class UserSettings(odm.Model):
     )
     executive_summary = odm.Boolean(default=True, description="Should executive summary sections be shown?")
     expand_min_score = odm.Integer(default=500, description="Auto-expand section when score bigger then this")
-    generate_alert = odm.Boolean(default=False, description="Generate an alert?")
-    ignore_cache = odm.Boolean(default=False, description="Ignore service caching?")
+    generate_alert = odm.Boolean(default=False, description="Generate an alert?",
+                              deprecation="This will be moved to the \"default\" submission profile")
+    ignore_cache = odm.Boolean(default=False, description="Ignore service caching?",
+                              deprecation="This will be moved to the \"default\" submission profile")
 
-    #the following 1 line can be removed after assemblyline 4.6+
-    ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?")
-    ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?")
-    ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?")
-    malicious = odm.Boolean(default=False, description="Is the file submitted already known to be malicious?")
-    priority = odm.Integer(default=1000, description="Default priority for the submissions",
-                           min=1, max=constants.MAX_PRIORITY)
-    profile = odm.Boolean(default=False, description="Should the submission do extra profiling?")
-    service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings")
-    services = odm.Compound(ServiceSelection, default={}, description="Default service selection")
+    # the following 1 line can be removed after assemblyline 4.6+
+    ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?",
+                              deprecation="This is replaced by `ignore_recursion_prevention`")
+    ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?",
+                              deprecation="This will be moved to the \"default\" submission profile")
+    ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?",
+                              deprecation="This will be moved to the \"default\" submission profile")
+    priority = odm.Integer(default=1000, min=1, max=constants.MAX_PRIORITY,
+                           description="Default priority for the submissions",
+                           deprecation="This will be moved to the \"default\" submission profile")
+    preferred_submission_profile = odm.Optional(odm.Text(), description="Preferred submission profile")
+    submission_profiles = odm.Mapping(odm.Compound(SubmissionProfileParams), default={},
+                                      description="Default submission profile settings")
+    service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings",
+                              deprecation="This will be moved to the \"default\" submission profile")
+    services = odm.Compound(ServiceSelection, default={}, description="Default service selection",
+                              deprecation="This will be moved to the \"default\" submission profile")
     submission_view = odm.Enum(values=VIEWS, default="report", description="Default view for completed submissions")
-    ttl = odm.Integer(default=30, description="Default submission TTL, in days")
+    ttl = odm.Integer(default=30, description="Default submission TTL, in days",
+                              deprecation="This will be moved to the \"default\" submission profile")
+
+
+DEFAULT_USER_PROFILE_SETTINGS = {
+    "classification": Classification.UNRESTRICTED,
+    "deep_scan": False,
+    "download_encoding": "cart",
+    "default_external_sources": [],
+    "default_zip_password": "infected",
+    "executive_summary": True,
+    "expand_min_score": 500,
+    "generate_alert": False,
+    "ignore_cache": False,
+    "ignore_dynamic_recursion_prevention": False,
+    "ignore_recursion_prevention": False,
+    "ignore_filtering": False,
+    "priority": 1000,
+    "service_spec": {},
+    "services": {},
+    "submission_view": "report",
+    "ttl": 30
+}

assemblyline/odm/random_data/__init__.py

@@ -405,7 +405,7 @@ def create_users(ds, log=None):
         "type": [TYPES.admin]})
     ds.user.save('admin', user_data)
     ds.user_settings.save('admin', UserSettings({"ignore_cache": True, "deep_scan": True,
-                                                 "default_zip_password": "infected", "default_download_encoding": "cart"}))
+                                                 "default_zip_password": "infected", "default_download_encoding": "cart", "submission_profiles": {}}))
     if log:
         log.info(f"\tU:{user_data.uname}   P:{admin_pass}")