From 7bcc7457e01bbd7f72006423bf1ae8c406548572 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:03:36 +0000 Subject: [PATCH 01/30] Add a new `poweruser` role type that can do more than a normal user --- assemblyline/odm/models/user.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/assemblyline/odm/models/user.py b/assemblyline/odm/models/user.py index af2217af4..401c884ee 100644 --- a/assemblyline/odm/models/user.py +++ b/assemblyline/odm/models/user.py @@ -12,7 +12,8 @@ ("signature_importer", 3), ("viewer", 4), ("submitter", 5), - ("custom", 6) + ("custom", 6), + ("poweruser", 7) ]) ROLES = StringTable('ROLES', [ @@ -52,12 +53,14 @@ ("badlist_manage", 32), ("archive_comment", 33), ("assistant_use", 34), + ("submission_customize", 35) ]) SCOPES = {"r", "w", "rw", "c"} USER_TYPES = [ TYPES.admin, # Perform administartive task and has access to all roles + TYPES.poweruser, # Advanced user of the system TYPES.user, # Normal user of the system TYPES.signature_manager, # Super user that also has access to roles for managing signatures in the system TYPES.signature_importer, # Has access to roles for importing signatures in the system @@ -99,7 +102,11 @@ ROLES.badlist_manage, # Manage (add/delete) badlist items } -USER_ROLES = USER_ROLES_BASIC.union({ +USER_ROLES_ADVANCED = USER_ROLES_BASIC.union({ + ROLES.submission_customize # Allowed to customize submission properties +}) + +USER_ROLES = USER_ROLES_ADVANCED.union({ ROLES.administration, # Perform administrative tasks ROLES.file_purge, # Purge files from the filestore ROLES.replay_system, # Manage status of file/submission/alerts during the replay process @@ -122,6 +129,7 @@ ROLES.signature_manage }), TYPES.user: USER_ROLES_BASIC, + TYPES.poweruser: USER_ROLES_ADVANCED, TYPES.viewer: { ROLES.alert_view, ROLES.apikey_access, @@ -173,6 +181,7 @@ ROLES.submission_create, ROLES.submission_delete, ROLES.submission_manage, + ROLES.submission_customize, ROLES.retrohunt_run, ], "E": [ From d3794da8e0047f754cc48a6b03372d935697a5d9 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:26:48 +0000 Subject: [PATCH 02/30] Add a submission profile configuration for basic user usage --- assemblyline/odm/models/config.py | 41 ++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a8bfcd9ab..85c179cb4 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1,8 +1,10 @@ from typing import Any, Dict, List from assemblyline import odm +from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta +from assemblyline.odm.models.submission import SubmissionParams, DEFAULT_SRV_SEL AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -15,6 +17,7 @@ DEFAULT_SUBMISSION_QUOTA = 5 DEFAULT_ASYNC_SUBMISSION_QUOTA = 0 +Classification = get_classification() @odm.model(index=False, store=False, description="Password Requirement") class PasswordRequirement(odm.Model): @@ -1941,6 +1944,39 @@ class MetadataConfig(odm.Model): } } +@odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") +class SubmissionProfile(odm.Model): + name = odm.Text(description="Submission profile name") + classification = odm.ClassificationString(default=Classification.UNRESTRICTED, + description="Submission profile classification") + params = odm.Compound(SubmissionParams, description="Submission parameters for profile") + +DEFAULT_SUBMISSION_PROFILES = [ + { + # Only perform static analysis + "name": "Static Analysis" + }, + { + # Perform static analysis along with dynamic analysis + "name": "Dynamic Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] + } + } + + }, + { + # Perform static analysis along with internet connected services + "name": "Static Analysis with Internet", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + ["Internet Connected"] + } + } + + }, +] @odm.model(index=False, store=False, description="Default values for parameters for submissions that may be overridden on a per submission basis") @@ -1970,6 +2006,8 @@ class Submission(odm.Model): description="Tag types that show up in the submission summary") verdicts = odm.Compound(Verdicts, default=DEFAULT_VERDICTS, description="Minimum score value to get the specified verdict.") + profiles = odm.List(odm.Compound(SubmissionProfile), + description="Submission profiles with preset submission parameters") DEFAULT_SUBMISSION = { @@ -1986,7 +2024,8 @@ class Submission(odm.Model): 'sha256_sources': [], 'file_sources': [], 'tag_types': DEFAULT_TAG_TYPES, - 'verdicts': DEFAULT_VERDICTS + 'verdicts': DEFAULT_VERDICTS, + 'profiles': DEFAULT_SUBMISSION_PROFILES } From e49f48b1a314f6b6e2e14d186a3183b8df421b07 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:47:30 +0000 Subject: [PATCH 03/30] Omit `runtime_excluded` from ServiceSelection model (unused) --- assemblyline/odm/models/submission.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/assemblyline/odm/models/submission.py b/assemblyline/odm/models/submission.py index 2d9991ed4..6e5ad8dfb 100644 --- a/assemblyline/odm/models/submission.py +++ b/assemblyline/odm/models/submission.py @@ -26,8 +26,6 @@ class ServiceSelection(odm.Model): description="List of services to rescan when initial run scores as malicious") resubmit = odm.List(odm.Keyword(), default=DEFAULT_RESUBMIT, description="Add to service selection when resubmitting") - runtime_excluded = odm.List(odm.Keyword(), default=[], description="List of runtime excluded services") - # Fields in the parameters used to calculate hashes used for result caching _KEY_HASHED_FIELDS = { From a8a03eaa0a2320ff278d656c749d93231aebafd4 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:48:24 +0000 Subject: [PATCH 04/30] Create a SubmissionProfileParams model for submission profiles --- assemblyline/odm/models/config.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 85c179cb4..a3d7ee12a 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -4,7 +4,7 @@ from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta -from assemblyline.odm.models.submission import SubmissionParams, DEFAULT_SRV_SEL +from assemblyline.odm.models.submission import ServiceSelection, DEFAULT_SRV_SEL AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -1944,12 +1944,34 @@ class MetadataConfig(odm.Model): } } +@odm.model(index=True, store=False, description="Submission Parameters for profile") +class SubmissionProfileParams(odm.Model): + deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") + ignore_cache = odm.Boolean(default=False, description="Ignore the cached service results?") + ignore_dynamic_recursion_prevention = odm.Boolean(default=False, + description="Should we ignore dynamic recursion prevention?") + ignore_filtering = odm.Boolean(default=False, description="Should we ignore filtering services?") + ignore_size = odm.Boolean(default=False, description="Ignore the file size limits?") + max_extracted = odm.Integer(default=500, description="Max number of extracted files") + max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") + priority = odm.Integer(default=1000, description="Priority of the scan") + services = odm.Compound(ServiceSelection, default={}, description="Service selection") + service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, index=False, store=False, + description="Service-specific parameters") + auto_archive = odm.Boolean(default=False, + description="Does the submission automatically goes into the archive when completed?") + delete_after_archive = odm.Boolean( + default=False, + description="When the submission is archived, should we delete it from hot storage right away?") + use_archive_alternate_dtl = odm.Boolean(default=False, + description="Should we use the alternate dtl while archiving?") + @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") class SubmissionProfile(odm.Model): name = odm.Text(description="Submission profile name") classification = odm.ClassificationString(default=Classification.UNRESTRICTED, description="Submission profile classification") - params = odm.Compound(SubmissionParams, description="Submission parameters for profile") + params = odm.Compound(SubmissionProfileParams, description="Submission parameters for profile") DEFAULT_SUBMISSION_PROFILES = [ { From 6af6eb008716620b3c379b9d09f39b5c2151a63d Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:55:27 +0000 Subject: [PATCH 05/30] Add params to Static Analysis profile --- assemblyline/odm/models/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a3d7ee12a..f4e1dac33 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1976,7 +1976,12 @@ class SubmissionProfile(odm.Model): DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis - "name": "Static Analysis" + "name": "Static Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + } + } }, { # Perform static analysis along with dynamic analysis From 2b9755e38ddfffd8a7d2c3c0348825a568c893ff Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:32:38 +0000 Subject: [PATCH 06/30] Remove unused submission parameter from v3 --- assemblyline/odm/models/submission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/assemblyline/odm/models/submission.py b/assemblyline/odm/models/submission.py index 6e5ad8dfb..e1423dee6 100644 --- a/assemblyline/odm/models/submission.py +++ b/assemblyline/odm/models/submission.py @@ -58,7 +58,6 @@ class SubmissionParams(odm.Model): max_extracted = odm.Integer(default=500, description="Max number of extracted files") max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") priority = odm.Integer(default=1000, description="Priority of the scan") - profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") psid = odm.Optional(odm.UUID(), description="Parent submission ID") quota_item = odm.Boolean(default=False, description="Does this submission count against quota?") services = odm.Compound(ServiceSelection, default={}, description="Service selection") From 116afd66051bc8406b3a09ee371926b1c9b34ee0 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:44:43 +0000 Subject: [PATCH 07/30] Add `generate_alert` and `ttl` as enforced parameters --- assemblyline/odm/models/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index f4e1dac33..896f2961c 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1947,6 +1947,7 @@ class MetadataConfig(odm.Model): @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") + generate_alert = odm.Boolean(default=False, description="Should this submission generate an alert?") ignore_cache = odm.Boolean(default=False, description="Ignore the cached service results?") ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Should we ignore dynamic recursion prevention?") @@ -1963,6 +1964,7 @@ class SubmissionProfileParams(odm.Model): delete_after_archive = odm.Boolean( default=False, description="When the submission is archived, should we delete it from hot storage right away?") + ttl = odm.Integer(default=0, description="Time, in days, to live for this submission") use_archive_alternate_dtl = odm.Boolean(default=False, description="Should we use the alternate dtl while archiving?") From f9808e98655260cd5b0b585b475a742675c294c8 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 9 Aug 2024 19:25:25 +0000 Subject: [PATCH 08/30] Update SubmissionProfileParams model to all be optional fields --- assemblyline/odm/models/config.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 896f2961c..a8e3a6e59 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1946,26 +1946,25 @@ class MetadataConfig(odm.Model): @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): - deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") - generate_alert = odm.Boolean(default=False, description="Should this submission generate an alert?") - ignore_cache = odm.Boolean(default=False, description="Ignore the cached service results?") - ignore_dynamic_recursion_prevention = odm.Boolean(default=False, + deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") + generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") + ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") + ignore_dynamic_recursion_prevention = odm.Optional(odm.Boolean(), description="Should we ignore dynamic recursion prevention?") - ignore_filtering = odm.Boolean(default=False, description="Should we ignore filtering services?") - ignore_size = odm.Boolean(default=False, description="Ignore the file size limits?") - max_extracted = odm.Integer(default=500, description="Max number of extracted files") - max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") - priority = odm.Integer(default=1000, description="Priority of the scan") - services = odm.Compound(ServiceSelection, default={}, description="Service selection") - service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, index=False, store=False, + ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") + ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") + max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") + max_supplementary = odm.Optional(odm.Integer(), description="Max number of supplementary files") + priority = odm.Optional(odm.Integer(), description="Priority of the scan") + services = odm.Optional(odm.Compound(ServiceSelection), description="Service selection") + service_spec = odm.Optional(odm.Mapping(odm.Mapping(odm.Any())), index=False, store=False, description="Service-specific parameters") - auto_archive = odm.Boolean(default=False, + auto_archive = odm.Optional(odm.Boolean(), description="Does the submission automatically goes into the archive when completed?") - delete_after_archive = odm.Boolean( - default=False, + delete_after_archive = odm.Optional(odm.Boolean(), description="When the submission is archived, should we delete it from hot storage right away?") - ttl = odm.Integer(default=0, description="Time, in days, to live for this submission") - use_archive_alternate_dtl = odm.Boolean(default=False, + ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") + use_archive_alternate_dtl = odm.Optional(odm.Boolean(), description="Should we use the alternate dtl while archiving?") @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") From 25733c7647e90df53c5b5dc069b53d5c202a9f98 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:35:09 +0000 Subject: [PATCH 09/30] Allow users to change some properties of submission profiles --- assemblyline/odm/models/config.py | 4 +++- assemblyline/odm/models/user.py | 10 ++-------- assemblyline/odm/models/user_settings.py | 4 ++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a8e3a6e59..ae17f2a3a 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1972,7 +1972,9 @@ class SubmissionProfile(odm.Model): name = odm.Text(description="Submission profile name") classification = odm.ClassificationString(default=Classification.UNRESTRICTED, description="Submission profile classification") - params = odm.Compound(SubmissionProfileParams, description="Submission parameters for profile") + params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") + editable_params = odm.Mapping(odm.List(odm.Text()), default={}, + description="A list of service-specific parameters that can be configured") DEFAULT_SUBMISSION_PROFILES = [ { diff --git a/assemblyline/odm/models/user.py b/assemblyline/odm/models/user.py index 401c884ee..616447d68 100644 --- a/assemblyline/odm/models/user.py +++ b/assemblyline/odm/models/user.py @@ -13,7 +13,6 @@ ("viewer", 4), ("submitter", 5), ("custom", 6), - ("poweruser", 7) ]) ROLES = StringTable('ROLES', [ @@ -60,7 +59,6 @@ SCOPES = {"r", "w", "rw", "c"} USER_TYPES = [ TYPES.admin, # Perform administartive task and has access to all roles - TYPES.poweruser, # Advanced user of the system TYPES.user, # Normal user of the system TYPES.signature_manager, # Super user that also has access to roles for managing signatures in the system TYPES.signature_importer, # Has access to roles for importing signatures in the system @@ -100,13 +98,10 @@ ROLES.retrohunt_run, # Run yara searches ROLES.badlist_view, # View badlist items ROLES.badlist_manage, # Manage (add/delete) badlist items + ROLES.submission_customize # Allowed to customize submission properties } -USER_ROLES_ADVANCED = USER_ROLES_BASIC.union({ - ROLES.submission_customize # Allowed to customize submission properties -}) - -USER_ROLES = USER_ROLES_ADVANCED.union({ +USER_ROLES = USER_ROLES_BASIC.union({ ROLES.administration, # Perform administrative tasks ROLES.file_purge, # Purge files from the filestore ROLES.replay_system, # Manage status of file/submission/alerts during the replay process @@ -129,7 +124,6 @@ ROLES.signature_manage }), TYPES.user: USER_ROLES_BASIC, - TYPES.poweruser: USER_ROLES_ADVANCED, TYPES.viewer: { ROLES.alert_view, ROLES.apikey_access, diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index 37bd3a39c..0ca7a9ab6 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -1,5 +1,6 @@ from assemblyline import odm from assemblyline.common import forge +from assemblyline.odm.models.config import SubmissionProfileParams from assemblyline.odm.models.submission import ServiceSelection Classification = forge.get_classification() @@ -31,6 +32,9 @@ class UserSettings(odm.Model): malicious = odm.Boolean(default=False, description="Is the file submitted already known to be malicious?") priority = odm.Integer(default=1000, description="Default priority for the submissions") profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") + preferred_submission_profile = odm.Optional(odm.Text(), description="Preferred submission profile") + submission_profiles = odm.Mapping(odm.Compound(SubmissionProfileParams), default={}, + description="Default submission profile settings") service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings") services = odm.Compound(ServiceSelection, default={}, description="Default service selection") submission_view = odm.Enum(values=VIEWS, default="report", description="Default view for completed submissions") From f1be7a4a3d39f12c073e88a72fd35283bd45b6dc Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:38:13 +0000 Subject: [PATCH 10/30] Allow for configuring core params --- assemblyline/odm/models/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 0408fa001..37938fd06 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2008,7 +2008,7 @@ class SubmissionProfile(odm.Model): description="Submission profile classification") params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of service-specific parameters that can be configured") + description="A list of specific parameters that can be configured. The format of this configuration is `(core|).`.") DEFAULT_SUBMISSION_PROFILES = [ { From 3dabed1fac7ced00c224c9c6a9234a0dae960cca Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 01:32:10 +0000 Subject: [PATCH 11/30] Formatted the submission profiles models --- assemblyline/odm/models/config.py | 35 +++++++++++++++--------- assemblyline/odm/models/user_settings.py | 2 +- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 1bef57c1f..a486610e7 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -4,7 +4,7 @@ from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta -from assemblyline.odm.models.submission import ServiceSelection, DEFAULT_SRV_SEL +from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, ServiceSelection AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -19,6 +19,7 @@ Classification = get_classification() + @odm.model(index=False, store=False, description="Password Requirement") class PasswordRequirement(odm.Model): lower: bool = odm.Boolean(description="Password must contain lowercase letters") @@ -213,11 +214,14 @@ class OAuthProvider(odm.Model): redirect_uri: str = odm.Optional(odm.Keyword(), description="URI to redirect to after authentication with OAuth provider") request_token_url: str = odm.Optional(odm.Keyword(), description="URL to request token") - request_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters to request token") + request_token_params: Dict[str, str] = odm.Optional( + odm.Mapping(odm.Keyword()), description="Parameters to request token") access_token_url: str = odm.Optional(odm.Keyword(), description="URL to get access token") - access_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Parameters to get access token") + access_token_params: Dict[str, str] = odm.Optional(odm.Mapping( + odm.Keyword()), description="Parameters to get access token") authorize_url: str = odm.Optional(odm.Keyword(), description="URL used to authorize access to a resource") - authorize_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters used to authorize access to a resource") + authorize_params: Dict[str, str] = odm.Optional(odm.Mapping( + odm.Keyword()), description="Parameters used to authorize access to a resource") api_base_url: str = odm.Optional(odm.Keyword(), description="Base URL for downloading the user's and groups info") client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Keyword arguments passed to the different URLs") @@ -1673,6 +1677,7 @@ class APIProxies(odm.Model): DEFAULT_API_PROXIES = {} DOWNLOAD_ENCODINGS = ["cart", "raw", "zip"] + @odm.model(index=False, store=False, description="UI Configuration") class UI(odm.Model): ai: AI = odm.Compound(AI, default=DEFAULT_AI, description="AI support for the UI") @@ -1700,7 +1705,8 @@ class UI(odm.Model): description="Default API quotas values") discover_url: str = odm.Optional(odm.Keyword(), description="Discover URL") download_encoding = odm.Enum(values=DOWNLOAD_ENCODINGS, description="Which encoding will be used for downloads?") - default_zip_password = odm.Optional(odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files") + default_zip_password = odm.Optional( + odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files") email: str = odm.Optional(odm.Email(), description="Assemblyline admins email address") enforce_quota: bool = odm.Boolean(description="Enforce the user's quotas?") external_links: List[ExternalLinks] = odm.List( @@ -1981,13 +1987,14 @@ class MetadataConfig(odm.Model): } } + @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") ignore_dynamic_recursion_prevention = odm.Optional(odm.Boolean(), - description="Should we ignore dynamic recursion prevention?") + description="Should we ignore dynamic recursion prevention?") ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") @@ -1995,14 +2002,15 @@ class SubmissionProfileParams(odm.Model): priority = odm.Optional(odm.Integer(), description="Priority of the scan") services = odm.Optional(odm.Compound(ServiceSelection), description="Service selection") service_spec = odm.Optional(odm.Mapping(odm.Mapping(odm.Any())), index=False, store=False, - description="Service-specific parameters") + description="Service-specific parameters") auto_archive = odm.Optional(odm.Boolean(), - description="Does the submission automatically goes into the archive when completed?") + description="Does the submission automatically goes into the archive when completed?") delete_after_archive = odm.Optional(odm.Boolean(), - description="When the submission is archived, should we delete it from hot storage right away?") + description="When the submission is archived, should we delete it from hot storage right away?") ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") use_archive_alternate_dtl = odm.Optional(odm.Boolean(), - description="Should we use the alternate dtl while archiving?") + description="Should we use the alternate dtl while archiving?") + @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") class SubmissionProfile(odm.Model): @@ -2013,10 +2021,11 @@ class SubmissionProfile(odm.Model): editable_params = odm.Mapping(odm.List(odm.Text()), default={}, description="A list of specific parameters that can be configured. The format of this configuration is `(core|).`.") + DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis - "name": "Static Analysis", + "name": "static", "params": { "services": { "selected": DEFAULT_SRV_SEL @@ -2025,7 +2034,7 @@ class SubmissionProfile(odm.Model): }, { # Perform static analysis along with dynamic analysis - "name": "Dynamic Analysis", + "name": "dynamic", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] @@ -2035,7 +2044,7 @@ class SubmissionProfile(odm.Model): }, { # Perform static analysis along with internet connected services - "name": "Static Analysis with Internet", + "name": "static_with_internet", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index 7c11af28a..dc753a7ee 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -28,7 +28,7 @@ class UserSettings(odm.Model): generate_alert = odm.Boolean(default=False, description="Generate an alert?") ignore_cache = odm.Boolean(default=False, description="Ignore service caching?") - #the following 1 line can be removed after assemblyline 4.6+ + # the following 1 line can be removed after assemblyline 4.6+ ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?") ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?") ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?") From 1bdaf0de230d8ab70a9da2db09ce74e42fdef09e Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 28 Jan 2025 22:44:24 +0000 Subject: [PATCH 12/30] Add addtional fields to SubmissionProfiles --- assemblyline/odm/models/config.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a486610e7..31482dd3a 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2008,6 +2008,7 @@ class SubmissionProfileParams(odm.Model): delete_after_archive = odm.Optional(odm.Boolean(), description="When the submission is archived, should we delete it from hot storage right away?") ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") + type = odm.Optional(odm.Keyword(), description="Type of submission") use_archive_alternate_dtl = odm.Optional(odm.Boolean(), description="Should we use the alternate dtl while archiving?") @@ -2015,40 +2016,60 @@ class SubmissionProfileParams(odm.Model): @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") class SubmissionProfile(odm.Model): name = odm.Text(description="Submission profile name") + display_name = odm.Text(description="Submission profile display name") classification = odm.ClassificationString(default=Classification.UNRESTRICTED, description="Submission profile classification") params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of specific parameters that can be configured. The format of this configuration is `(core|).`.") + description="A list of parameters that can be configured for this profile. The keys are the service names or \"core\" and the values are the parameters that can be configured.") +DEFAULT_EDITABLE_PARAMS = { + # Default editable params that are used in all bundled profiles + "core": ["priority", "type", "ttl"], + "CAPA": ["renderer"], + "CAPE": ["password", "analysis_timeout_in_seconds"], + "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"], + "Extract": ["password"], + "Intezer": ["dynamic_submit"], + "URLDownloader": ["proxy"], + "URLCreator": ["minimum_maliciousness"], + "XLMMacroDeobfuscator": ["password"] +} + DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis "name": "static", + "display_name": "Static Analysis", "params": { "services": { "selected": DEFAULT_SRV_SEL } - } + }, + "editable_params": DEFAULT_EDITABLE_PARAMS }, { # Perform static analysis along with dynamic analysis "name": "dynamic", + "display_name": "Static Analysis with Dynamic Analysis", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] } - } + }, + "editable_params": DEFAULT_EDITABLE_PARAMS }, { # Perform static analysis along with internet connected services "name": "static_with_internet", + "display_name": "Static Analysis with Internet Connected Services", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] - } + }, + "editable_params": DEFAULT_EDITABLE_PARAMS } }, From 1825cc512890ebde92135f3ea00e4fe3ec85fcba Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 1 Feb 2025 07:50:19 +0000 Subject: [PATCH 13/30] Update models --- assemblyline/odm/models/config.py | 10 +++-- assemblyline/odm/models/user_settings.py | 57 ++++++++++++++++++------ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 31482dd3a..7f1b9c416 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1990,11 +1990,13 @@ class MetadataConfig(odm.Model): @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): + classification = odm.Optional(odm.Classification(), + description="Original classification of the submission") deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") - ignore_dynamic_recursion_prevention = odm.Optional(odm.Boolean(), - description="Should we ignore dynamic recursion prevention?") + ignore_recursion_prevention = odm.Optional(odm.Boolean(), + description="Should we ignore recursion prevention?") ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") @@ -2021,12 +2023,12 @@ class SubmissionProfile(odm.Model): description="Submission profile classification") params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of parameters that can be configured for this profile. The keys are the service names or \"core\" and the values are the parameters that can be configured.") + description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.") DEFAULT_EDITABLE_PARAMS = { # Default editable params that are used in all bundled profiles - "core": ["priority", "type", "ttl"], + "submission": ["classification", "deep_scan", "generate_alert", "ignore_filtering", "priority", "type", "ttl"], "CAPA": ["renderer"], "CAPE": ["password", "analysis_timeout_in_seconds"], "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"], diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index dc753a7ee..8b1436c89 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -12,9 +12,10 @@ @odm.model(index=False, store=False, description="Model of User Settings") class UserSettings(odm.Model): classification = odm.Classification(default=Classification.UNRESTRICTED, - description="Default submission classification") - deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") - description = odm.Keyword(default="", description="Default description") + description="Default submission classification", + deprecation="This will be moved to the \"default\" submission profile") + deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?", + deprecation="This will be moved to the \"default\" submission profile") download_encoding = odm.Enum(values=ENCODINGS, default="cart", description="Default download encoding when downloading files") default_external_sources = odm.List(odm.Keyword(), default=[], @@ -25,20 +26,48 @@ class UserSettings(odm.Model): ) executive_summary = odm.Boolean(default=True, description="Should executive summary sections be shown?") expand_min_score = odm.Integer(default=500, description="Auto-expand section when score bigger then this") - generate_alert = odm.Boolean(default=False, description="Generate an alert?") - ignore_cache = odm.Boolean(default=False, description="Ignore service caching?") + generate_alert = odm.Boolean(default=False, description="Generate an alert?", + deprecation="This will be moved to the \"default\" submission profile") + ignore_cache = odm.Boolean(default=False, description="Ignore service caching?", + deprecation="This will be moved to the \"default\" submission profile") # the following 1 line can be removed after assemblyline 4.6+ - ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?") - ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?") - ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?") - malicious = odm.Boolean(default=False, description="Is the file submitted already known to be malicious?") - priority = odm.Integer(default=1000, description="Default priority for the submissions") - profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") + ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?", + deprecation="This is replaced by `ignore_recursion_prevention`") + ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?", + deprecation="This will be moved to the \"default\" submission profile") + ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?", + deprecation="This will be moved to the \"default\" submission profile") + priority = odm.Integer(default=1000, description="Default priority for the submissions", + deprecation="This will be moved to the \"default\" submission profile") preferred_submission_profile = odm.Optional(odm.Text(), description="Preferred submission profile") submission_profiles = odm.Mapping(odm.Compound(SubmissionProfileParams), default={}, description="Default submission profile settings") - service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings") - services = odm.Compound(ServiceSelection, default={}, description="Default service selection") + service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings", + deprecation="This will be moved to the \"default\" submission profile") + services = odm.Compound(ServiceSelection, default={}, description="Default service selection", + deprecation="This will be moved to the \"default\" submission profile") submission_view = odm.Enum(values=VIEWS, default="report", description="Default view for completed submissions") - ttl = odm.Integer(default=30, description="Default submission TTL, in days") + ttl = odm.Integer(default=30, description="Default submission TTL, in days", + deprecation="This will be moved to the \"default\" submission profile") + + +DEFAULT_USER_PROFILE_SETTINGS = { + "classification": Classification.UNRESTRICTED, + "deep_scan": False, + "download_encoding": "cart", + "default_external_sources": [], + "default_zip_password": "infected", + "executive_summary": True, + "expand_min_score": 500, + "generate_alert": False, + "ignore_cache": False, + "ignore_dynamic_recursion_prevention": False, + "ignore_recursion_prevention": False, + "ignore_filtering": False, + "priority": 1000, + "service_spec": {}, + "services": {}, + "submission_view": "report", + "ttl": 30 +} From 9815a3bdc29eae702f411dc84d7e3179f57235a9 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:03:36 +0000 Subject: [PATCH 14/30] Add a new `poweruser` role type that can do more than a normal user --- assemblyline/odm/models/user.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/assemblyline/odm/models/user.py b/assemblyline/odm/models/user.py index a9aaf2c90..4b49c23ea 100644 --- a/assemblyline/odm/models/user.py +++ b/assemblyline/odm/models/user.py @@ -12,7 +12,8 @@ ("signature_importer", 3), ("viewer", 4), ("submitter", 5), - ("custom", 6) + ("custom", 6), + ("poweruser", 7) ]) ROLES = StringTable('ROLES', [ @@ -52,12 +53,14 @@ ("badlist_manage", 32), ("archive_comment", 33), ("assistant_use", 34), + ("submission_customize", 35) ]) SCOPES = {"r", "w", "rw", "c"} USER_TYPES = [ TYPES.admin, # Perform administartive task and has access to all roles + TYPES.poweruser, # Advanced user of the system TYPES.user, # Normal user of the system TYPES.signature_manager, # Super user that also has access to roles for managing signatures in the system TYPES.signature_importer, # Has access to roles for importing signatures in the system @@ -99,7 +102,11 @@ ROLES.badlist_manage, # Manage (add/delete) badlist items } -USER_ROLES = USER_ROLES_BASIC.union({ +USER_ROLES_ADVANCED = USER_ROLES_BASIC.union({ + ROLES.submission_customize # Allowed to customize submission properties +}) + +USER_ROLES = USER_ROLES_ADVANCED.union({ ROLES.administration, # Perform administrative tasks ROLES.file_purge, # Purge files from the filestore ROLES.replay_system, # Manage status of file/submission/alerts during the replay process @@ -122,6 +129,7 @@ ROLES.signature_manage }), TYPES.user: USER_ROLES_BASIC, + TYPES.poweruser: USER_ROLES_ADVANCED, TYPES.viewer: { ROLES.alert_view, ROLES.apikey_access, @@ -173,6 +181,7 @@ ROLES.submission_create, ROLES.submission_delete, ROLES.submission_manage, + ROLES.submission_customize, ROLES.retrohunt_run, ], "E": [ From c9a1a5a81f9e27baa640643d80c607574810fac0 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:26:48 +0000 Subject: [PATCH 15/30] Add a submission profile configuration for basic user usage --- assemblyline/odm/models/config.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index bb85845b0..fc52c8389 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2,8 +2,10 @@ from assemblyline import odm from assemblyline.common.constants import PRIORITIES +from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta +from assemblyline.odm.models.submission import SubmissionParams, DEFAULT_SRV_SEL AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -16,6 +18,7 @@ DEFAULT_SUBMISSION_QUOTA = 5 DEFAULT_ASYNC_SUBMISSION_QUOTA = 0 +Classification = get_classification() @odm.model(index=False, store=False, description="Password Requirement") class PasswordRequirement(odm.Model): @@ -1989,6 +1992,39 @@ class MetadataConfig(odm.Model): } } +@odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") +class SubmissionProfile(odm.Model): + name = odm.Text(description="Submission profile name") + classification = odm.ClassificationString(default=Classification.UNRESTRICTED, + description="Submission profile classification") + params = odm.Compound(SubmissionParams, description="Submission parameters for profile") + +DEFAULT_SUBMISSION_PROFILES = [ + { + # Only perform static analysis + "name": "Static Analysis" + }, + { + # Perform static analysis along with dynamic analysis + "name": "Dynamic Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] + } + } + + }, + { + # Perform static analysis along with internet connected services + "name": "Static Analysis with Internet", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + ["Internet Connected"] + } + } + + }, +] TEMPORARY_KEY_TYPE = [ # Keep this key as submission wide list merging equal items @@ -2031,6 +2067,8 @@ class Submission(odm.Model): temporary_keys: dict[str, str] = odm.mapping(odm.enum(TEMPORARY_KEY_TYPE), description="Set the operation that will be used to update values " "using this key in the temporary submission data.") + profiles = odm.List(odm.Compound(SubmissionProfile), + description="Submission profiles with preset submission parameters") DEFAULT_TEMPORARY_KEYS = { @@ -2055,6 +2093,7 @@ class Submission(odm.Model): 'verdicts': DEFAULT_VERDICTS, 'default_temporary_keys': DEFAULT_TEMPORARY_KEYS, 'temporary_keys': {}, + 'profiles': DEFAULT_SUBMISSION_PROFILES } From 80a2bc3b4e65541acbbd3434d200848061b484b3 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:47:30 +0000 Subject: [PATCH 16/30] Omit `runtime_excluded` from ServiceSelection model (unused) --- assemblyline/odm/models/submission.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/assemblyline/odm/models/submission.py b/assemblyline/odm/models/submission.py index 23615a4a0..d9b118e11 100644 --- a/assemblyline/odm/models/submission.py +++ b/assemblyline/odm/models/submission.py @@ -26,8 +26,6 @@ class ServiceSelection(odm.Model): description="List of services to rescan when initial run scores as malicious") resubmit = odm.List(odm.Keyword(), default=DEFAULT_RESUBMIT, description="Add to service selection when resubmitting") - runtime_excluded = odm.List(odm.Keyword(), default=[], description="List of runtime excluded services") - # Fields in the parameters used to calculate hashes used for result caching _KEY_HASHED_FIELDS = { From 5144bcf221e6d837b65752a455b0412569ad85a3 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:48:24 +0000 Subject: [PATCH 17/30] Create a SubmissionProfileParams model for submission profiles --- assemblyline/odm/models/config.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index fc52c8389..a667ca4cc 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -5,7 +5,7 @@ from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta -from assemblyline.odm.models.submission import SubmissionParams, DEFAULT_SRV_SEL +from assemblyline.odm.models.submission import ServiceSelection, DEFAULT_SRV_SEL AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -1992,12 +1992,34 @@ class MetadataConfig(odm.Model): } } +@odm.model(index=True, store=False, description="Submission Parameters for profile") +class SubmissionProfileParams(odm.Model): + deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") + ignore_cache = odm.Boolean(default=False, description="Ignore the cached service results?") + ignore_dynamic_recursion_prevention = odm.Boolean(default=False, + description="Should we ignore dynamic recursion prevention?") + ignore_filtering = odm.Boolean(default=False, description="Should we ignore filtering services?") + ignore_size = odm.Boolean(default=False, description="Ignore the file size limits?") + max_extracted = odm.Integer(default=500, description="Max number of extracted files") + max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") + priority = odm.Integer(default=1000, description="Priority of the scan") + services = odm.Compound(ServiceSelection, default={}, description="Service selection") + service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, index=False, store=False, + description="Service-specific parameters") + auto_archive = odm.Boolean(default=False, + description="Does the submission automatically goes into the archive when completed?") + delete_after_archive = odm.Boolean( + default=False, + description="When the submission is archived, should we delete it from hot storage right away?") + use_archive_alternate_dtl = odm.Boolean(default=False, + description="Should we use the alternate dtl while archiving?") + @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") class SubmissionProfile(odm.Model): name = odm.Text(description="Submission profile name") classification = odm.ClassificationString(default=Classification.UNRESTRICTED, description="Submission profile classification") - params = odm.Compound(SubmissionParams, description="Submission parameters for profile") + params = odm.Compound(SubmissionProfileParams, description="Submission parameters for profile") DEFAULT_SUBMISSION_PROFILES = [ { From 1a49574a62a21bf6c843cfd1cde4482799c8ab57 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:55:27 +0000 Subject: [PATCH 18/30] Add params to Static Analysis profile --- assemblyline/odm/models/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a667ca4cc..fa47e6a48 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2024,7 +2024,12 @@ class SubmissionProfile(odm.Model): DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis - "name": "Static Analysis" + "name": "Static Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + } + } }, { # Perform static analysis along with dynamic analysis From 6a7d0c988ca58102032612f6f6b223ae40d1e5f4 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:32:38 +0000 Subject: [PATCH 19/30] Remove unused submission parameter from v3 --- assemblyline/odm/models/submission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/assemblyline/odm/models/submission.py b/assemblyline/odm/models/submission.py index d9b118e11..df599b5a7 100644 --- a/assemblyline/odm/models/submission.py +++ b/assemblyline/odm/models/submission.py @@ -65,7 +65,6 @@ class SubmissionParams(odm.Model): max_extracted = odm.Integer(default=500, description="Max number of extracted files") max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") priority = odm.Integer(default=1000, description="Priority of the scan", min=1, max=constants.MAX_PRIORITY) - profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") psid = odm.Optional(odm.UUID(), description="Parent submission ID") quota_item = odm.Boolean(default=False, description="Does this submission count against quota?") services = odm.Compound(ServiceSelection, default={}, description="Service selection") From 088775ce743b8a592e416c46529aecef5ee6acbc Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:44:43 +0000 Subject: [PATCH 20/30] Add `generate_alert` and `ttl` as enforced parameters --- assemblyline/odm/models/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index fa47e6a48..ad8fceea4 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1995,6 +1995,7 @@ class MetadataConfig(odm.Model): @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") + generate_alert = odm.Boolean(default=False, description="Should this submission generate an alert?") ignore_cache = odm.Boolean(default=False, description="Ignore the cached service results?") ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Should we ignore dynamic recursion prevention?") @@ -2011,6 +2012,7 @@ class SubmissionProfileParams(odm.Model): delete_after_archive = odm.Boolean( default=False, description="When the submission is archived, should we delete it from hot storage right away?") + ttl = odm.Integer(default=0, description="Time, in days, to live for this submission") use_archive_alternate_dtl = odm.Boolean(default=False, description="Should we use the alternate dtl while archiving?") From a91f2f1e448a90bf89e6d05a20ca705f29fec09e Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 9 Aug 2024 19:25:25 +0000 Subject: [PATCH 21/30] Update SubmissionProfileParams model to all be optional fields --- assemblyline/odm/models/config.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index ad8fceea4..6f340c27a 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1994,26 +1994,25 @@ class MetadataConfig(odm.Model): @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): - deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") - generate_alert = odm.Boolean(default=False, description="Should this submission generate an alert?") - ignore_cache = odm.Boolean(default=False, description="Ignore the cached service results?") - ignore_dynamic_recursion_prevention = odm.Boolean(default=False, + deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") + generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") + ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") + ignore_dynamic_recursion_prevention = odm.Optional(odm.Boolean(), description="Should we ignore dynamic recursion prevention?") - ignore_filtering = odm.Boolean(default=False, description="Should we ignore filtering services?") - ignore_size = odm.Boolean(default=False, description="Ignore the file size limits?") - max_extracted = odm.Integer(default=500, description="Max number of extracted files") - max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") - priority = odm.Integer(default=1000, description="Priority of the scan") - services = odm.Compound(ServiceSelection, default={}, description="Service selection") - service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, index=False, store=False, + ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") + ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") + max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") + max_supplementary = odm.Optional(odm.Integer(), description="Max number of supplementary files") + priority = odm.Optional(odm.Integer(), description="Priority of the scan") + services = odm.Optional(odm.Compound(ServiceSelection), description="Service selection") + service_spec = odm.Optional(odm.Mapping(odm.Mapping(odm.Any())), index=False, store=False, description="Service-specific parameters") - auto_archive = odm.Boolean(default=False, + auto_archive = odm.Optional(odm.Boolean(), description="Does the submission automatically goes into the archive when completed?") - delete_after_archive = odm.Boolean( - default=False, + delete_after_archive = odm.Optional(odm.Boolean(), description="When the submission is archived, should we delete it from hot storage right away?") - ttl = odm.Integer(default=0, description="Time, in days, to live for this submission") - use_archive_alternate_dtl = odm.Boolean(default=False, + ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") + use_archive_alternate_dtl = odm.Optional(odm.Boolean(), description="Should we use the alternate dtl while archiving?") @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") From 255d4b60633f01a0c4d70892562fb2508538e25e Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:35:09 +0000 Subject: [PATCH 22/30] Allow users to change some properties of submission profiles --- assemblyline/odm/models/config.py | 4 +++- assemblyline/odm/models/user.py | 10 ++-------- assemblyline/odm/models/user_settings.py | 4 ++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 6f340c27a..66478a69e 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2020,7 +2020,9 @@ class SubmissionProfile(odm.Model): name = odm.Text(description="Submission profile name") classification = odm.ClassificationString(default=Classification.UNRESTRICTED, description="Submission profile classification") - params = odm.Compound(SubmissionProfileParams, description="Submission parameters for profile") + params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") + editable_params = odm.Mapping(odm.List(odm.Text()), default={}, + description="A list of service-specific parameters that can be configured") DEFAULT_SUBMISSION_PROFILES = [ { diff --git a/assemblyline/odm/models/user.py b/assemblyline/odm/models/user.py index 4b49c23ea..82911c2b4 100644 --- a/assemblyline/odm/models/user.py +++ b/assemblyline/odm/models/user.py @@ -13,7 +13,6 @@ ("viewer", 4), ("submitter", 5), ("custom", 6), - ("poweruser", 7) ]) ROLES = StringTable('ROLES', [ @@ -60,7 +59,6 @@ SCOPES = {"r", "w", "rw", "c"} USER_TYPES = [ TYPES.admin, # Perform administartive task and has access to all roles - TYPES.poweruser, # Advanced user of the system TYPES.user, # Normal user of the system TYPES.signature_manager, # Super user that also has access to roles for managing signatures in the system TYPES.signature_importer, # Has access to roles for importing signatures in the system @@ -100,13 +98,10 @@ ROLES.retrohunt_run, # Run yara searches ROLES.badlist_view, # View badlist items ROLES.badlist_manage, # Manage (add/delete) badlist items + ROLES.submission_customize # Allowed to customize submission properties } -USER_ROLES_ADVANCED = USER_ROLES_BASIC.union({ - ROLES.submission_customize # Allowed to customize submission properties -}) - -USER_ROLES = USER_ROLES_ADVANCED.union({ +USER_ROLES = USER_ROLES_BASIC.union({ ROLES.administration, # Perform administrative tasks ROLES.file_purge, # Purge files from the filestore ROLES.replay_system, # Manage status of file/submission/alerts during the replay process @@ -129,7 +124,6 @@ ROLES.signature_manage }), TYPES.user: USER_ROLES_BASIC, - TYPES.poweruser: USER_ROLES_ADVANCED, TYPES.viewer: { ROLES.alert_view, ROLES.apikey_access, diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index 1716ea123..a5f03393a 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -1,5 +1,6 @@ from assemblyline import odm from assemblyline.common import forge, constants +from assemblyline.odm.models.config import SubmissionProfileParams from assemblyline.odm.models.submission import ServiceSelection Classification = forge.get_classification() @@ -35,6 +36,9 @@ class UserSettings(odm.Model): priority = odm.Integer(default=1000, description="Default priority for the submissions", min=1, max=constants.MAX_PRIORITY) profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") + preferred_submission_profile = odm.Optional(odm.Text(), description="Preferred submission profile") + submission_profiles = odm.Mapping(odm.Compound(SubmissionProfileParams), default={}, + description="Default submission profile settings") service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings") services = odm.Compound(ServiceSelection, default={}, description="Default service selection") submission_view = odm.Enum(values=VIEWS, default="report", description="Default view for completed submissions") From d700c1c9c1ec8867f3eeb54e1eeda0981359f8ae Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:38:13 +0000 Subject: [PATCH 23/30] Allow for configuring core params --- assemblyline/odm/models/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 66478a69e..f16d72e64 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2022,7 +2022,7 @@ class SubmissionProfile(odm.Model): description="Submission profile classification") params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of service-specific parameters that can be configured") + description="A list of specific parameters that can be configured. The format of this configuration is `(core|).`.") DEFAULT_SUBMISSION_PROFILES = [ { From 1cd81f0f024709ed433d963515d2086271b190d8 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 01:32:10 +0000 Subject: [PATCH 24/30] Formatted the submission profiles models --- assemblyline/odm/models/config.py | 35 +++++++++++++++--------- assemblyline/odm/models/user_settings.py | 2 +- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index f16d72e64..a9edffddf 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -5,7 +5,7 @@ from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta -from assemblyline.odm.models.submission import ServiceSelection, DEFAULT_SRV_SEL +from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, ServiceSelection AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -20,6 +20,7 @@ Classification = get_classification() + @odm.model(index=False, store=False, description="Password Requirement") class PasswordRequirement(odm.Model): lower: bool = odm.Boolean(description="Password must contain lowercase letters") @@ -214,11 +215,14 @@ class OAuthProvider(odm.Model): redirect_uri: str = odm.Optional(odm.Keyword(), description="URI to redirect to after authentication with OAuth provider") request_token_url: str = odm.Optional(odm.Keyword(), description="URL to request token") - request_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters to request token") + request_token_params: Dict[str, str] = odm.Optional( + odm.Mapping(odm.Keyword()), description="Parameters to request token") access_token_url: str = odm.Optional(odm.Keyword(), description="URL to get access token") - access_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Parameters to get access token") + access_token_params: Dict[str, str] = odm.Optional(odm.Mapping( + odm.Keyword()), description="Parameters to get access token") authorize_url: str = odm.Optional(odm.Keyword(), description="URL used to authorize access to a resource") - authorize_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters used to authorize access to a resource") + authorize_params: Dict[str, str] = odm.Optional(odm.Mapping( + odm.Keyword()), description="Parameters used to authorize access to a resource") api_base_url: str = odm.Optional(odm.Keyword(), description="Base URL for downloading the user's and groups info") client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Keyword arguments passed to the different URLs") @@ -1676,6 +1680,7 @@ class APIProxies(odm.Model): DEFAULT_API_PROXIES = {} DOWNLOAD_ENCODINGS = ["cart", "raw", "zip"] + @odm.model(index=False, store=False, description="UI Configuration") class UI(odm.Model): ai: AI = odm.Compound(AI, default=DEFAULT_AI, description="AI support for the UI") @@ -1704,7 +1709,8 @@ class UI(odm.Model): description="Default API quotas values") discover_url: str = odm.Optional(odm.Keyword(), description="Discover URL") download_encoding = odm.Enum(values=DOWNLOAD_ENCODINGS, description="Which encoding will be used for downloads?") - default_zip_password = odm.Optional(odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files") + default_zip_password = odm.Optional( + odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files") email: str = odm.Optional(odm.Email(), description="Assemblyline admins email address") enforce_quota: bool = odm.Boolean(description="Enforce the user's quotas?") external_links: List[ExternalLinks] = odm.List( @@ -1992,13 +1998,14 @@ class MetadataConfig(odm.Model): } } + @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") ignore_dynamic_recursion_prevention = odm.Optional(odm.Boolean(), - description="Should we ignore dynamic recursion prevention?") + description="Should we ignore dynamic recursion prevention?") ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") @@ -2006,14 +2013,15 @@ class SubmissionProfileParams(odm.Model): priority = odm.Optional(odm.Integer(), description="Priority of the scan") services = odm.Optional(odm.Compound(ServiceSelection), description="Service selection") service_spec = odm.Optional(odm.Mapping(odm.Mapping(odm.Any())), index=False, store=False, - description="Service-specific parameters") + description="Service-specific parameters") auto_archive = odm.Optional(odm.Boolean(), - description="Does the submission automatically goes into the archive when completed?") + description="Does the submission automatically goes into the archive when completed?") delete_after_archive = odm.Optional(odm.Boolean(), - description="When the submission is archived, should we delete it from hot storage right away?") + description="When the submission is archived, should we delete it from hot storage right away?") ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") use_archive_alternate_dtl = odm.Optional(odm.Boolean(), - description="Should we use the alternate dtl while archiving?") + description="Should we use the alternate dtl while archiving?") + @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") class SubmissionProfile(odm.Model): @@ -2024,10 +2032,11 @@ class SubmissionProfile(odm.Model): editable_params = odm.Mapping(odm.List(odm.Text()), default={}, description="A list of specific parameters that can be configured. The format of this configuration is `(core|).`.") + DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis - "name": "Static Analysis", + "name": "static", "params": { "services": { "selected": DEFAULT_SRV_SEL @@ -2036,7 +2045,7 @@ class SubmissionProfile(odm.Model): }, { # Perform static analysis along with dynamic analysis - "name": "Dynamic Analysis", + "name": "dynamic", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] @@ -2046,7 +2055,7 @@ class SubmissionProfile(odm.Model): }, { # Perform static analysis along with internet connected services - "name": "Static Analysis with Internet", + "name": "static_with_internet", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index a5f03393a..b060b16c6 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -28,7 +28,7 @@ class UserSettings(odm.Model): generate_alert = odm.Boolean(default=False, description="Generate an alert?") ignore_cache = odm.Boolean(default=False, description="Ignore service caching?") - #the following 1 line can be removed after assemblyline 4.6+ + # the following 1 line can be removed after assemblyline 4.6+ ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?") ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?") ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?") From 1c2fce9968f7a4ee8c711a8f09ba6853a2252850 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 28 Jan 2025 22:44:24 +0000 Subject: [PATCH 25/30] Add addtional fields to SubmissionProfiles --- assemblyline/odm/models/config.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a9edffddf..25a3ea9f8 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2019,6 +2019,7 @@ class SubmissionProfileParams(odm.Model): delete_after_archive = odm.Optional(odm.Boolean(), description="When the submission is archived, should we delete it from hot storage right away?") ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") + type = odm.Optional(odm.Keyword(), description="Type of submission") use_archive_alternate_dtl = odm.Optional(odm.Boolean(), description="Should we use the alternate dtl while archiving?") @@ -2026,40 +2027,60 @@ class SubmissionProfileParams(odm.Model): @odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") class SubmissionProfile(odm.Model): name = odm.Text(description="Submission profile name") + display_name = odm.Text(description="Submission profile display name") classification = odm.ClassificationString(default=Classification.UNRESTRICTED, description="Submission profile classification") params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of specific parameters that can be configured. The format of this configuration is `(core|).`.") + description="A list of parameters that can be configured for this profile. The keys are the service names or \"core\" and the values are the parameters that can be configured.") +DEFAULT_EDITABLE_PARAMS = { + # Default editable params that are used in all bundled profiles + "core": ["priority", "type", "ttl"], + "CAPA": ["renderer"], + "CAPE": ["password", "analysis_timeout_in_seconds"], + "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"], + "Extract": ["password"], + "Intezer": ["dynamic_submit"], + "URLDownloader": ["proxy"], + "URLCreator": ["minimum_maliciousness"], + "XLMMacroDeobfuscator": ["password"] +} + DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis "name": "static", + "display_name": "Static Analysis", "params": { "services": { "selected": DEFAULT_SRV_SEL } - } + }, + "editable_params": DEFAULT_EDITABLE_PARAMS }, { # Perform static analysis along with dynamic analysis "name": "dynamic", + "display_name": "Static Analysis with Dynamic Analysis", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] } - } + }, + "editable_params": DEFAULT_EDITABLE_PARAMS }, { # Perform static analysis along with internet connected services "name": "static_with_internet", + "display_name": "Static Analysis with Internet Connected Services", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] - } + }, + "editable_params": DEFAULT_EDITABLE_PARAMS } }, From a3e9afe1c2576c35980abf96ca1a227f7903f0ac Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 1 Feb 2025 07:50:19 +0000 Subject: [PATCH 26/30] Update models --- assemblyline/odm/models/config.py | 10 ++-- assemblyline/odm/models/user_settings.py | 59 ++++++++++++++++++------ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 25a3ea9f8..fe9542089 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2001,11 +2001,13 @@ class MetadataConfig(odm.Model): @odm.model(index=True, store=False, description="Submission Parameters for profile") class SubmissionProfileParams(odm.Model): + classification = odm.Optional(odm.Classification(), + description="Original classification of the submission") deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") - ignore_dynamic_recursion_prevention = odm.Optional(odm.Boolean(), - description="Should we ignore dynamic recursion prevention?") + ignore_recursion_prevention = odm.Optional(odm.Boolean(), + description="Should we ignore recursion prevention?") ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") @@ -2032,12 +2034,12 @@ class SubmissionProfile(odm.Model): description="Submission profile classification") params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of parameters that can be configured for this profile. The keys are the service names or \"core\" and the values are the parameters that can be configured.") + description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.") DEFAULT_EDITABLE_PARAMS = { # Default editable params that are used in all bundled profiles - "core": ["priority", "type", "ttl"], + "submission": ["classification", "deep_scan", "generate_alert", "ignore_filtering", "priority", "type", "ttl"], "CAPA": ["renderer"], "CAPE": ["password", "analysis_timeout_in_seconds"], "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"], diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index b060b16c6..14d1a54d2 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -12,9 +12,10 @@ @odm.model(index=False, store=False, description="Model of User Settings") class UserSettings(odm.Model): classification = odm.Classification(default=Classification.UNRESTRICTED, - description="Default submission classification") - deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") - description = odm.Keyword(default="", description="Default description") + description="Default submission classification", + deprecation="This will be moved to the \"default\" submission profile") + deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?", + deprecation="This will be moved to the \"default\" submission profile") download_encoding = odm.Enum(values=ENCODINGS, default="cart", description="Default download encoding when downloading files") default_external_sources = odm.List(odm.Keyword(), default=[], @@ -25,21 +26,49 @@ class UserSettings(odm.Model): ) executive_summary = odm.Boolean(default=True, description="Should executive summary sections be shown?") expand_min_score = odm.Integer(default=500, description="Auto-expand section when score bigger then this") - generate_alert = odm.Boolean(default=False, description="Generate an alert?") - ignore_cache = odm.Boolean(default=False, description="Ignore service caching?") + generate_alert = odm.Boolean(default=False, description="Generate an alert?", + deprecation="This will be moved to the \"default\" submission profile") + ignore_cache = odm.Boolean(default=False, description="Ignore service caching?", + deprecation="This will be moved to the \"default\" submission profile") # the following 1 line can be removed after assemblyline 4.6+ - ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?") - ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?") - ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?") - malicious = odm.Boolean(default=False, description="Is the file submitted already known to be malicious?") - priority = odm.Integer(default=1000, description="Default priority for the submissions", - min=1, max=constants.MAX_PRIORITY) - profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") + ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?", + deprecation="This is replaced by `ignore_recursion_prevention`") + ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?", + deprecation="This will be moved to the \"default\" submission profile") + ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?", + deprecation="This will be moved to the \"default\" submission profile") + priority = odm.Integer(default=1000, min=1, max=constants.MAX_PRIORITY, + description="Default priority for the submissions", + deprecation="This will be moved to the \"default\" submission profile") preferred_submission_profile = odm.Optional(odm.Text(), description="Preferred submission profile") submission_profiles = odm.Mapping(odm.Compound(SubmissionProfileParams), default={}, description="Default submission profile settings") - service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings") - services = odm.Compound(ServiceSelection, default={}, description="Default service selection") + service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings", + deprecation="This will be moved to the \"default\" submission profile") + services = odm.Compound(ServiceSelection, default={}, description="Default service selection", + deprecation="This will be moved to the \"default\" submission profile") submission_view = odm.Enum(values=VIEWS, default="report", description="Default view for completed submissions") - ttl = odm.Integer(default=30, description="Default submission TTL, in days") + ttl = odm.Integer(default=30, description="Default submission TTL, in days", + deprecation="This will be moved to the \"default\" submission profile") + + +DEFAULT_USER_PROFILE_SETTINGS = { + "classification": Classification.UNRESTRICTED, + "deep_scan": False, + "download_encoding": "cart", + "default_external_sources": [], + "default_zip_password": "infected", + "executive_summary": True, + "expand_min_score": 500, + "generate_alert": False, + "ignore_cache": False, + "ignore_dynamic_recursion_prevention": False, + "ignore_recursion_prevention": False, + "ignore_filtering": False, + "priority": 1000, + "service_spec": {}, + "services": {}, + "submission_view": "report", + "ttl": 30 +} From 805353746f97be349a960f67a98a44811b191cc1 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 1 Feb 2025 23:53:45 +0000 Subject: [PATCH 27/30] Fix merge conflicts --- assemblyline/odm/models/config.py | 9 --------- assemblyline/odm/models/user_settings.py | 4 ---- 2 files changed, 13 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index 2bacedb1c..fe9542089 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -1,10 +1,7 @@ from typing import Any, Dict, List from assemblyline import odm -<<<<<<< HEAD from assemblyline.common.constants import PRIORITIES -======= ->>>>>>> refs/remotes/origin/AL-2646 from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta @@ -2127,14 +2124,11 @@ class Submission(odm.Model): description="Tag types that show up in the submission summary") verdicts = odm.Compound(Verdicts, default=DEFAULT_VERDICTS, description="Minimum score value to get the specified verdict.") -<<<<<<< HEAD default_temporary_keys: dict[str, str] = odm.mapping(odm.enum(TEMPORARY_KEY_TYPE), description="temporary_keys values for well known services.") temporary_keys: dict[str, str] = odm.mapping(odm.enum(TEMPORARY_KEY_TYPE), description="Set the operation that will be used to update values " "using this key in the temporary submission data.") -======= ->>>>>>> refs/remotes/origin/AL-2646 profiles = odm.List(odm.Compound(SubmissionProfile), description="Submission profiles with preset submission parameters") @@ -2159,11 +2153,8 @@ class Submission(odm.Model): 'file_sources': [], 'tag_types': DEFAULT_TAG_TYPES, 'verdicts': DEFAULT_VERDICTS, -<<<<<<< HEAD 'default_temporary_keys': DEFAULT_TEMPORARY_KEYS, 'temporary_keys': {}, -======= ->>>>>>> refs/remotes/origin/AL-2646 'profiles': DEFAULT_SUBMISSION_PROFILES } diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index ab1b1502e..14d1a54d2 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -1,9 +1,5 @@ from assemblyline import odm -<<<<<<< HEAD from assemblyline.common import forge, constants -======= -from assemblyline.common import forge ->>>>>>> refs/remotes/origin/AL-2646 from assemblyline.odm.models.config import SubmissionProfileParams from assemblyline.odm.models.submission import ServiceSelection From 3db3ffa4b757170773c61997ac02393f5b19602d Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 8 Feb 2025 07:07:04 +0000 Subject: [PATCH 28/30] Fix static w/ internet profile + add ignore_cache to default editable params --- assemblyline/odm/models/config.py | 33 +++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index a4a05c5b2..f2404b45f 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2028,20 +2028,9 @@ class SubmissionProfileParams(odm.Model): description="Should we use the alternate dtl while archiving?") -@odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") -class SubmissionProfile(odm.Model): - name = odm.Text(description="Submission profile name") - display_name = odm.Text(description="Submission profile display name") - classification = odm.ClassificationString(default=Classification.UNRESTRICTED, - description="Submission profile classification") - params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") - editable_params = odm.Mapping(odm.List(odm.Text()), default={}, - description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.") - - DEFAULT_EDITABLE_PARAMS = { - # Default editable params that are used in all bundled profiles - "submission": ["classification", "deep_scan", "generate_alert", "ignore_filtering", "priority", "type", "ttl"], + # Default editable params that are used in all profiles + "submission": ["classification", "deep_scan", "ignore_cache", "generate_alert", "ignore_filtering", "priority", "type", "ttl"], "CAPA": ["renderer"], "CAPE": ["password", "analysis_timeout_in_seconds"], "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"], @@ -2052,6 +2041,17 @@ class SubmissionProfile(odm.Model): "XLMMacroDeobfuscator": ["password"] } +@odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") +class SubmissionProfile(odm.Model): + name = odm.Text(description="Submission profile name") + display_name = odm.Text(description="Submission profile display name") + classification = odm.ClassificationString(default=Classification.UNRESTRICTED, + description="Submission profile classification") + params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") + editable_params = odm.Mapping(odm.List(odm.Text()), default=DEFAULT_EDITABLE_PARAMS, + description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.") + + DEFAULT_SUBMISSION_PROFILES = [ { # Only perform static analysis @@ -2062,7 +2062,6 @@ class SubmissionProfile(odm.Model): "selected": DEFAULT_SRV_SEL } }, - "editable_params": DEFAULT_EDITABLE_PARAMS }, { # Perform static analysis along with dynamic analysis @@ -2073,8 +2072,6 @@ class SubmissionProfile(odm.Model): "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] } }, - "editable_params": DEFAULT_EDITABLE_PARAMS - }, { # Perform static analysis along with internet connected services @@ -2084,9 +2081,7 @@ class SubmissionProfile(odm.Model): "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] }, - "editable_params": DEFAULT_EDITABLE_PARAMS - } - + }, }, ] From aa3e87cea0c1933258b1590c82bb95a764d60730 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 11 Feb 2025 21:16:26 +0000 Subject: [PATCH 29/30] Update profile name to be more accurate --- assemblyline/odm/models/config.py | 6 +++--- assemblyline/odm/random_data/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index f2404b45f..c65e6b4cd 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2065,8 +2065,8 @@ class SubmissionProfile(odm.Model): }, { # Perform static analysis along with dynamic analysis - "name": "dynamic", - "display_name": "Static Analysis with Dynamic Analysis", + "name": "static_with_dynamic", + "display_name": "Static + Dynamic Analysis", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] @@ -2076,7 +2076,7 @@ class SubmissionProfile(odm.Model): { # Perform static analysis along with internet connected services "name": "static_with_internet", - "display_name": "Static Analysis with Internet Connected Services", + "display_name": "Static Analysis with Internet Access", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] diff --git a/assemblyline/odm/random_data/__init__.py b/assemblyline/odm/random_data/__init__.py index 48b35684e..2eaa91d3f 100644 --- a/assemblyline/odm/random_data/__init__.py +++ b/assemblyline/odm/random_data/__init__.py @@ -405,7 +405,7 @@ def create_users(ds, log=None): "type": [TYPES.admin]}) ds.user.save('admin', user_data) ds.user_settings.save('admin', UserSettings({"ignore_cache": True, "deep_scan": True, - "default_zip_password": "infected", "default_download_encoding": "cart"})) + "default_zip_password": "infected", "default_download_encoding": "cart", "submission_profiles": {}})) if log: log.info(f"\tU:{user_data.uname} P:{admin_pass}") From ac1a66bb2fbe7ead444caa48a552d0c220489718 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 12 Feb 2025 14:49:17 +0000 Subject: [PATCH 30/30] Add descriptions to better describe submission profiles --- assemblyline/odm/models/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index c65e6b4cd..f5c5f95c2 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2050,6 +2050,7 @@ class SubmissionProfile(odm.Model): params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") editable_params = odm.Mapping(odm.List(odm.Text()), default=DEFAULT_EDITABLE_PARAMS, description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.") + description = odm.Optional(odm.Text(), description="A description of what the profile does") DEFAULT_SUBMISSION_PROFILES = [ @@ -2062,6 +2063,7 @@ class SubmissionProfile(odm.Model): "selected": DEFAULT_SRV_SEL } }, + "description": "Analyze files using static analysis techniques and extract information from the file without executing it, such as metadata, strings, and structural information." }, { # Perform static analysis along with dynamic analysis @@ -2072,16 +2074,18 @@ class SubmissionProfile(odm.Model): "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] } }, + "description": "Analyze files using static analysis techniques along with executing them in a controlled environment to observe their behavior and capture runtime activities, interactions with the system, network communications, and any malicious behavior exhibited by the file during execution." }, { # Perform static analysis along with internet connected services "name": "static_with_internet", - "display_name": "Static Analysis with Internet Access", + "display_name": "Internet-Connected Static Analysis", "params": { "services": { "selected": DEFAULT_SRV_SEL + ["Internet Connected"] }, }, + "description": "Combine traditional static analysis techniques with internet-connected services to gather additional information and context about the file being analyzed." }, ]