diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py index b77fbc090..f5c5f95c2 100644 --- a/assemblyline/odm/models/config.py +++ b/assemblyline/odm/models/config.py @@ -2,8 +2,10 @@ from assemblyline import odm from assemblyline.common.constants import PRIORITIES +from assemblyline.common.forge import get_classification from assemblyline.odm.models.service import EnvironmentVariable from assemblyline.odm.models.service_delta import DockerConfigDelta +from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, ServiceSelection AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group', 'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota', @@ -16,6 +18,8 @@ DEFAULT_SUBMISSION_QUOTA = 5 DEFAULT_ASYNC_SUBMISSION_QUOTA = 0 +Classification = get_classification() + @odm.model(index=False, store=False, description="Password Requirement") class PasswordRequirement(odm.Model): @@ -211,11 +215,14 @@ class OAuthProvider(odm.Model): redirect_uri: str = odm.Optional(odm.Keyword(), description="URI to redirect to after authentication with OAuth provider") request_token_url: str = odm.Optional(odm.Keyword(), description="URL to request token") - request_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters to request token") + request_token_params: Dict[str, str] = odm.Optional( + odm.Mapping(odm.Keyword()), description="Parameters to request token") access_token_url: str = odm.Optional(odm.Keyword(), description="URL to get access token") - access_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Parameters to get access token") + access_token_params: Dict[str, str] = odm.Optional(odm.Mapping( + odm.Keyword()), description="Parameters to get access token") authorize_url: str = odm.Optional(odm.Keyword(), description="URL used to authorize access to a resource") - authorize_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters used to authorize access to a resource") + authorize_params: Dict[str, str] = odm.Optional(odm.Mapping( + odm.Keyword()), description="Parameters used to authorize access to a resource") api_base_url: str = odm.Optional(odm.Keyword(), description="Base URL for downloading the user's and groups info") client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Keyword arguments passed to the different URLs") @@ -1675,6 +1682,7 @@ class APIProxies(odm.Model): DEFAULT_API_PROXIES = {} DOWNLOAD_ENCODINGS = ["cart", "raw", "zip"] + @odm.model(index=False, store=False, description="UI Configuration") class UI(odm.Model): ai: AI = odm.Compound(AI, default=DEFAULT_AI, description="AI support for the UI") @@ -1703,7 +1711,8 @@ class UI(odm.Model): description="Default API quotas values") discover_url: str = odm.Optional(odm.Keyword(), description="Discover URL") download_encoding = odm.Enum(values=DOWNLOAD_ENCODINGS, description="Which encoding will be used for downloads?") - default_zip_password = odm.Optional(odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files") + default_zip_password = odm.Optional( + odm.Text(), description="Default user-defined password for creating password protected ZIPs when downloading files") email: str = odm.Optional(odm.Email(), description="Assemblyline admins email address") enforce_quota: bool = odm.Boolean(description="Enforce the user's quotas?") external_links: List[ExternalLinks] = odm.List( @@ -1992,6 +2001,94 @@ class MetadataConfig(odm.Model): } +@odm.model(index=True, store=False, description="Submission Parameters for profile") +class SubmissionProfileParams(odm.Model): + classification = odm.Optional(odm.Classification(), + description="Original classification of the submission") + deep_scan = odm.Optional(odm.Boolean(), description="Should a deep scan be performed?") + generate_alert = odm.Optional(odm.Boolean(), description="Should this submission generate an alert?") + ignore_cache = odm.Optional(odm.Boolean(), description="Ignore the cached service results?") + ignore_recursion_prevention = odm.Optional(odm.Boolean(), + description="Should we ignore recursion prevention?") + ignore_filtering = odm.Optional(odm.Boolean(), description="Should we ignore filtering services?") + ignore_size = odm.Optional(odm.Boolean(), description="Ignore the file size limits?") + max_extracted = odm.Optional(odm.Integer(), description="Max number of extracted files") + max_supplementary = odm.Optional(odm.Integer(), description="Max number of supplementary files") + priority = odm.Optional(odm.Integer(), description="Priority of the scan") + services = odm.Optional(odm.Compound(ServiceSelection), description="Service selection") + service_spec = odm.Optional(odm.Mapping(odm.Mapping(odm.Any())), index=False, store=False, + description="Service-specific parameters") + auto_archive = odm.Optional(odm.Boolean(), + description="Does the submission automatically goes into the archive when completed?") + delete_after_archive = odm.Optional(odm.Boolean(), + description="When the submission is archived, should we delete it from hot storage right away?") + ttl = odm.Optional(odm.Integer(), description="Time, in days, to live for this submission") + type = odm.Optional(odm.Keyword(), description="Type of submission") + use_archive_alternate_dtl = odm.Optional(odm.Boolean(), + description="Should we use the alternate dtl while archiving?") + + +DEFAULT_EDITABLE_PARAMS = { + # Default editable params that are used in all profiles + "submission": ["classification", "deep_scan", "ignore_cache", "generate_alert", "ignore_filtering", "priority", "type", "ttl"], + "CAPA": ["renderer"], + "CAPE": ["password", "analysis_timeout_in_seconds"], + "DocumentPreview": ["analyze_render", "max_pages_rendered", "run_ocr_on_first_n_pages"], + "Extract": ["password"], + "Intezer": ["dynamic_submit"], + "URLDownloader": ["proxy"], + "URLCreator": ["minimum_maliciousness"], + "XLMMacroDeobfuscator": ["password"] +} + +@odm.model(index=False, store=False, description="Configuration for defining submission profiles for basic users") +class SubmissionProfile(odm.Model): + name = odm.Text(description="Submission profile name") + display_name = odm.Text(description="Submission profile display name") + classification = odm.ClassificationString(default=Classification.UNRESTRICTED, + description="Submission profile classification") + params = odm.Compound(SubmissionProfileParams, description="Default submission parameters for profile") + editable_params = odm.Mapping(odm.List(odm.Text()), default=DEFAULT_EDITABLE_PARAMS, + description="A list of parameters that can be configured for this profile. The keys are the service names or \"submission\" and the values are the parameters that can be configured.") + description = odm.Optional(odm.Text(), description="A description of what the profile does") + + +DEFAULT_SUBMISSION_PROFILES = [ + { + # Only perform static analysis + "name": "static", + "display_name": "Static Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + } + }, + "description": "Analyze files using static analysis techniques and extract information from the file without executing it, such as metadata, strings, and structural information." + }, + { + # Perform static analysis along with dynamic analysis + "name": "static_with_dynamic", + "display_name": "Static + Dynamic Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + ["Dynamic Analysis"] + } + }, + "description": "Analyze files using static analysis techniques along with executing them in a controlled environment to observe their behavior and capture runtime activities, interactions with the system, network communications, and any malicious behavior exhibited by the file during execution." + }, + { + # Perform static analysis along with internet connected services + "name": "static_with_internet", + "display_name": "Internet-Connected Static Analysis", + "params": { + "services": { + "selected": DEFAULT_SRV_SEL + ["Internet Connected"] + }, + }, + "description": "Combine traditional static analysis techniques with internet-connected services to gather additional information and context about the file being analyzed." + }, +] + TEMPORARY_KEY_TYPE = [ # Keep this key as submission wide list merging equal items 'union', @@ -2033,6 +2130,8 @@ class Submission(odm.Model): temporary_keys: dict[str, str] = odm.mapping(odm.enum(TEMPORARY_KEY_TYPE), description="Set the operation that will be used to update values " "using this key in the temporary submission data.") + profiles = odm.List(odm.Compound(SubmissionProfile), + description="Submission profiles with preset submission parameters") DEFAULT_TEMPORARY_KEYS = { @@ -2057,6 +2156,7 @@ class Submission(odm.Model): 'verdicts': DEFAULT_VERDICTS, 'default_temporary_keys': DEFAULT_TEMPORARY_KEYS, 'temporary_keys': {}, + 'profiles': DEFAULT_SUBMISSION_PROFILES } diff --git a/assemblyline/odm/models/submission.py b/assemblyline/odm/models/submission.py index 23615a4a0..df599b5a7 100644 --- a/assemblyline/odm/models/submission.py +++ b/assemblyline/odm/models/submission.py @@ -26,8 +26,6 @@ class ServiceSelection(odm.Model): description="List of services to rescan when initial run scores as malicious") resubmit = odm.List(odm.Keyword(), default=DEFAULT_RESUBMIT, description="Add to service selection when resubmitting") - runtime_excluded = odm.List(odm.Keyword(), default=[], description="List of runtime excluded services") - # Fields in the parameters used to calculate hashes used for result caching _KEY_HASHED_FIELDS = { @@ -67,7 +65,6 @@ class SubmissionParams(odm.Model): max_extracted = odm.Integer(default=500, description="Max number of extracted files") max_supplementary = odm.Integer(default=500, description="Max number of supplementary files") priority = odm.Integer(default=1000, description="Priority of the scan", min=1, max=constants.MAX_PRIORITY) - profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") psid = odm.Optional(odm.UUID(), description="Parent submission ID") quota_item = odm.Boolean(default=False, description="Does this submission count against quota?") services = odm.Compound(ServiceSelection, default={}, description="Service selection") diff --git a/assemblyline/odm/models/user.py b/assemblyline/odm/models/user.py index a9aaf2c90..82911c2b4 100644 --- a/assemblyline/odm/models/user.py +++ b/assemblyline/odm/models/user.py @@ -12,7 +12,7 @@ ("signature_importer", 3), ("viewer", 4), ("submitter", 5), - ("custom", 6) + ("custom", 6), ]) ROLES = StringTable('ROLES', [ @@ -52,6 +52,7 @@ ("badlist_manage", 32), ("archive_comment", 33), ("assistant_use", 34), + ("submission_customize", 35) ]) @@ -97,6 +98,7 @@ ROLES.retrohunt_run, # Run yara searches ROLES.badlist_view, # View badlist items ROLES.badlist_manage, # Manage (add/delete) badlist items + ROLES.submission_customize # Allowed to customize submission properties } USER_ROLES = USER_ROLES_BASIC.union({ @@ -173,6 +175,7 @@ ROLES.submission_create, ROLES.submission_delete, ROLES.submission_manage, + ROLES.submission_customize, ROLES.retrohunt_run, ], "E": [ diff --git a/assemblyline/odm/models/user_settings.py b/assemblyline/odm/models/user_settings.py index 1716ea123..14d1a54d2 100644 --- a/assemblyline/odm/models/user_settings.py +++ b/assemblyline/odm/models/user_settings.py @@ -1,5 +1,6 @@ from assemblyline import odm from assemblyline.common import forge, constants +from assemblyline.odm.models.config import SubmissionProfileParams from assemblyline.odm.models.submission import ServiceSelection Classification = forge.get_classification() @@ -11,9 +12,10 @@ @odm.model(index=False, store=False, description="Model of User Settings") class UserSettings(odm.Model): classification = odm.Classification(default=Classification.UNRESTRICTED, - description="Default submission classification") - deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?") - description = odm.Keyword(default="", description="Default description") + description="Default submission classification", + deprecation="This will be moved to the \"default\" submission profile") + deep_scan = odm.Boolean(default=False, description="Should a deep scan be performed?", + deprecation="This will be moved to the \"default\" submission profile") download_encoding = odm.Enum(values=ENCODINGS, default="cart", description="Default download encoding when downloading files") default_external_sources = odm.List(odm.Keyword(), default=[], @@ -24,18 +26,49 @@ class UserSettings(odm.Model): ) executive_summary = odm.Boolean(default=True, description="Should executive summary sections be shown?") expand_min_score = odm.Integer(default=500, description="Auto-expand section when score bigger then this") - generate_alert = odm.Boolean(default=False, description="Generate an alert?") - ignore_cache = odm.Boolean(default=False, description="Ignore service caching?") + generate_alert = odm.Boolean(default=False, description="Generate an alert?", + deprecation="This will be moved to the \"default\" submission profile") + ignore_cache = odm.Boolean(default=False, description="Ignore service caching?", + deprecation="This will be moved to the \"default\" submission profile") - #the following 1 line can be removed after assemblyline 4.6+ - ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?") - ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?") - ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?") - malicious = odm.Boolean(default=False, description="Is the file submitted already known to be malicious?") - priority = odm.Integer(default=1000, description="Default priority for the submissions", - min=1, max=constants.MAX_PRIORITY) - profile = odm.Boolean(default=False, description="Should the submission do extra profiling?") - service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings") - services = odm.Compound(ServiceSelection, default={}, description="Default service selection") + # the following 1 line can be removed after assemblyline 4.6+ + ignore_dynamic_recursion_prevention = odm.Boolean(default=False, description="Ignore dynamic recursion prevention?", + deprecation="This is replaced by `ignore_recursion_prevention`") + ignore_recursion_prevention = odm.Boolean(default=False, description="Ignore all service recursion prevention?", + deprecation="This will be moved to the \"default\" submission profile") + ignore_filtering = odm.Boolean(default=False, description="Ignore filtering services?", + deprecation="This will be moved to the \"default\" submission profile") + priority = odm.Integer(default=1000, min=1, max=constants.MAX_PRIORITY, + description="Default priority for the submissions", + deprecation="This will be moved to the \"default\" submission profile") + preferred_submission_profile = odm.Optional(odm.Text(), description="Preferred submission profile") + submission_profiles = odm.Mapping(odm.Compound(SubmissionProfileParams), default={}, + description="Default submission profile settings") + service_spec = odm.Mapping(odm.Mapping(odm.Any()), default={}, description="Default service specific settings", + deprecation="This will be moved to the \"default\" submission profile") + services = odm.Compound(ServiceSelection, default={}, description="Default service selection", + deprecation="This will be moved to the \"default\" submission profile") submission_view = odm.Enum(values=VIEWS, default="report", description="Default view for completed submissions") - ttl = odm.Integer(default=30, description="Default submission TTL, in days") + ttl = odm.Integer(default=30, description="Default submission TTL, in days", + deprecation="This will be moved to the \"default\" submission profile") + + +DEFAULT_USER_PROFILE_SETTINGS = { + "classification": Classification.UNRESTRICTED, + "deep_scan": False, + "download_encoding": "cart", + "default_external_sources": [], + "default_zip_password": "infected", + "executive_summary": True, + "expand_min_score": 500, + "generate_alert": False, + "ignore_cache": False, + "ignore_dynamic_recursion_prevention": False, + "ignore_recursion_prevention": False, + "ignore_filtering": False, + "priority": 1000, + "service_spec": {}, + "services": {}, + "submission_view": "report", + "ttl": 30 +} diff --git a/assemblyline/odm/random_data/__init__.py b/assemblyline/odm/random_data/__init__.py index 48b35684e..2eaa91d3f 100644 --- a/assemblyline/odm/random_data/__init__.py +++ b/assemblyline/odm/random_data/__init__.py @@ -405,7 +405,7 @@ def create_users(ds, log=None): "type": [TYPES.admin]}) ds.user.save('admin', user_data) ds.user_settings.save('admin', UserSettings({"ignore_cache": True, "deep_scan": True, - "default_zip_password": "infected", "default_download_encoding": "cart"})) + "default_zip_password": "infected", "default_download_encoding": "cart", "submission_profiles": {}})) if log: log.info(f"\tU:{user_data.uname} P:{admin_pass}")