From 7b889b676c6ea51975bd69e989526050247b6f40 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:44:56 +0000 Subject: [PATCH 01/47] Guard submit & ingest API from unauthorized submission customization --- assemblyline_ui/api/v4/ingest.py | 5 +++-- assemblyline_ui/api/v4/submit.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index fc28e7a3..3bf08ed1 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -267,8 +267,9 @@ def ingest_single_file(**kwargs): "type": "INGEST" }) - # Apply provided params - s_params.update(data.get("params", {})) + # Apply provided params (if the user is allowed to) + if ROLES.submission_customize in user['roles']: + s_params.update(data.get("params", {})) # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 2de55e7a..49af8a54 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -328,7 +328,11 @@ def submit(**kwargs): # Create task object s_params = ui_to_submission_params(user_settings) - s_params.update(data.get("params", {})) + + # Apply provided params (if the user is allowed to) + if ROLES.submission_customize in user['roles']: + s_params.update(data.get("params", {})) + default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources if 'groups' not in s_params: s_params['groups'] = [g for g in user['groups'] if g in s_params['classification']] From 4592c65f937048b672839099a4e1431346ebe826 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:46:09 +0000 Subject: [PATCH 02/47] Setup using submission profiles on submission --- assemblyline_ui/api/v4/submit.py | 37 +++++++++++++++++++++++++------- assemblyline_ui/api/v4/user.py | 7 +++++- assemblyline_ui/config.py | 7 +++++- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 49af8a54..49cd9670 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -11,12 +11,13 @@ from assemblyline.common.dict_utils import flatten from assemblyline.common.str_utils import safe_str from assemblyline.common.uid import get_random_id +from assemblyline.odm.models.config import SubmissionProfile from assemblyline.odm.messages.submission import Submission from assemblyline.odm.models.user import ROLES from assemblyline_core.submission_client import SubmissionClient, SubmissionException from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import ARCHIVESTORE, STORAGE, TEMP_SUBMIT_DIR, FILESTORE, config, \ - CLASSIFICATION as Classification, IDENTIFY, metadata_validator + CLASSIFICATION as Classification, IDENTIFY, metadata_validator, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ FETCH_METHODS @@ -245,15 +246,17 @@ def submit(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "profile": "Static Analysis", # Name of submission profile to use + + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v3/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v4/user/submission_params// } Data Block (Binary): @@ -328,10 +331,28 @@ def submit(**kwargs): # Create task object s_params = ui_to_submission_params(user_settings) + s_profile = SUBMISSION_PROFILES.get(data.get('profile')) + if s_profile and not Classification.is_accessible(user['classification'], s_profile.classification): + # User isn't allowed to use the submission profile specified + return make_api_response({}, f"You aren't allowed to use '{s_profile.name}' submission profile", 400) - # Apply provided params (if the user is allowed to) if ROLES.submission_customize in user['roles']: + # Apply provided params (if the user is allowed to) s_params.update(data.get("params", {})) + elif s_profile: + # Apply the profile (but allow the user to change some properties) + s_params.update(s_profile.params.as_primitives()) + params_data = data.get("params", {}) + for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: + if param in params_data: + # Overwrite/Set parameter with user-defined input + s_params[param] = params_data[param] + + else: + # No profile specified, raise an exception back to the user + return make_api_response({}, "You must specify a submission profile. " \ + f"One of: {list(SUBMISSION_PROFILES.keys())}", 400) + default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources if 'groups' not in s_params: diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index f2808ed8..4d9aa2e1 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -13,7 +13,7 @@ from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, UI_MESSAGING, \ - VERSION, config, AI_AGENT, UI_METADATA_VALIDATION + VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( @@ -91,10 +91,12 @@ def who_am_i(**kwargs): "max_dtl": 30, # Maximum number of days retrohunt job stay in the system }, "submission": { # Submission Configuration + "configurable_params": [], # Submission parameters that are configurable when using profiles "dtl": 10, # Default number of days submission stay in the system "max_dtl": 30, # Maximum number of days submission stay in the system "file_sources": [], # List of file sources to perform remote submission into the system "metadata": {}, # Metadata compliance policy to submit to the system + "profiles": {}, # Submission profiles "verdicts": { # Verdict scoring configuration "info": 0, # Default minimum score for info "suspicious": 300, # Default minimum score for suspicious @@ -213,10 +215,13 @@ def who_am_i(**kwargs): "max_dtl": config.retrohunt.max_dtl, }, "submission": { + "configurable_params": USER_CONFIGURABLE_SUBMISSION_PARAMS, "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, + "profiles": {name: profile.params.as_primitives() for name, profile in SUBMISSION_PROFILES.items() \ + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification)}, "verdicts": { "info": config.submission.verdicts.info, "suspicious": config.submission.verdicts.suspicious, diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index d1e32991..515eee5d 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -9,7 +9,8 @@ from assemblyline.common.version import BUILD_MINOR, FRAMEWORK_VERSION, SYSTEM_VERSION from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator from assemblyline.filestore import FileStore -from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP +from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP, SubmissionProfileParams +from assemblyline.odm.models.submission import SubmissionParams from assemblyline.remote.datatypes import get_client from assemblyline.remote.datatypes.cache import Cache from assemblyline.remote.datatypes.daily_quota_tracker import DailyQuotaTracker @@ -170,5 +171,9 @@ def get_signup_queue(key): ARCHIVE_MANAGER: ArchiveManager = ArchiveManager( config=config, datastore=STORAGE, filestore=FILESTORE, identify=IDENTIFY) SERVICE_LIST = forge.CachedObject(STORAGE.list_all_services, kwargs=dict(as_obj=False, full=True)) +SUBMISSION_PROFILES = {profile.name: profile for profile in config.submission.profiles} +USER_CONFIGURABLE_SUBMISSION_PARAMS = list(set(SubmissionParams.fields().keys()) - \ + set(SubmissionProfileParams.fields().keys()) - \ + set(['quota_item', 'type', 'groups'])) # End global ################################################################# From c01daec42c5cf5e76aaafdcb807e3c85054a5442 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:46:32 +0000 Subject: [PATCH 03/47] Use a method function for setting user-specified submission parameters --- assemblyline_ui/api/v4/ingest.py | 10 +++++---- assemblyline_ui/api/v4/submit.py | 29 ++++++------------------ assemblyline_ui/helper/submission.py | 23 +++++++++++++++++-- test/test_ingest.py | 32 ++++++++++++++++++++++++++- test/test_submit.py | 33 +++++++++++++++++++++++++++- 5 files changed, 97 insertions(+), 30 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index 3bf08ed1..b2282e0e 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -21,7 +21,7 @@ STORAGE, config, FILESTORE, metadata_validator from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ - FETCH_METHODS + FETCH_METHODS, update_submission_parameters from assemblyline_ui.helper.user import check_async_submission_quota, decrement_submission_ingest_quota, \ load_user_settings @@ -267,9 +267,11 @@ def ingest_single_file(**kwargs): "type": "INGEST" }) - # Apply provided params (if the user is allowed to) - if ROLES.submission_customize in user['roles']: - s_params.update(data.get("params", {})) + # Update submission parameters as specified by the user + try: + update_submission_parameters(s_params, data, user) + except Exception as e: + return make_api_response({}, str(e), 400) # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 49cd9670..d3f4266e 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -17,10 +17,10 @@ from assemblyline_core.submission_client import SubmissionClient, SubmissionException from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import ARCHIVESTORE, STORAGE, TEMP_SUBMIT_DIR, FILESTORE, config, \ - CLASSIFICATION as Classification, IDENTIFY, metadata_validator, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS + CLASSIFICATION as Classification, IDENTIFY, metadata_validator from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ - FETCH_METHODS + FETCH_METHODS, update_submission_parameters from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota, load_user_settings SUB_API = 'submit' @@ -331,27 +331,12 @@ def submit(**kwargs): # Create task object s_params = ui_to_submission_params(user_settings) - s_profile = SUBMISSION_PROFILES.get(data.get('profile')) - if s_profile and not Classification.is_accessible(user['classification'], s_profile.classification): - # User isn't allowed to use the submission profile specified - return make_api_response({}, f"You aren't allowed to use '{s_profile.name}' submission profile", 400) - - if ROLES.submission_customize in user['roles']: - # Apply provided params (if the user is allowed to) - s_params.update(data.get("params", {})) - elif s_profile: - # Apply the profile (but allow the user to change some properties) - s_params.update(s_profile.params.as_primitives()) - params_data = data.get("params", {}) - for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: - if param in params_data: - # Overwrite/Set parameter with user-defined input - s_params[param] = params_data[param] - else: - # No profile specified, raise an exception back to the user - return make_api_response({}, "You must specify a submission profile. " \ - f"One of: {list(SUBMISSION_PROFILES.keys())}", 400) + # Update submission parameters as specified by the user + try: + update_submission_parameters(s_params, data, user) + except Exception as e: + return make_api_response({}, str(e), 400) default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index aff9a066..cad5d9d3 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -16,7 +16,7 @@ from assemblyline.odm.models.config import HASH_PATTERN_MAP from assemblyline.odm.messages.submission import SubmissionMessage from assemblyline.odm.models.user import ROLES -from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE +from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS # Baseline fetch methods FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url']) @@ -30,6 +30,7 @@ MYIP = '127.0.0.1' + ############################# # download functions class FileTooBigException(Exception): @@ -147,7 +148,25 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo - +def update_submission_parameters(s_params: dict, data: dict, user: dict): + s_profile = SUBMISSION_PROFILES.get(data.get('profile')) + # Apply provided params (if the user is allowed to) + if ROLES.submission_customize in user['roles']: + s_params.update(data.get("params", {})) + elif s_profile: + if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification): + # User isn't allowed to use the submission profile specified + raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") + # Apply the profile (but allow the user to change some properties) + s_params.update(s_profile.params.as_primitives()) + params_data = data.get("params", {}) + for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: + if param in params_data: + # Overwrite/Set parameter with user-defined input + s_params[param] = params_data[param] + else: + # No profile specified, raise an exception back to the user + raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") diff --git a/test/test_ingest.py b/test/test_ingest.py index d63b2e84..d1d58b8d 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -6,7 +6,7 @@ import random import tempfile -from conftest import get_api_data +from conftest import get_api_data, APIError from assemblyline.common import forge from assemblyline.odm.messages.submission import Submission @@ -326,3 +326,33 @@ def test_get_message_list_with_paging(datastore, login_session): message_list += resp for x in range(NUM_FILES): assert message_list[x] == messages[x] + +def test_ingest_submission_profile(datastore, login_session, scheduler): + _, session, host = login_session + iq.delete() + + # Make the user a simple user and try to submit + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'type', 'user')]) + byte_str = get_random_phrase(wmin=30, wmax=75).encode() + sha256 = hashlib.sha256(byte_str).hexdigest() + data = { + 'base64': base64.b64encode(byte_str).decode('ascii'), + 'metadata': {'test': 'test_submit_base64_nameless'} + } + with pytest.raises(APIError, match="You must specify a submission profile"): + # A basic user must specify a submission profile name + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with no parameters + data['profile'] = "Static Analysis" + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with a parameter you aren't allowed to set + # The system should silently ignore your parameter and still create a submission + data['params'] = {'deep_scan': True} + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Restore original roles for later tests + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), + (datastore.user.UPDATE_APPEND, 'type', 'admin')]) diff --git a/test/test_submit.py b/test/test_submit.py index 60df0227..9d117db2 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -6,7 +6,7 @@ import random import tempfile -from conftest import get_api_data +from conftest import get_api_data, APIError from assemblyline.common import forge from assemblyline.odm.models.config import HASH_PATTERN_MAP @@ -282,3 +282,34 @@ def test_submit_base64_nameless(datastore, login_session, scheduler): msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) assert msg.submission.sid == resp['sid'] + +def test_submit_submission_profile(datastore, login_session, scheduler): + _, session, host = login_session + sq.delete() + + # Make the user a simple user and try to submit + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'type', 'user')]) + byte_str = get_random_phrase(wmin=30, wmax=75).encode() + sha256 = hashlib.sha256(byte_str).hexdigest() + data = { + 'base64': base64.b64encode(byte_str).decode('ascii'), + 'metadata': {'test': 'test_submit_base64_nameless'} + } + with pytest.raises(APIError, match="You must specify a submission profile"): + # A basic user must specify a submission profile name + get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with no parameters + data['profile'] = "Static Analysis" + get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with a parameter you aren't allowed to set + # The system should silently ignore your parameter and still create a submission + data['params'] = {'deep_scan': True} + resp = get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + assert resp['params']['deep_scan'] == False + + # Restore original roles for later tests + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), + (datastore.user.UPDATE_APPEND, 'type', 'admin')]) From a4943400a73ab8bc5aa1118c999934c4a569259b Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:34:46 +0000 Subject: [PATCH 04/47] Rename parameter to deconflict with pre-existing `profile` parameter --- assemblyline_ui/api/v4/ingest.py | 14 ++++++++------ assemblyline_ui/api/v4/submission.py | 2 -- assemblyline_ui/api/v4/submit.py | 14 +++++++------- assemblyline_ui/api/v4/ui.py | 9 ++++++++- assemblyline_ui/api/v4/user.py | 6 +----- assemblyline_ui/config.py | 4 +--- assemblyline_ui/helper/submission.py | 13 +++++++------ 7 files changed, 32 insertions(+), 30 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index b2282e0e..6eca2882 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -150,15 +150,17 @@ def ingest_single_file(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "profile_name": "Static Analysis", # Name of submission profile to use + + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v3/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v3/user/submission_params// "generate_alert": False, # Generate an alert in our alerting system or not "notification_queue": None, # Name of the notification queue diff --git a/assemblyline_ui/api/v4/submission.py b/assemblyline_ui/api/v4/submission.py index 5f8e68a2..3a74e4e8 100644 --- a/assemblyline_ui/api/v4/submission.py +++ b/assemblyline_ui/api/v4/submission.py @@ -312,7 +312,6 @@ def get_full_results(sid, **kwargs): }, "state": "completed", # State of the submission "submission": { # Submission Block - "profile": true, # Should keep stats about execution? "description": "", # Submission description "ttl": 30, # Submission days to live "ignore_filtering": false, # Ignore filtering services? @@ -451,7 +450,6 @@ def get_submission(sid, **kwargs): ["FNAME", "sha256"], ...], # Each file = List of name/sha256 "errors": [], # List of error keys (sha256.ServiceName) "submission": { # Submission Block - "profile": true, # Should keep stats about execution? "description": "", # Submission description "ttl": 30, # Submission days to live "ignore_filtering": false, # Ignore filtering services? diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index d3f4266e..9e4441da 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -246,17 +246,17 @@ def submit(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "profile": "Static Analysis", # Name of submission profile to use + "profile_name": "Static Analysis", # Name of submission profile to use - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v4/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v4/user/submission_params// } Data Block (Binary): diff --git a/assemblyline_ui/api/v4/ui.py b/assemblyline_ui/api/v4/ui.py index ed8268c5..d4059523 100644 --- a/assemblyline_ui/api/v4/ui.py +++ b/assemblyline_ui/api/v4/ui.py @@ -16,7 +16,7 @@ from assemblyline_ui.config import TEMP_DIR, STORAGE, FILESTORE, config, CLASSIFICATION as Classification, \ IDENTIFY, metadata_validator from assemblyline_ui.helper.service import ui_to_submission_params -from assemblyline_ui.helper.submission import submission_received +from assemblyline_ui.helper.submission import submission_received, update_submission_parameters from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota from assemblyline_core.submission_client import SubmissionClient, SubmissionException @@ -259,6 +259,13 @@ def start_ui_submission(ui_sid, **kwargs): # Submit to dispatcher try: params = ui_to_submission_params(ui_params) + + # Update submission parameters as specified by the user + try: + update_submission_parameters(params, ui_params, user) + except Exception as e: + return make_api_response({}, str(e), 400) + metadata = params.pop("metadata", {}) # Enforce maximum DTL diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 4d9aa2e1..005570d5 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -13,7 +13,7 @@ from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, UI_MESSAGING, \ - VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS + VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( @@ -215,7 +215,6 @@ def who_am_i(**kwargs): "max_dtl": config.retrohunt.max_dtl, }, "submission": { - "configurable_params": USER_CONFIGURABLE_SUBMISSION_PARAMS, "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, "file_sources": file_sources, @@ -883,7 +882,6 @@ def get_user_settings(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "description": "", # Default description for this user's submissions "download_encoding": "blah", # Default encoding for downloaded files @@ -921,7 +919,6 @@ def set_user_settings(username, **kwargs): Data Block: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "default_zip_password": "zippy" # Default password used for protected file downloads "description": "", # Default description for this user's submissions @@ -982,7 +979,6 @@ def get_user_submission_params(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "description": "", # Default description for this user's submissions "priority": 1000, # Default submission priority diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index 515eee5d..bbf4c95f 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -172,8 +172,6 @@ def get_signup_queue(key): config=config, datastore=STORAGE, filestore=FILESTORE, identify=IDENTIFY) SERVICE_LIST = forge.CachedObject(STORAGE.list_all_services, kwargs=dict(as_obj=False, full=True)) SUBMISSION_PROFILES = {profile.name: profile for profile in config.submission.profiles} -USER_CONFIGURABLE_SUBMISSION_PARAMS = list(set(SubmissionParams.fields().keys()) - \ - set(SubmissionProfileParams.fields().keys()) - \ - set(['quota_item', 'type', 'groups'])) + # End global ################################################################# diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index cad5d9d3..e22a6a70 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -16,7 +16,7 @@ from assemblyline.odm.models.config import HASH_PATTERN_MAP from assemblyline.odm.messages.submission import SubmissionMessage from assemblyline.odm.models.user import ROLES -from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS +from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES # Baseline fetch methods FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url']) @@ -149,7 +149,7 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo def update_submission_parameters(s_params: dict, data: dict, user: dict): - s_profile = SUBMISSION_PROFILES.get(data.get('profile')) + s_profile = SUBMISSION_PROFILES.get(data.get('profile_name')) # Apply provided params (if the user is allowed to) if ROLES.submission_customize in user['roles']: s_params.update(data.get("params", {})) @@ -159,11 +159,12 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params.update(s_profile.params.as_primitives()) + s_fields = s_profile.params.fields() params_data = data.get("params", {}) - for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: - if param in params_data: - # Overwrite/Set parameter with user-defined input - s_params[param] = params_data[param] + for param, value in params_data.items(): + if param in s_fields and s_fields[param].default_set == True: + # Set parameter with user-defined input since it wasn't explicitly declared in the configuration + s_params[param] = value else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") From f8806b749d262351e3332c8d07ae242fcf2fd232 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:36:11 +0000 Subject: [PATCH 05/47] Update tests --- test/test_ingest.py | 2 +- test/test_submit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_ingest.py b/test/test_ingest.py index d1d58b8d..6a40f05f 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -345,7 +345,7 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile'] = "Static Analysis" + data['profile_name'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set diff --git a/test/test_submit.py b/test/test_submit.py index 9d117db2..09ffb8ba 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -301,7 +301,7 @@ def test_submit_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile'] = "Static Analysis" + data['profile_name'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set From 40a143455b9eadd3007db9f007a34ddc806b70f5 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:46:04 +0000 Subject: [PATCH 06/47] Rename parameter for clarity --- assemblyline_ui/api/v4/ingest.py | 2 +- assemblyline_ui/api/v4/submit.py | 2 +- assemblyline_ui/api/v4/user.py | 7 +++++++ assemblyline_ui/helper/submission.py | 2 +- test/test_ingest.py | 2 +- test/test_submit.py | 2 +- 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index 6eca2882..88df7f0d 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -152,7 +152,7 @@ def ingest_single_file(**kwargs): // OPTIONAL VALUES "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "profile_name": "Static Analysis", # Name of submission profile to use + "submission_profile": "Static Analysis", # Name of submission profile to use "metadata": { # Submission metadata "key": val, # Key/Value pair for metadata parameters diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 9e4441da..5e38b1e2 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -248,7 +248,7 @@ def submit(**kwargs): // OPTIONAL VALUES "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "profile_name": "Static Analysis", # Name of submission profile to use + "submission_profile": "Static Analysis", # Name of submission profile to use "metadata": { # Submission metadata "key": val, # Key/Value pair for metadata parameters diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 005570d5..30f7f9ed 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -192,6 +192,13 @@ def who_am_i(**kwargs): [file_sources["sha256"]["sources"].append(x.name) for x in config.submission.sha256_sources if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] + submission_profiles = {} + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) + for p_cls in profile.params.fields().values() if p_cls.default_set == False} + user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index e22a6a70..5d8a5ebf 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -149,7 +149,7 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo def update_submission_parameters(s_params: dict, data: dict, user: dict): - s_profile = SUBMISSION_PROFILES.get(data.get('profile_name')) + s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) # Apply provided params (if the user is allowed to) if ROLES.submission_customize in user['roles']: s_params.update(data.get("params", {})) diff --git a/test/test_ingest.py b/test/test_ingest.py index 6a40f05f..f8118c91 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -345,7 +345,7 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile_name'] = "Static Analysis" + data['submission_profile'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set diff --git a/test/test_submit.py b/test/test_submit.py index 09ffb8ba..3818286a 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -301,7 +301,7 @@ def test_submit_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile_name'] = "Static Analysis" + data['submission_profile'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set From c8c8fc7279611a807c925a0cd6f9c95b960b21d4 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:45:45 +0000 Subject: [PATCH 07/47] Allow users to set parameters that aren't enforced by profile --- assemblyline_ui/api/v4/user.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 30f7f9ed..44c89b24 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -7,7 +7,7 @@ from assemblyline.common.security import (check_password_requirements, get_password_hash, get_password_requirement_message) from assemblyline.datastore.exceptions import SearchException -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.models.user import (ACL_MAP, ROLES, USER_ROLES, USER_TYPE_DEP, USER_TYPES, User, load_roles, load_roles_form_acls) from assemblyline.odm.models.user_favorites import Favorite @@ -193,11 +193,16 @@ def who_am_i(**kwargs): if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] submission_profiles = {} - for name, profile in SUBMISSION_PROFILES.items(): - if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): - # We want to pass forward the configurations that have been explicitly set as a configuration - submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) - for p_cls in profile.params.fields().values() if p_cls.default_set == False} + if config.submission.profiles == DEFAULT_SUBMISSION_PROFILES: + # If these are exactly the same as the default values, then it's accessible to everyone + submission_profiles = {profile['name']: profile['params'] for profile in DEFAULT_SUBMISSION_PROFILES} + else: + # Filter profiles based on accessibility to the user + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) + for p_cls in profile.params.fields().values() if p_cls.default_set == False} user_data['configuration'] = { "auth": { @@ -226,8 +231,7 @@ def who_am_i(**kwargs): "max_dtl": config.submission.max_dtl, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, - "profiles": {name: profile.params.as_primitives() for name, profile in SUBMISSION_PROFILES.items() \ - if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification)}, + "profiles": submission_profiles, "verdicts": { "info": config.submission.verdicts.info, "suspicious": config.submission.verdicts.suspicious, From f718e55e0e6a0da296c15bafdea75436a772e809 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:29:31 +0000 Subject: [PATCH 08/47] Expand service categories to make it easier for the UI to lock down configurations --- assemblyline_ui/api/v4/user.py | 16 ++++++++++++++-- assemblyline_ui/helper/submission.py | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 44c89b24..c500cbb5 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -1,3 +1,4 @@ +from copy import deepcopy from typing import List from assemblyline.odm.models.config import ExternalLinks from flask import request, session as flsk_session @@ -21,7 +22,7 @@ API_PRIV_MAP) from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException -from .federated_lookup import filtered_tag_names +from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names SUB_API = 'user' @@ -192,10 +193,11 @@ def who_am_i(**kwargs): [file_sources["sha256"]["sources"].append(x.name) for x in config.submission.sha256_sources if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] + # Prepare submission profile configurations for UI submission_profiles = {} if config.submission.profiles == DEFAULT_SUBMISSION_PROFILES: # If these are exactly the same as the default values, then it's accessible to everyone - submission_profiles = {profile['name']: profile['params'] for profile in DEFAULT_SUBMISSION_PROFILES} + submission_profiles = {profile['name']: deepcopy(profile['params']) for profile in DEFAULT_SUBMISSION_PROFILES} else: # Filter profiles based on accessibility to the user for name, profile in SUBMISSION_PROFILES.items(): @@ -204,6 +206,16 @@ def who_am_i(**kwargs): submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) for p_cls in profile.params.fields().values() if p_cls.default_set == False} + # Expand service categories if used in submission profiles (assists with the UI locking down service selection) + service_categories = list(STORAGE.service.facet('category').keys()) + for profile in submission_profiles.values(): + for key, services in profile.get("services", {}).items(): + expanded_services = list() + for srv in services: + if srv in service_categories: + expanded_services.extend([i['name'] for i in STORAGE.service.search(f"category:{srv}", as_obj=False, fl="name")['items']]) + profile['services'][key] = list(set(services).union(set(expanded_services))) + user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 5d8a5ebf..70b083f0 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -9,6 +9,7 @@ from typing import List from urllib.parse import urlparse +from assemblyline.common.dict_utils import recursive_update from assemblyline.common.file import make_uri_file from assemblyline.common.isotime import now_as_iso from assemblyline.common.str_utils import safe_str @@ -158,7 +159,7 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): # User isn't allowed to use the submission profile specified raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) - s_params.update(s_profile.params.as_primitives()) + s_params = recursive_update(s_params, s_profile.params.as_primitives()) s_fields = s_profile.params.fields() params_data = data.get("params", {}) for param, value in params_data.items(): From ea02e03b69bef2f1de46a30c3c57f8b38aedc80c Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 9 Aug 2024 19:26:08 +0000 Subject: [PATCH 09/47] Patch testing --- assemblyline_ui/api/v4/user.py | 14 ++++---------- assemblyline_ui/helper/submission.py | 9 ++------- test/test_ingest.py | 7 +++++-- test/test_submit.py | 10 +++++++--- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index c500cbb5..507c59ce 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -195,16 +195,10 @@ def who_am_i(**kwargs): # Prepare submission profile configurations for UI submission_profiles = {} - if config.submission.profiles == DEFAULT_SUBMISSION_PROFILES: - # If these are exactly the same as the default values, then it's accessible to everyone - submission_profiles = {profile['name']: deepcopy(profile['params']) for profile in DEFAULT_SUBMISSION_PROFILES} - else: - # Filter profiles based on accessibility to the user - for name, profile in SUBMISSION_PROFILES.items(): - if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): - # We want to pass forward the configurations that have been explicitly set as a configuration - submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) - for p_cls in profile.params.fields().values() if p_cls.default_set == False} + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = profile.params.as_primitives(strip_null=True) # Expand service categories if used in submission profiles (assists with the UI locking down service selection) service_categories = list(STORAGE.service.facet('category').keys()) diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 70b083f0..15191922 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -159,13 +159,8 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): # User isn't allowed to use the submission profile specified raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) - s_params = recursive_update(s_params, s_profile.params.as_primitives()) - s_fields = s_profile.params.fields() - params_data = data.get("params", {}) - for param, value in params_data.items(): - if param in s_fields and s_fields[param].default_set == True: - # Set parameter with user-defined input since it wasn't explicitly declared in the configuration - s_params[param] = value + s_params = recursive_update(s_params, data.get("params", {})) + s_params = recursive_update(s_params, s_profile.params.as_primitives(strip_null=True)) else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") diff --git a/test/test_ingest.py b/test/test_ingest.py index f8118c91..3ecd35bc 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -10,7 +10,7 @@ from assemblyline.common import forge from assemblyline.odm.messages.submission import Submission -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.models.file import File from assemblyline.odm.randomizer import random_model_obj, get_random_phrase from assemblyline.odm.random_data import create_users, wipe_users, create_services, wipe_services @@ -345,11 +345,14 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['submission_profile'] = "Static Analysis" + profile = DEFAULT_SUBMISSION_PROFILES[0] + data['submission_profile'] = profile["name"] get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set # The system should silently ignore your parameter and still create a submission + data['params'] = {'services': {'selected': ['blah']}} + # But also try setting a parameter that you are allowed to set data['params'] = {'deep_scan': True} get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) diff --git a/test/test_submit.py b/test/test_submit.py index 3818286a..47c71c54 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -9,7 +9,7 @@ from conftest import get_api_data, APIError from assemblyline.common import forge -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions from assemblyline.odm.randomizer import get_random_phrase from assemblyline.remote.datatypes.queues.named import NamedQueue @@ -301,14 +301,18 @@ def test_submit_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['submission_profile'] = "Static Analysis" + profile = DEFAULT_SUBMISSION_PROFILES[0] + data['submission_profile'] = profile['name'] get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set # The system should silently ignore your parameter and still create a submission + data['params'] = {'services': {'selected': ['blah']}} + # But also try setting a parameter that you are allowed to set data['params'] = {'deep_scan': True} resp = get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) - assert resp['params']['deep_scan'] == False + assert resp['params']['services']['selected'] == profile['params']['services']['selected'] + assert resp['params']['deep_scan'] == True # Restore original roles for later tests datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), From 091ac3798617e5481a20f92fa3fe47b9e9d35a35 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:35:33 +0000 Subject: [PATCH 10/47] Modify APIs to allow editing/fetching of user submission profiles --- assemblyline_ui/api/v4/user.py | 1 + assemblyline_ui/config.py | 3 +-- assemblyline_ui/helper/service.py | 37 ++++++++++++++++++++++++++++++- assemblyline_ui/helper/user.py | 14 +++++++++++- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 507c59ce..e850ca73 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -199,6 +199,7 @@ def who_am_i(**kwargs): if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): # We want to pass forward the configurations that have been explicitly set as a configuration submission_profiles[name] = profile.params.as_primitives(strip_null=True) + submission_profiles[name]["editable_params"] = profile.editable_params # Expand service categories if used in submission profiles (assists with the UI locking down service selection) service_categories = list(STORAGE.service.facet('category').keys()) diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index bbf4c95f..3164dfd6 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -9,8 +9,7 @@ from assemblyline.common.version import BUILD_MINOR, FRAMEWORK_VERSION, SYSTEM_VERSION from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator from assemblyline.filestore import FileStore -from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP, SubmissionProfileParams -from assemblyline.odm.models.submission import SubmissionParams +from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP from assemblyline.remote.datatypes import get_client from assemblyline.remote.datatypes.cache import Cache from assemblyline.remote.datatypes.daily_quota_tracker import DailyQuotaTracker diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 249bf859..9116dcb7 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -1,6 +1,7 @@ from copy import copy from typing import Any, Optional -from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST +from assemblyline.common.dict_utils import recursive_update +from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST, SUBMISSION_PROFILES, STORAGE from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, SubmissionParams from assemblyline.odm.models.user_settings import UserSettings @@ -8,6 +9,40 @@ USER_SETTINGS_FIELDS = list(UserSettings.fields().keys()) SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) +def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): + out = {} + for profile in SUBMISSION_PROFILES.values(): + if CLASSIFICATION.is_accessible(classification, profile.classification): + user_default_profile = user_default_values.get(profile.name, {}) + + params = copy(profile.params.as_primitives(strip_null=True)) + out[profile.name] = recursive_update(params, user_default_values.get(profile.name, {})) + + service_spec = [] + # If there are any editable service parameters that haven't been set, then assign their default values + for service, editable_params in profile.editable_params.items(): + service_obj = STORAGE.get_service_with_delta(service, as_obj=False) + if not service_obj: + continue + param_object = {'name': service, "params": []} + profile_service_spec = user_default_profile.get('service_spec', {}).get(service, {}) + for param in service_obj['submission_params']: + if param['name'] not in editable_params: + # Service parameter isn't allowed to be overridden in this profile + continue + + new_param = copy(param) + if profile_service_spec.get(param['name']): + # Overwrite with user-specific value for profile + new_param['value'] = profile_service_spec[param['name']] + new_param["hide"] = False + param_object["params"].append(new_param) + service_spec.append(param_object) + + # Overwrite 'service_spec' value to be compliant with frontend rendering + out[profile.name]['service_spec'] = service_spec + return out + def get_default_service_spec(srv_list=None, user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): if not srv_list: srv_list = SERVICE_LIST diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index ed433ef2..4d511383 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -7,7 +7,7 @@ from assemblyline.odm.models.user_settings import UserSettings from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, SUBMISSION_TRACKER, \ config, CLASSIFICATION as Classification, SERVICE_LIST -from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services +from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services, get_default_submission_profiles from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException, AuthenticationException ACCOUNT_USER_MODIFIABLE = ["name", "avatar", "password"] @@ -252,6 +252,10 @@ def load_user_settings(user): # Only display services that a user is allowed to see settings['service_spec'] = get_default_service_spec(srv_list, settings.get('service_spec', {}), user_classfication) settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) + settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], + user_classfication) + settings['preferred_submission_profile'] = user.get('preferred_submission_profile') or \ + list(settings['submission_profiles'].keys())[0] settings['default_zip_password'] = settings.get('default_zip_password', None) # Normalize the user's classification @@ -263,4 +267,12 @@ def load_user_settings(user): def save_user_settings(username, data): data["services"] = {'selected': simplify_services(data["services"])} + # Ensure submission profile changes are valid + for profile_name, profile_spec in data["submission_profiles"].items(): + saved_spec = {} + for spec in profile_spec["service_spec"]: + saved_spec[spec["name"]] = {param['name']: param['value'] for param in spec["params"] if param['name'] in config.submission.profiles[profile_name].editable_params} + data["submission_profiles"][profile_name]["service_spec"] = saved_spec + + return STORAGE.user_settings.save(username, data) From b117ae316795959f33074e343eaf591a30f9152b Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 01:30:27 +0000 Subject: [PATCH 11/47] Changed the submission profile's loading and setting methods --- assemblyline_ui/api/v4/user.py | 73 +++++++++++++++++++------- assemblyline_ui/helper/service.py | 36 +++---------- assemblyline_ui/helper/user.py | 85 +++++++++++++++++++++++++------ 3 files changed, 132 insertions(+), 62 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index e7e9e58c..56a50539 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -1,29 +1,55 @@ from copy import deepcopy from typing import List -from assemblyline.odm.models.config import ExternalLinks -from flask import request, session as flsk_session from assemblyline.common.comms import send_activated_email, send_authorize_email from assemblyline.common.isotime import now_as_iso -from assemblyline.common.security import (check_password_requirements, get_password_hash, - get_password_requirement_message) +from assemblyline.common.security import ( + check_password_requirements, + get_password_hash, + get_password_requirement_message, +) from assemblyline.datastore.exceptions import SearchException -from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES -from assemblyline.odm.models.user import (ACL_MAP, ROLES, USER_ROLES, USER_TYPE_DEP, USER_TYPES, User, load_roles, - load_roles_form_acls) +from assemblyline.odm.models.config import HASH_PATTERN_MAP, ExternalLinks +from assemblyline.odm.models.user import ( + ACL_MAP, + ROLES, + USER_ROLES, + USER_TYPE_DEP, + USER_TYPES, + User, + load_roles, + load_roles_form_acls, +) from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint -from assemblyline_ui.config import AI_AGENT, APPS_LIST, CLASSIFICATION, CLASSIFICATION_ALIASES, \ - DAILY_QUOTA_TRACKER, LOGGER, STORAGE,SUBMISSION_PROFILES, UI_MESSAGING,UI_METADATA_VALIDATION, VERSION, config +from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names +from assemblyline_ui.config import ( + AI_AGENT, + APPS_LIST, + CLASSIFICATION, + CLASSIFICATION_ALIASES, + DAILY_QUOTA_TRACKER, + LOGGER, + STORAGE, + SUBMISSION_PROFILES, + UI_MESSAGING, + UI_METADATA_VALIDATION, + VERSION, + config, +) from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( - get_default_user_quotas, get_dynamic_classification, load_user_settings, save_user_account, save_user_settings, - API_PRIV_MAP) + API_PRIV_MAP, + get_default_user_quotas, + get_dynamic_classification, + load_user_settings, + save_user_account, + save_user_settings, +) from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException - -from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names - +from flask import request +from flask import session as flsk_session SUB_API = 'user' user_api = make_subapi_blueprint(SUB_API, api_version=4) @@ -211,7 +237,8 @@ def who_am_i(**kwargs): expanded_services = list() for srv in services: if srv in service_categories: - expanded_services.extend([i['name'] for i in STORAGE.service.search(f"category:{srv}", as_obj=False, fl="name")['items']]) + expanded_services.extend([i['name'] for i in STORAGE.service.search( + f"category:{srv}", as_obj=False, fl="name")['items']]) profile['services'][key] = list(set(services).union(set(expanded_services))) user_data['configuration'] = { @@ -963,8 +990,6 @@ def set_user_settings(username, **kwargs): } """ user = kwargs['user'] - if username != user['uname'] and ROLES.administration not in user['roles']: - raise AccessDeniedException("You are not allowed to set settings for another user then yourself.") try: data = request.json @@ -972,7 +997,19 @@ def set_user_settings(username, **kwargs): if not data.get('default_zip_password', ''): return make_api_response({"success": False}, "Encryption password can't be empty.", 403) - if save_user_settings(username, data): + # Changing your own settings + if username == user['uname']: + if ROLES.administration not in user['roles'] or ROLES.self_manage not in user['roles']: + raise AccessDeniedException("You are not allowed to modify your own settings.") + edit_user = user + + # Changing someone else's settings + else: + if ROLES.administration not in user['roles']: + raise AccessDeniedException("You are not allowed to set settings for another user then yourself.") + edit_user = STORAGE.user.get(username, as_obj=False) + + if save_user_settings(edit_user, data): return make_api_response({"success": True}) else: return make_api_response({"success": False}, "Failed to save user's settings", 500) diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 9116dcb7..5a1a279a 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -9,40 +9,20 @@ USER_SETTINGS_FIELDS = list(UserSettings.fields().keys()) SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) + def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): out = {} for profile in SUBMISSION_PROFILES.values(): if CLASSIFICATION.is_accessible(classification, profile.classification): - user_default_profile = user_default_values.get(profile.name, {}) - - params = copy(profile.params.as_primitives(strip_null=True)) - out[profile.name] = recursive_update(params, user_default_values.get(profile.name, {})) - - service_spec = [] - # If there are any editable service parameters that haven't been set, then assign their default values - for service, editable_params in profile.editable_params.items(): - service_obj = STORAGE.get_service_with_delta(service, as_obj=False) - if not service_obj: - continue - param_object = {'name': service, "params": []} - profile_service_spec = user_default_profile.get('service_spec', {}).get(service, {}) - for param in service_obj['submission_params']: - if param['name'] not in editable_params: - # Service parameter isn't allowed to be overridden in this profile - continue - - new_param = copy(param) - if profile_service_spec.get(param['name']): - # Overwrite with user-specific value for profile - new_param['value'] = profile_service_spec[param['name']] - new_param["hide"] = False - param_object["params"].append(new_param) - service_spec.append(param_object) - - # Overwrite 'service_spec' value to be compliant with frontend rendering - out[profile.name]['service_spec'] = service_spec + profile_values = copy(profile.as_primitives(strip_null=True)) + out[profile.name] = recursive_update(profile_values, user_default_values.get(profile.name, {})) + + out[profile.name].pop("name") + out[profile.name].pop("classification") + return out + def get_default_service_spec(srv_list=None, user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): if not srv_list: srv_list = SERVICE_LIST diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 651bb19c..8462723d 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,14 +1,30 @@ from typing import Optional -from flask import session as flsk_session - from assemblyline.common.str_utils import safe_str -from assemblyline.odm.models.user import User, load_roles, ROLES +from assemblyline.odm.models.config import SubmissionProfileParams +from assemblyline.odm.models.user import ROLES, User, load_roles from assemblyline.odm.models.user_settings import UserSettings -from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, SUBMISSION_TRACKER, \ - config, CLASSIFICATION as Classification, SERVICE_LIST, DOWNLOAD_ENCODING, DEFAULT_ZIP_PASSWORD -from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services, get_default_submission_profiles -from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException, AuthenticationException +from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER +from assemblyline_ui.config import CLASSIFICATION as Classification +from assemblyline_ui.config import ( + DAILY_QUOTA_TRACKER, + DEFAULT_ZIP_PASSWORD, + DOWNLOAD_ENCODING, + LOGGER, + SERVICE_LIST, + STORAGE, + SUBMISSION_PROFILES, + SUBMISSION_TRACKER, + config, +) +from assemblyline_ui.helper.service import ( + get_default_service_list, + get_default_service_spec, + get_default_submission_profiles, + simplify_services, +) +from assemblyline_ui.http_exceptions import AccessDeniedException, AuthenticationException, InvalidDataException +from flask import session as flsk_session ACCOUNT_USER_MODIFIABLE = ["name", "avatar", "password"] @@ -302,15 +318,52 @@ def load_user_settings(user): return settings -def save_user_settings(username, data): - data["services"] = {'selected': simplify_services(data["services"])} +def save_user_settings(user, data): + username = user.get('uname', None) + if username == None: + raise Exception("Invalid username") + + out = STORAGE.user_settings.get(username).as_primitives() + for key in out.keys(): + if key in data and key not in ["services", "service_spec", "submission_profiles"]: + out[key] = data.get(key, None) + + out["services"] = {'selected': simplify_services(data["services"])} + + classification = user.get("classification", None) + submission_customize = ROLES.submission_customize in user['roles'] + srv_list = [x['name'] for x in SERVICE_LIST if x['enabled']] + srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] + srv_list = list(set(srv_list)) + + submission_profiles = {} + for name, profile in SUBMISSION_PROFILES.items(): + if Classification.is_accessible(classification, profile.classification): + user_params = data.get('submission_profiles', {}).get(profile.name, {}).get('params', {}) + + # Applying the submission params + profile_defaults = SubmissionProfileParams().as_primitives() + default_keys = profile["editable_params"].get("submit", []) + for key in profile_defaults.keys(): + if key in user_params and key not in ["services", "service_spec"] and \ + (submission_customize or key in default_keys): + profile["params"][key] = user_params.get(key, None) + + # Applying the selected services + if submission_customize: + profile["params"]["services"]["selected"] = [x for x in user_params.get( + 'services', {}).get('selected', []) if x in srv_list] + + # Applying the service specs + profile["params"]["service_spec"] = {} + for svr_name, spec in user_params.get("service_spec", {}).items(): + for p_name, p_value in spec.items(): + if (p_name in profile["editable_params"].get(svr_name, []) or submission_customize) and \ + svr_name in srv_list: + profile["params"]["service_spec"].setdefault(svr_name, {}).setdefault(p_name, p_value) - # Ensure submission profile changes are valid - for profile_name, profile_spec in data["submission_profiles"].items(): - saved_spec = {} - for spec in profile_spec["service_spec"]: - saved_spec[spec["name"]] = {param['name']: param['value'] for param in spec["params"] if param['name'] in config.submission.profiles[profile_name].editable_params} - data["submission_profiles"][profile_name]["service_spec"] = saved_spec + submission_profiles[name] = profile.params.as_primitives(strip_null=True) + out["submission_profiles"] = submission_profiles - return STORAGE.user_settings.save(username, data) + return STORAGE.user_settings.save(username, out) From 08d9121e2774d230c2c5154a068ce4797eecdb2c Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 02:10:26 +0000 Subject: [PATCH 12/47] Set the preferred_submission_profile if it doesn't exist in the existing settings --- assemblyline_ui/helper/user.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 8462723d..c05a9682 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -336,6 +336,11 @@ def save_user_settings(user, data): srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] srv_list = list(set(srv_list)) + if data.get('preferred_submission_profile', None) not in SUBMISSION_PROFILES.keys(): + out['preferred_submission_profile'] = SUBMISSION_PROFILES.keys()[0] + else: + out['preferred_submission_profile'] = data['preferred_submission_profile'] + submission_profiles = {} for name, profile in SUBMISSION_PROFILES.items(): if Classification.is_accessible(classification, profile.classification): From 7d13a5b41e0ef714805ebc9e8556611269693590 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 02:26:44 +0000 Subject: [PATCH 13/47] Fixed the loading of the preferred_submission_profile --- assemblyline_ui/helper/user.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index c05a9682..87b30b87 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -308,13 +308,14 @@ def load_user_settings(user): settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], user_classfication) - settings['preferred_submission_profile'] = user.get('preferred_submission_profile') or \ - list(settings['submission_profiles'].keys())[0] settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) # Normalize the user's classification settings['classification'] = Classification.normalize_classification(settings['classification']) + if settings.get('preferred_submission_profile', None) not in list(settings['submission_profiles'].keys()): + settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] + return settings From 8a3486bdbf1e935d891abfb74ee12b75ad88a663 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Fri, 3 Jan 2025 11:49:41 +0000 Subject: [PATCH 14/47] Added the max file size to the /whoami path --- assemblyline_ui/api/v4/user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 56a50539..98cdfe1f 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -121,6 +121,7 @@ def who_am_i(**kwargs): "configurable_params": [], # Submission parameters that are configurable when using profiles "dtl": 10, # Default number of days submission stay in the system "max_dtl": 30, # Maximum number of days submission stay in the system + "max_file_size": 104857600, # Maximum size for files submitted in the system "file_sources": [], # List of file sources to perform remote submission into the system "metadata": {}, # Metadata compliance policy to submit to the system "profiles": {}, # Submission profiles @@ -266,6 +267,7 @@ def who_am_i(**kwargs): "submission": { "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, + "max_file_size": config.submission.max_file_size, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, "profiles": submission_profiles, From 716809d1732ead9d1b8ec1e679b5537706171c1d Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Fri, 17 Jan 2025 18:27:32 +0000 Subject: [PATCH 15/47] minor change to the load_user_settings --- assemblyline_ui/helper/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 87b30b87..41b67bb2 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -285,7 +285,7 @@ def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - settings = STORAGE.user_settings.get_if_exists(user['uname'], as_obj=False) + settings = STORAGE.user_settings.get_if_exists(user['uname']).as_primitives(strip_null=True) srv_list = [x for x in SERVICE_LIST if x['enabled']] if not settings: def_srv_list = None From a3bb2545c72049a5f4bffc4085ce16b0217ea0be Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 23 Jan 2025 18:19:52 +0000 Subject: [PATCH 16/47] Bugfix: New users should use default settings --- assemblyline_ui/helper/user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 41b67bb2..727c5e08 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -285,12 +285,13 @@ def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - settings = STORAGE.user_settings.get_if_exists(user['uname']).as_primitives(strip_null=True) + settings = STORAGE.user_settings.get_if_exists(user['uname']) srv_list = [x for x in SERVICE_LIST if x['enabled']] if not settings: def_srv_list = None settings = default_settings else: + settings = settings.as_primitives(strip_null=True) # Make sure all defaults are there for key, item in default_settings.items(): if key not in settings: From 876500c08fe39746ff209b4dd94da2ec181ecf95 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 1 Feb 2025 07:50:38 +0000 Subject: [PATCH 17/47] Update APIs to handle changes to submission profiles --- assemblyline_ui/api/v4/user.py | 31 ++++---- assemblyline_ui/helper/service.py | 12 ++-- assemblyline_ui/helper/submission.py | 30 ++++++-- assemblyline_ui/helper/user.py | 103 +++++++++++++++------------ 4 files changed, 103 insertions(+), 73 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 98cdfe1f..1381a352 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -228,8 +228,12 @@ def who_am_i(**kwargs): for name, profile in SUBMISSION_PROFILES.items(): if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): # We want to pass forward the configurations that have been explicitly set as a configuration - submission_profiles[name] = profile.params.as_primitives(strip_null=True) - submission_profiles[name]["editable_params"] = profile.editable_params + submission_profiles[name] = profile.as_primitives(strip_null=True) + + # Remove unwanted/redundant data from being shared + for field in ['name', 'classification']: + submission_profiles[name].pop(field, None) + # Expand service categories if used in submission profiles (assists with the UI locking down service selection) service_categories = list(STORAGE.service.facet('category').keys()) @@ -933,19 +937,14 @@ def get_user_settings(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled - "classification": "", # Default classification for this user sumbissions - "description": "", # Default description for this user's submissions - "download_encoding": "blah", # Default encoding for downloaded files - "default_zip_password": "pass", # Default password for password protected ZIP - "expand_min_score": 100, # Default minimum score to auto-expand sections - "priority": 1000, # Default submission priority - "service_spec": [], # Default Service specific parameters - "ignore_cache": true, # Should file be reprocessed even if there are cached results - "groups": [ ... ], # Default groups selection for the user scans - "ttl": 30, # Default time to live in days of the users submissions - "services": [ ... ], # Default list of selected services - "ignore_filtering": false # Should filtering services by ignored? + "default_external_sources": [], # Default file sources for this user + "default_zip_password": "infected", # Default password for password protected ZIP + "download_encoding": "blah", # Default encoding for downloaded files, + "executive_summary": false, # Should the executive summary be shown by default + "expand_min_score": 100, # Default minimum score to auto-expand sections + "preferred_submission_profile": "default", # Default submission profile + "submission_profiles": [], # List of submission profiles + "submission_view": "report", # Default submission view } """ user = kwargs['user'] @@ -1001,7 +1000,7 @@ def set_user_settings(username, **kwargs): # Changing your own settings if username == user['uname']: - if ROLES.administration not in user['roles'] or ROLES.self_manage not in user['roles']: + if ROLES.administration not in user['roles'] and ROLES.self_manage not in user['roles']: raise AccessDeniedException("You are not allowed to modify your own settings.") edit_user = user diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 5a1a279a..ab269a9a 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -10,16 +10,16 @@ SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) -def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): +def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED, + include_default=False): out = {} + if include_default: + out['default'] = user_default_values.get('default', {}) + for profile in SUBMISSION_PROFILES.values(): if CLASSIFICATION.is_accessible(classification, profile.classification): - profile_values = copy(profile.as_primitives(strip_null=True)) + profile_values = copy(profile.params.as_primitives(strip_null=True)) out[profile.name] = recursive_update(profile_values, user_default_values.get(profile.name, {})) - - out[profile.name].pop("name") - out[profile.name].pop("classification") - return out diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 15191922..f3f3b420 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -14,8 +14,9 @@ from assemblyline.common.isotime import now_as_iso from assemblyline.common.str_utils import safe_str from assemblyline.common.iprange import is_ip_reserved -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, SubmissionProfile from assemblyline.odm.messages.submission import SubmissionMessage + from assemblyline.odm.models.user import ROLES from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES @@ -45,6 +46,26 @@ class InvalidUrlException(Exception): class ForbiddenLocation(Exception): pass +def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submission_customize=False) -> dict: + validated_profile = profile.params.as_primitives(strip_null=True) + + for param_type, list_of_params in profile.editable_params.items(): + if param_type == "submission": + # Submission-level parameters + for p in list(updates.keys()): + if p not in ['services', 'service_spec'] and \ + (p not in list_of_params and not submission_customize): + # Submission parameter isn't allowed to be modified based on profile configuration + updates.pop(p) + else: + # Service-level parameters + service_spec = updates.get('service_spec', {}).get(param_type, {}) + for key in list(service_spec.keys()): + if key not in list_of_params and not submission_customize: + # Service parameter isn't allowed to be changed + service_spec.pop(key) + + return recursive_update(validated_profile, updates) def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: dict, out_file: str, default_external_sources: List[str]): @@ -151,8 +172,9 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di def update_submission_parameters(s_params: dict, data: dict, user: dict): s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) + submission_customize = ROLES.submission_customize in user['roles'] # Apply provided params (if the user is allowed to) - if ROLES.submission_customize in user['roles']: + if submission_customize: s_params.update(data.get("params", {})) elif s_profile: if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification): @@ -160,14 +182,12 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params = recursive_update(s_params, data.get("params", {})) - s_params = recursive_update(s_params, s_profile.params.as_primitives(strip_null=True)) + s_params = apply_changes_to_profile(s_params, s_profile, submission_customize) else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") - - def refang_url(url): ''' Refangs a url of text. Based on source of: https://pypi.org/project/defang/ diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 727c5e08..c5877b6b 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,9 +1,9 @@ from typing import Optional from assemblyline.common.str_utils import safe_str -from assemblyline.odm.models.config import SubmissionProfileParams +from assemblyline.odm.models.config import SubmissionProfileParams, SubmissionProfile from assemblyline.odm.models.user import ROLES, User, load_roles -from assemblyline.odm.models.user_settings import UserSettings +from assemblyline.odm.models.user_settings import UserSettings, DEFAULT_USER_PROFILE_SETTINGS from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER from assemblyline_ui.config import CLASSIFICATION as Classification from assemblyline_ui.config import ( @@ -23,6 +23,7 @@ get_default_submission_profiles, simplify_services, ) +from assemblyline_ui.helper.submission import apply_changes_to_profile from assemblyline_ui.http_exceptions import AccessDeniedException, AuthenticationException, InvalidDataException from flask import session as flsk_session @@ -274,19 +275,20 @@ def get_dynamic_classification(current_c12n, user_info): return new_c12n -def get_default_user_settings(user): - return UserSettings({"classification": Classification.default_user_classification(user), - "ttl": config.submission.dtl, - "default_zip_password": DEFAULT_ZIP_PASSWORD, - "download_encoding": DOWNLOAD_ENCODING}).as_primitives() +def get_default_user_settings(user: dict) -> dict: + settings = DEFAULT_USER_PROFILE_SETTINGS + settings.update({"classification": Classification.default_user_classification(user), + "ttl": config.submission.dtl, + "default_zip_password": DEFAULT_ZIP_PASSWORD, + "download_encoding": DOWNLOAD_ENCODING}) + return UserSettings(settings).as_primitives() def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - + submission_customize = ROLES.submission_customize in user['roles'] settings = STORAGE.user_settings.get_if_exists(user['uname']) - srv_list = [x for x in SERVICE_LIST if x['enabled']] if not settings: def_srv_list = None settings = default_settings @@ -304,18 +306,31 @@ def load_user_settings(user): def_srv_list = settings.get('services', {}).get('selected', None) + srv_list = [x for x in SERVICE_LIST if x['enabled']] + settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) + # Normalize the user's classification + settings['classification'] = Classification.normalize_classification(settings['classification']) + + # Check if the user has instantiated their default submission profile + if submission_customize and not settings['submission_profiles'].get('default'): + settings['submission_profiles']['default'] = SubmissionProfileParams({key: value for key, value in settings.items() if key in SubmissionProfileParams.fields()}).as_primitives() + # Only display services that a user is allowed to see settings['service_spec'] = get_default_service_spec(srv_list, settings.get('service_spec', {}), user_classfication) settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], - user_classfication) - settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) + user_classfication, include_default=submission_customize) - # Normalize the user's classification - settings['classification'] = Classification.normalize_classification(settings['classification']) - if settings.get('preferred_submission_profile', None) not in list(settings['submission_profiles'].keys()): - settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] + # Check if the user has a preferred submission profile + if not settings.get('preferred_submission_profile'): + # No preferred submission profile, set one based on the user's roles + if submission_customize: + # User can customize their submission, set the preferred profile to the legacy default + settings['preferred_submission_profile'] = 'default' + else: + # User cannot customize their submission, set the preferred profile to first one on the list + settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] return settings @@ -330,7 +345,7 @@ def save_user_settings(user, data): if key in data and key not in ["services", "service_spec", "submission_profiles"]: out[key] = data.get(key, None) - out["services"] = {'selected': simplify_services(data["services"])} + out["services"] = {'selected': simplify_services(data.get("services", []))} classification = user.get("classification", None) submission_customize = ROLES.submission_customize in user['roles'] @@ -338,38 +353,34 @@ def save_user_settings(user, data): srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] srv_list = list(set(srv_list)) - if data.get('preferred_submission_profile', None) not in SUBMISSION_PROFILES.keys(): - out['preferred_submission_profile'] = SUBMISSION_PROFILES.keys()[0] - else: - out['preferred_submission_profile'] = data['preferred_submission_profile'] + accessible_profiles = [name for name, profile in SUBMISSION_PROFILES.items() \ + if Classification.is_accessible(classification, profile.classification)] + + # Check submission profile preference selection + preferred_submission_profile = data.get('preferred_submission_profile', None) + if submission_customize: + # User is allowed to customize their own default profile + accessible_profiles += ['default'] + + if preferred_submission_profile in accessible_profiles: + out['preferred_submission_profile'] = preferred_submission_profile submission_profiles = {} - for name, profile in SUBMISSION_PROFILES.items(): - if Classification.is_accessible(classification, profile.classification): - user_params = data.get('submission_profiles', {}).get(profile.name, {}).get('params', {}) - - # Applying the submission params - profile_defaults = SubmissionProfileParams().as_primitives() - default_keys = profile["editable_params"].get("submit", []) - for key in profile_defaults.keys(): - if key in user_params and key not in ["services", "service_spec"] and \ - (submission_customize or key in default_keys): - profile["params"][key] = user_params.get(key, None) - - # Applying the selected services - if submission_customize: - profile["params"]["services"]["selected"] = [x for x in user_params.get( - 'services', {}).get('selected', []) if x in srv_list] - - # Applying the service specs - profile["params"]["service_spec"] = {} - for svr_name, spec in user_params.get("service_spec", {}).items(): - for p_name, p_value in spec.items(): - if (p_name in profile["editable_params"].get(svr_name, []) or submission_customize) and \ - svr_name in srv_list: - profile["params"]["service_spec"].setdefault(svr_name, {}).setdefault(p_name, p_value) - - submission_profiles[name] = profile.params.as_primitives(strip_null=True) + for name in accessible_profiles: + user_params = data.get('submission_profiles', {}).get(name, {}) + profile_config: Optional[SubmissionProfile] = SUBMISSION_PROFILES.get(name) + + if profile_config == None: + if name == "default": + # There is no restriction on what you can set for your default submission profile + # Set profile based on preferences set at the root-level + data["services"] = out['services'] + submission_profiles[name] = SubmissionProfileParams({key: value for key, value in data.items() + if key in SubmissionProfileParams.fields()}).as_primitives() + + else: + # Apply changes to the profile relative to what's allowed to be changed based on configuration + submission_profiles[name] = apply_changes_to_profile(profile_config, user_params, submission_customize) out["submission_profiles"] = submission_profiles From 3ef5eaf5c7572f41ef84949aef158ee23b8148d1 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:44:56 +0000 Subject: [PATCH 18/47] Guard submit & ingest API from unauthorized submission customization --- assemblyline_ui/api/v4/ingest.py | 5 +++-- assemblyline_ui/api/v4/submit.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index ff611f18..4b914fdd 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -270,8 +270,9 @@ def ingest_single_file(**kwargs): "type": "INGEST" }) - # Apply provided params - s_params.update(data.get("params", {})) + # Apply provided params (if the user is allowed to) + if ROLES.submission_customize in user['roles']: + s_params.update(data.get("params", {})) # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 43149454..643231c0 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -339,7 +339,11 @@ def submit(**kwargs): # Create task object s_params = ui_to_submission_params(user_settings) - s_params.update(data.get("params", {})) + + # Apply provided params (if the user is allowed to) + if ROLES.submission_customize in user['roles']: + s_params.update(data.get("params", {})) + default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources if 'groups' not in s_params: s_params['groups'] = [g for g in user['groups'] if g in s_params['classification']] From f2769f0656b07566430e487021f986736a85237a Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:46:09 +0000 Subject: [PATCH 19/47] Setup using submission profiles on submission --- assemblyline_ui/api/v4/submit.py | 37 +++++++++++++++++++++++++------- assemblyline_ui/api/v4/user.py | 9 ++++++-- assemblyline_ui/config.py | 7 +++++- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 643231c0..cfd47289 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -12,12 +12,13 @@ from assemblyline.common.dict_utils import flatten from assemblyline.common.str_utils import safe_str from assemblyline.common.uid import get_random_id +from assemblyline.odm.models.config import SubmissionProfile from assemblyline.odm.messages.submission import Submission from assemblyline.odm.models.user import ROLES from assemblyline_core.submission_client import SubmissionClient, SubmissionException from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import ARCHIVESTORE, STORAGE, TEMP_SUBMIT_DIR, FILESTORE, config, \ - CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER + CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ FETCH_METHODS, URL_GENERATORS @@ -255,15 +256,17 @@ def submit(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "profile": "Static Analysis", # Name of submission profile to use + + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v3/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v4/user/submission_params// } Data Block (Binary): @@ -339,10 +342,28 @@ def submit(**kwargs): # Create task object s_params = ui_to_submission_params(user_settings) + s_profile = SUBMISSION_PROFILES.get(data.get('profile')) + if s_profile and not Classification.is_accessible(user['classification'], s_profile.classification): + # User isn't allowed to use the submission profile specified + return make_api_response({}, f"You aren't allowed to use '{s_profile.name}' submission profile", 400) - # Apply provided params (if the user is allowed to) if ROLES.submission_customize in user['roles']: + # Apply provided params (if the user is allowed to) s_params.update(data.get("params", {})) + elif s_profile: + # Apply the profile (but allow the user to change some properties) + s_params.update(s_profile.params.as_primitives()) + params_data = data.get("params", {}) + for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: + if param in params_data: + # Overwrite/Set parameter with user-defined input + s_params[param] = params_data[param] + + else: + # No profile specified, raise an exception back to the user + return make_api_response({}, "You must specify a submission profile. " \ + f"One of: {list(SUBMISSION_PROFILES.keys())}", 400) + default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources if 'groups' not in s_params: diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 6821794e..aa2a0427 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -12,8 +12,8 @@ load_roles_form_acls) from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint -from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, CLASSIFICATION_ALIASES, DAILY_QUOTA_TRACKER, LOGGER, \ - STORAGE, UI_MESSAGING, VERSION, config, AI_AGENT, UI_METADATA_VALIDATION +from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, UI_MESSAGING, \ + VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( @@ -91,10 +91,12 @@ def who_am_i(**kwargs): "max_dtl": 30, # Maximum number of days retrohunt job stay in the system }, "submission": { # Submission Configuration + "configurable_params": [], # Submission parameters that are configurable when using profiles "dtl": 10, # Default number of days submission stay in the system "max_dtl": 30, # Maximum number of days submission stay in the system "file_sources": [], # List of file sources to perform remote submission into the system "metadata": {}, # Metadata compliance policy to submit to the system + "profiles": {}, # Submission profiles "verdicts": { # Verdict scoring configuration "info": 0, # Default minimum score for info "suspicious": 300, # Default minimum score for suspicious @@ -216,10 +218,13 @@ def who_am_i(**kwargs): "max_dtl": config.retrohunt.max_dtl, }, "submission": { + "configurable_params": USER_CONFIGURABLE_SUBMISSION_PARAMS, "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, + "profiles": {name: profile.params.as_primitives() for name, profile in SUBMISSION_PROFILES.items() \ + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification)}, "verdicts": { "info": config.submission.verdicts.info, "suspicious": config.submission.verdicts.suspicious, diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index 423ef964..d89f575a 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -9,7 +9,8 @@ from assemblyline.common.version import BUILD_MINOR, FRAMEWORK_VERSION, SYSTEM_VERSION from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator from assemblyline.filestore import FileStore -from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP +from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP, SubmissionProfileParams +from assemblyline.odm.models.submission import SubmissionParams from assemblyline.remote.datatypes import get_client from assemblyline.remote.datatypes.cache import Cache from assemblyline.remote.datatypes.daily_quota_tracker import DailyQuotaTracker @@ -182,5 +183,9 @@ def get_signup_queue(key): ARCHIVE_MANAGER: ArchiveManager = ArchiveManager( config=config, datastore=STORAGE, filestore=FILESTORE, identify=IDENTIFY) SERVICE_LIST = forge.CachedObject(STORAGE.list_all_services, kwargs=dict(as_obj=False, full=True)) +SUBMISSION_PROFILES = {profile.name: profile for profile in config.submission.profiles} +USER_CONFIGURABLE_SUBMISSION_PARAMS = list(set(SubmissionParams.fields().keys()) - \ + set(SubmissionProfileParams.fields().keys()) - \ + set(['quota_item', 'type', 'groups'])) # End global ################################################################# From 1f33917bf6ec1aa8c3103ad531d9082b073b6eb8 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:46:32 +0000 Subject: [PATCH 20/47] Use a method function for setting user-specified submission parameters --- assemblyline_ui/api/v4/ingest.py | 11 ++++++---- assemblyline_ui/api/v4/submit.py | 27 +++++------------------ assemblyline_ui/helper/submission.py | 23 +++++++++++++++++-- test/test_ingest.py | 30 +++++++++++++++++++++++++ test/test_submit.py | 33 +++++++++++++++++++++++++++- 5 files changed, 96 insertions(+), 28 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index 4b914fdd..3be9db76 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -22,7 +22,8 @@ STORAGE, config, FILESTORE, metadata_validator, LOGGER from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ - FETCH_METHODS, URL_GENERATORS + FETCH_METHODS, URL_GENERATORS, update_submission_parameters + from assemblyline_ui.helper.user import check_async_submission_quota, decrement_submission_ingest_quota, \ load_user_settings @@ -270,9 +271,11 @@ def ingest_single_file(**kwargs): "type": "INGEST" }) - # Apply provided params (if the user is allowed to) - if ROLES.submission_customize in user['roles']: - s_params.update(data.get("params", {})) + # Update submission parameters as specified by the user + try: + update_submission_parameters(s_params, data, user) + except Exception as e: + return make_api_response({}, str(e), 400) # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index cfd47289..dc61c9c5 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -21,7 +21,7 @@ CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ - FETCH_METHODS, URL_GENERATORS + FETCH_METHODS, URL_GENERATORS, update_submission_parameters from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota, load_user_settings SUB_API = 'submit' @@ -342,27 +342,12 @@ def submit(**kwargs): # Create task object s_params = ui_to_submission_params(user_settings) - s_profile = SUBMISSION_PROFILES.get(data.get('profile')) - if s_profile and not Classification.is_accessible(user['classification'], s_profile.classification): - # User isn't allowed to use the submission profile specified - return make_api_response({}, f"You aren't allowed to use '{s_profile.name}' submission profile", 400) - - if ROLES.submission_customize in user['roles']: - # Apply provided params (if the user is allowed to) - s_params.update(data.get("params", {})) - elif s_profile: - # Apply the profile (but allow the user to change some properties) - s_params.update(s_profile.params.as_primitives()) - params_data = data.get("params", {}) - for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: - if param in params_data: - # Overwrite/Set parameter with user-defined input - s_params[param] = params_data[param] - else: - # No profile specified, raise an exception back to the user - return make_api_response({}, "You must specify a submission profile. " \ - f"One of: {list(SUBMISSION_PROFILES.keys())}", 400) + # Update submission parameters as specified by the user + try: + update_submission_parameters(s_params, data, user) + except Exception as e: + return make_api_response({}, str(e), 400) default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 4d7c57c7..ce0436c5 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -16,7 +16,7 @@ from assemblyline.odm.models.config import HASH_PATTERN_MAP from assemblyline.odm.messages.submission import SubmissionMessage from assemblyline.odm.models.user import ROLES -from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE +from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS # Baseline fetch methods FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url']) @@ -37,6 +37,7 @@ MYIP = '127.0.0.1' + ############################# # download functions class FileTooBigException(Exception): @@ -181,7 +182,25 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo - +def update_submission_parameters(s_params: dict, data: dict, user: dict): + s_profile = SUBMISSION_PROFILES.get(data.get('profile')) + # Apply provided params (if the user is allowed to) + if ROLES.submission_customize in user['roles']: + s_params.update(data.get("params", {})) + elif s_profile: + if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification): + # User isn't allowed to use the submission profile specified + raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") + # Apply the profile (but allow the user to change some properties) + s_params.update(s_profile.params.as_primitives()) + params_data = data.get("params", {}) + for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: + if param in params_data: + # Overwrite/Set parameter with user-defined input + s_params[param] = params_data[param] + else: + # No profile specified, raise an exception back to the user + raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") diff --git a/test/test_ingest.py b/test/test_ingest.py index cf7e8822..b91b3c8c 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -356,3 +356,33 @@ def test_get_message_list_with_paging(datastore, login_session): message_list += resp for x in range(NUM_FILES): assert message_list[x] == messages[x] + +def test_ingest_submission_profile(datastore, login_session, scheduler): + _, session, host = login_session + iq.delete() + + # Make the user a simple user and try to submit + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'type', 'user')]) + byte_str = get_random_phrase(wmin=30, wmax=75).encode() + sha256 = hashlib.sha256(byte_str).hexdigest() + data = { + 'base64': base64.b64encode(byte_str).decode('ascii'), + 'metadata': {'test': 'test_submit_base64_nameless'} + } + with pytest.raises(APIError, match="You must specify a submission profile"): + # A basic user must specify a submission profile name + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with no parameters + data['profile'] = "Static Analysis" + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with a parameter you aren't allowed to set + # The system should silently ignore your parameter and still create a submission + data['params'] = {'deep_scan': True} + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Restore original roles for later tests + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), + (datastore.user.UPDATE_APPEND, 'type', 'admin')]) diff --git a/test/test_submit.py b/test/test_submit.py index 60df0227..9d117db2 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -6,7 +6,7 @@ import random import tempfile -from conftest import get_api_data +from conftest import get_api_data, APIError from assemblyline.common import forge from assemblyline.odm.models.config import HASH_PATTERN_MAP @@ -282,3 +282,34 @@ def test_submit_base64_nameless(datastore, login_session, scheduler): msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) assert msg.submission.sid == resp['sid'] + +def test_submit_submission_profile(datastore, login_session, scheduler): + _, session, host = login_session + sq.delete() + + # Make the user a simple user and try to submit + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'type', 'user')]) + byte_str = get_random_phrase(wmin=30, wmax=75).encode() + sha256 = hashlib.sha256(byte_str).hexdigest() + data = { + 'base64': base64.b64encode(byte_str).decode('ascii'), + 'metadata': {'test': 'test_submit_base64_nameless'} + } + with pytest.raises(APIError, match="You must specify a submission profile"): + # A basic user must specify a submission profile name + get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with no parameters + data['profile'] = "Static Analysis" + get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with a parameter you aren't allowed to set + # The system should silently ignore your parameter and still create a submission + data['params'] = {'deep_scan': True} + resp = get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + assert resp['params']['deep_scan'] == False + + # Restore original roles for later tests + datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), + (datastore.user.UPDATE_APPEND, 'type', 'admin')]) From 54763a06e2b5f3dff915a3decffe709e3a8b4c7a Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:34:46 +0000 Subject: [PATCH 21/47] Rename parameter to deconflict with pre-existing `profile` parameter --- assemblyline_ui/api/v4/ingest.py | 14 ++++++++------ assemblyline_ui/api/v4/submission.py | 2 -- assemblyline_ui/api/v4/submit.py | 14 +++++++------- assemblyline_ui/api/v4/ui.py | 9 ++++++++- assemblyline_ui/api/v4/user.py | 29 ++++++++++++---------------- assemblyline_ui/config.py | 4 +--- assemblyline_ui/helper/submission.py | 13 +++++++------ 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index 3be9db76..10a017a1 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -152,15 +152,17 @@ def ingest_single_file(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "profile_name": "Static Analysis", # Name of submission profile to use + + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v3/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v3/user/submission_params// "generate_alert": False, # Generate an alert in our alerting system or not "notification_queue": None, # Name of the notification queue diff --git a/assemblyline_ui/api/v4/submission.py b/assemblyline_ui/api/v4/submission.py index 5f8e68a2..3a74e4e8 100644 --- a/assemblyline_ui/api/v4/submission.py +++ b/assemblyline_ui/api/v4/submission.py @@ -312,7 +312,6 @@ def get_full_results(sid, **kwargs): }, "state": "completed", # State of the submission "submission": { # Submission Block - "profile": true, # Should keep stats about execution? "description": "", # Submission description "ttl": 30, # Submission days to live "ignore_filtering": false, # Ignore filtering services? @@ -451,7 +450,6 @@ def get_submission(sid, **kwargs): ["FNAME", "sha256"], ...], # Each file = List of name/sha256 "errors": [], # List of error keys (sha256.ServiceName) "submission": { # Submission Block - "profile": true, # Should keep stats about execution? "description": "", # Submission description "ttl": 30, # Submission days to live "ignore_filtering": false, # Ignore filtering services? diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index dc61c9c5..dd3f68e7 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -256,17 +256,17 @@ def submit(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "profile": "Static Analysis", # Name of submission profile to use + "profile_name": "Static Analysis", # Name of submission profile to use - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v4/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v4/user/submission_params// } Data Block (Binary): diff --git a/assemblyline_ui/api/v4/ui.py b/assemblyline_ui/api/v4/ui.py index 363890a1..e0dc1a7b 100644 --- a/assemblyline_ui/api/v4/ui.py +++ b/assemblyline_ui/api/v4/ui.py @@ -16,7 +16,7 @@ from assemblyline_ui.config import TEMP_DIR, STORAGE, FILESTORE, config, CLASSIFICATION as Classification, \ IDENTIFY, metadata_validator from assemblyline_ui.helper.service import ui_to_submission_params -from assemblyline_ui.helper.submission import submission_received +from assemblyline_ui.helper.submission import submission_received, update_submission_parameters from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota from assemblyline_core.submission_client import SubmissionClient, SubmissionException @@ -260,6 +260,13 @@ def start_ui_submission(ui_sid, **kwargs): # Submit to dispatcher try: params = ui_to_submission_params(ui_params) + + # Update submission parameters as specified by the user + try: + update_submission_parameters(params, ui_params, user) + except Exception as e: + return make_api_response({}, str(e), 400) + metadata = params.pop("metadata", {}) # Enforce maximum DTL diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index aa2a0427..eb4a3d4b 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -13,7 +13,7 @@ from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, UI_MESSAGING, \ - VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS + VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( @@ -218,7 +218,6 @@ def who_am_i(**kwargs): "max_dtl": config.retrohunt.max_dtl, }, "submission": { - "configurable_params": USER_CONFIGURABLE_SUBMISSION_PARAMS, "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, "file_sources": file_sources, @@ -889,19 +888,17 @@ def get_user_settings(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled - "classification": "", # Default classification for this user sumbissions - "description": "", # Default description for this user's submissions - "download_encoding": "blah", # Default encoding for downloaded files - "default_zip_password": "pass", # Default password for password protected ZIP - "expand_min_score": 100, # Default minimum score to auto-expand sections - "priority": 1000, # Default submission priority - "service_spec": [], # Default Service specific parameters - "ignore_cache": true, # Should file be reprocessed even if there are cached results - "groups": [ ... ], # Default groups selection for the user scans - "ttl": 30, # Default time to live in days of the users submissions - "services": [ ... ], # Default list of selected services - "ignore_filtering": false # Should filtering services by ignored? + "classification": "", # Default classification for this user sumbissions + "description": "", # Default description for this user's submissions + "download_encoding": "blah", # Default encoding for downloaded files + "expand_min_score": 100, # Default minimum score to auto-expand sections + "priority": 1000, # Default submission priority + "service_spec": [], # Default Service specific parameters + "ignore_cache": true, # Should file be reprocessed even if there are cached results + "groups": [ ... ], # Default groups selection for the user scans + "ttl": 30, # Default time to live in days of the users submissions + "services": [ ... ], # Default list of selected services + "ignore_filtering": false # Should filtering services by ignored? } """ user = kwargs['user'] @@ -928,7 +925,6 @@ def set_user_settings(username, **kwargs): Data Block: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "default_zip_password": "zippy" # Default password used for protected file downloads "description": "", # Default description for this user's submissions @@ -989,7 +985,6 @@ def get_user_submission_params(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "description": "", # Default description for this user's submissions "priority": 1000, # Default submission priority diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index d89f575a..b3b08c10 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -184,8 +184,6 @@ def get_signup_queue(key): config=config, datastore=STORAGE, filestore=FILESTORE, identify=IDENTIFY) SERVICE_LIST = forge.CachedObject(STORAGE.list_all_services, kwargs=dict(as_obj=False, full=True)) SUBMISSION_PROFILES = {profile.name: profile for profile in config.submission.profiles} -USER_CONFIGURABLE_SUBMISSION_PARAMS = list(set(SubmissionParams.fields().keys()) - \ - set(SubmissionProfileParams.fields().keys()) - \ - set(['quota_item', 'type', 'groups'])) + # End global ################################################################# diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index ce0436c5..77271415 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -16,7 +16,7 @@ from assemblyline.odm.models.config import HASH_PATTERN_MAP from assemblyline.odm.messages.submission import SubmissionMessage from assemblyline.odm.models.user import ROLES -from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS +from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES # Baseline fetch methods FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url']) @@ -183,7 +183,7 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo def update_submission_parameters(s_params: dict, data: dict, user: dict): - s_profile = SUBMISSION_PROFILES.get(data.get('profile')) + s_profile = SUBMISSION_PROFILES.get(data.get('profile_name')) # Apply provided params (if the user is allowed to) if ROLES.submission_customize in user['roles']: s_params.update(data.get("params", {})) @@ -193,11 +193,12 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params.update(s_profile.params.as_primitives()) + s_fields = s_profile.params.fields() params_data = data.get("params", {}) - for param in USER_CONFIGURABLE_SUBMISSION_PARAMS: - if param in params_data: - # Overwrite/Set parameter with user-defined input - s_params[param] = params_data[param] + for param, value in params_data.items(): + if param in s_fields and s_fields[param].default_set == True: + # Set parameter with user-defined input since it wasn't explicitly declared in the configuration + s_params[param] = value else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") From 89265785ae734be60d1e4492dc5957db2e0c0e2a Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:36:11 +0000 Subject: [PATCH 22/47] Update tests --- test/test_ingest.py | 2 +- test/test_submit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_ingest.py b/test/test_ingest.py index b91b3c8c..904e5117 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -375,7 +375,7 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile'] = "Static Analysis" + data['profile_name'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set diff --git a/test/test_submit.py b/test/test_submit.py index 9d117db2..09ffb8ba 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -301,7 +301,7 @@ def test_submit_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile'] = "Static Analysis" + data['profile_name'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set From 5bc9810ff90fbed7170c011ec07317ea0530a01a Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 15:46:04 +0000 Subject: [PATCH 23/47] Rename parameter for clarity --- assemblyline_ui/api/v4/ingest.py | 2 +- assemblyline_ui/api/v4/submit.py | 2 +- assemblyline_ui/api/v4/user.py | 7 +++++++ assemblyline_ui/helper/submission.py | 2 +- test/test_ingest.py | 2 +- test/test_submit.py | 2 +- 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index 10a017a1..eb9bc63f 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -154,7 +154,7 @@ def ingest_single_file(**kwargs): // OPTIONAL VALUES "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "profile_name": "Static Analysis", # Name of submission profile to use + "submission_profile": "Static Analysis", # Name of submission profile to use "metadata": { # Submission metadata "key": val, # Key/Value pair for metadata parameters diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index dd3f68e7..2de06863 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -258,7 +258,7 @@ def submit(**kwargs): // OPTIONAL VALUES "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "profile_name": "Static Analysis", # Name of submission profile to use + "submission_profile": "Static Analysis", # Name of submission profile to use "metadata": { # Submission metadata "key": val, # Key/Value pair for metadata parameters diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index eb4a3d4b..b8164e34 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -195,6 +195,13 @@ def who_am_i(**kwargs): [file_sources["sha256"]["sources"].append(x.name) for x in config.submission.sha256_sources if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] + submission_profiles = {} + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) + for p_cls in profile.params.fields().values() if p_cls.default_set == False} + user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 77271415..04a5b7a7 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -183,7 +183,7 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo def update_submission_parameters(s_params: dict, data: dict, user: dict): - s_profile = SUBMISSION_PROFILES.get(data.get('profile_name')) + s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) # Apply provided params (if the user is allowed to) if ROLES.submission_customize in user['roles']: s_params.update(data.get("params", {})) diff --git a/test/test_ingest.py b/test/test_ingest.py index 904e5117..520965a0 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -375,7 +375,7 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile_name'] = "Static Analysis" + data['submission_profile'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set diff --git a/test/test_submit.py b/test/test_submit.py index 09ffb8ba..3818286a 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -301,7 +301,7 @@ def test_submit_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['profile_name'] = "Static Analysis" + data['submission_profile'] = "Static Analysis" get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set From 3d4c6f14f7062c700fee711d7acfeed533fde1d2 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:45:45 +0000 Subject: [PATCH 24/47] Allow users to set parameters that aren't enforced by profile --- assemblyline_ui/api/v4/user.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index b8164e34..c7374774 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -7,7 +7,7 @@ from assemblyline.common.security import (check_password_requirements, get_password_hash, get_password_requirement_message) from assemblyline.datastore.exceptions import SearchException -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.models.user import (ACL_MAP, ROLES, USER_ROLES, USER_TYPE_DEP, USER_TYPES, User, load_roles, load_roles_form_acls) from assemblyline.odm.models.user_favorites import Favorite @@ -196,11 +196,16 @@ def who_am_i(**kwargs): if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] submission_profiles = {} - for name, profile in SUBMISSION_PROFILES.items(): - if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): - # We want to pass forward the configurations that have been explicitly set as a configuration - submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) - for p_cls in profile.params.fields().values() if p_cls.default_set == False} + if config.submission.profiles == DEFAULT_SUBMISSION_PROFILES: + # If these are exactly the same as the default values, then it's accessible to everyone + submission_profiles = {profile['name']: profile['params'] for profile in DEFAULT_SUBMISSION_PROFILES} + else: + # Filter profiles based on accessibility to the user + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) + for p_cls in profile.params.fields().values() if p_cls.default_set == False} user_data['configuration'] = { "auth": { @@ -229,8 +234,7 @@ def who_am_i(**kwargs): "max_dtl": config.submission.max_dtl, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, - "profiles": {name: profile.params.as_primitives() for name, profile in SUBMISSION_PROFILES.items() \ - if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification)}, + "profiles": submission_profiles, "verdicts": { "info": config.submission.verdicts.info, "suspicious": config.submission.verdicts.suspicious, From 792c2939246c44d9cf36971c9802f7d6b6f4b1e8 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:29:31 +0000 Subject: [PATCH 25/47] Expand service categories to make it easier for the UI to lock down configurations --- assemblyline_ui/api/v4/user.py | 16 ++++++++++++++-- assemblyline_ui/helper/submission.py | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index c7374774..2bdfe44a 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -1,3 +1,4 @@ +from copy import deepcopy from typing import List from assemblyline.odm.models.config import ExternalLinks from flask import request, session as flsk_session @@ -21,7 +22,7 @@ API_PRIV_MAP) from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException -from .federated_lookup import filtered_tag_names +from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names SUB_API = 'user' @@ -195,10 +196,11 @@ def who_am_i(**kwargs): [file_sources["sha256"]["sources"].append(x.name) for x in config.submission.sha256_sources if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] + # Prepare submission profile configurations for UI submission_profiles = {} if config.submission.profiles == DEFAULT_SUBMISSION_PROFILES: # If these are exactly the same as the default values, then it's accessible to everyone - submission_profiles = {profile['name']: profile['params'] for profile in DEFAULT_SUBMISSION_PROFILES} + submission_profiles = {profile['name']: deepcopy(profile['params']) for profile in DEFAULT_SUBMISSION_PROFILES} else: # Filter profiles based on accessibility to the user for name, profile in SUBMISSION_PROFILES.items(): @@ -207,6 +209,16 @@ def who_am_i(**kwargs): submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) for p_cls in profile.params.fields().values() if p_cls.default_set == False} + # Expand service categories if used in submission profiles (assists with the UI locking down service selection) + service_categories = list(STORAGE.service.facet('category').keys()) + for profile in submission_profiles.values(): + for key, services in profile.get("services", {}).items(): + expanded_services = list() + for srv in services: + if srv in service_categories: + expanded_services.extend([i['name'] for i in STORAGE.service.search(f"category:{srv}", as_obj=False, fl="name")['items']]) + profile['services'][key] = list(set(services).union(set(expanded_services))) + user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 04a5b7a7..3a17d417 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -9,6 +9,7 @@ from typing import List from urllib.parse import urlparse +from assemblyline.common.dict_utils import recursive_update from assemblyline.common.file import make_uri_file from assemblyline.common.isotime import now_as_iso from assemblyline.common.str_utils import safe_str @@ -192,7 +193,7 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): # User isn't allowed to use the submission profile specified raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) - s_params.update(s_profile.params.as_primitives()) + s_params = recursive_update(s_params, s_profile.params.as_primitives()) s_fields = s_profile.params.fields() params_data = data.get("params", {}) for param, value in params_data.items(): From e541c0f4147bba29c345129467aa7ccf68f7c741 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 9 Aug 2024 19:26:08 +0000 Subject: [PATCH 26/47] Patch testing --- assemblyline_ui/api/v4/user.py | 14 ++++---------- assemblyline_ui/helper/submission.py | 9 ++------- test/test_ingest.py | 7 +++++-- test/test_submit.py | 10 +++++++--- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 2bdfe44a..f541232f 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -198,16 +198,10 @@ def who_am_i(**kwargs): # Prepare submission profile configurations for UI submission_profiles = {} - if config.submission.profiles == DEFAULT_SUBMISSION_PROFILES: - # If these are exactly the same as the default values, then it's accessible to everyone - submission_profiles = {profile['name']: deepcopy(profile['params']) for profile in DEFAULT_SUBMISSION_PROFILES} - else: - # Filter profiles based on accessibility to the user - for name, profile in SUBMISSION_PROFILES.items(): - if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): - # We want to pass forward the configurations that have been explicitly set as a configuration - submission_profiles[name] = {p_cls.name: getattr(profile.params, p_cls.name) - for p_cls in profile.params.fields().values() if p_cls.default_set == False} + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = profile.params.as_primitives(strip_null=True) # Expand service categories if used in submission profiles (assists with the UI locking down service selection) service_categories = list(STORAGE.service.facet('category').keys()) diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 3a17d417..357d519d 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -193,13 +193,8 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): # User isn't allowed to use the submission profile specified raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) - s_params = recursive_update(s_params, s_profile.params.as_primitives()) - s_fields = s_profile.params.fields() - params_data = data.get("params", {}) - for param, value in params_data.items(): - if param in s_fields and s_fields[param].default_set == True: - # Set parameter with user-defined input since it wasn't explicitly declared in the configuration - s_params[param] = value + s_params = recursive_update(s_params, data.get("params", {})) + s_params = recursive_update(s_params, s_profile.params.as_primitives(strip_null=True)) else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") diff --git a/test/test_ingest.py b/test/test_ingest.py index 520965a0..785d9a50 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -10,7 +10,7 @@ from assemblyline.common import forge from assemblyline.odm.messages.submission import Submission -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.models.file import File from assemblyline.odm.randomizer import random_model_obj, get_random_phrase from assemblyline.odm.random_data import create_users, wipe_users, create_services, wipe_services @@ -375,11 +375,14 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['submission_profile'] = "Static Analysis" + profile = DEFAULT_SUBMISSION_PROFILES[0] + data['submission_profile'] = profile["name"] get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set # The system should silently ignore your parameter and still create a submission + data['params'] = {'services': {'selected': ['blah']}} + # But also try setting a parameter that you are allowed to set data['params'] = {'deep_scan': True} get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) diff --git a/test/test_submit.py b/test/test_submit.py index 3818286a..47c71c54 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -9,7 +9,7 @@ from conftest import get_api_data, APIError from assemblyline.common import forge -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions from assemblyline.odm.randomizer import get_random_phrase from assemblyline.remote.datatypes.queues.named import NamedQueue @@ -301,14 +301,18 @@ def test_submit_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with no parameters - data['submission_profile'] = "Static Analysis" + profile = DEFAULT_SUBMISSION_PROFILES[0] + data['submission_profile'] = profile['name'] get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) # Try using a submission profile with a parameter you aren't allowed to set # The system should silently ignore your parameter and still create a submission + data['params'] = {'services': {'selected': ['blah']}} + # But also try setting a parameter that you are allowed to set data['params'] = {'deep_scan': True} resp = get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) - assert resp['params']['deep_scan'] == False + assert resp['params']['services']['selected'] == profile['params']['services']['selected'] + assert resp['params']['deep_scan'] == True # Restore original roles for later tests datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), From 8aeb0a6989e473ee29cd61ae974debd20537c87e Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:35:33 +0000 Subject: [PATCH 27/47] Modify APIs to allow editing/fetching of user submission profiles --- assemblyline_ui/api/v4/user.py | 1 + assemblyline_ui/config.py | 3 +-- assemblyline_ui/helper/service.py | 37 ++++++++++++++++++++++++++++++- assemblyline_ui/helper/user.py | 14 +++++++++++- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index f541232f..707d3879 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -202,6 +202,7 @@ def who_am_i(**kwargs): if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): # We want to pass forward the configurations that have been explicitly set as a configuration submission_profiles[name] = profile.params.as_primitives(strip_null=True) + submission_profiles[name]["editable_params"] = profile.editable_params # Expand service categories if used in submission profiles (assists with the UI locking down service selection) service_categories = list(STORAGE.service.facet('category').keys()) diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index b3b08c10..9b596d1a 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -9,8 +9,7 @@ from assemblyline.common.version import BUILD_MINOR, FRAMEWORK_VERSION, SYSTEM_VERSION from assemblyline.datastore.helper import AssemblylineDatastore, MetadataValidator from assemblyline.filestore import FileStore -from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP, SubmissionProfileParams -from assemblyline.odm.models.submission import SubmissionParams +from assemblyline.odm.models.config import METADATA_FIELDTYPE_MAP from assemblyline.remote.datatypes import get_client from assemblyline.remote.datatypes.cache import Cache from assemblyline.remote.datatypes.daily_quota_tracker import DailyQuotaTracker diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 249bf859..9116dcb7 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -1,6 +1,7 @@ from copy import copy from typing import Any, Optional -from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST +from assemblyline.common.dict_utils import recursive_update +from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST, SUBMISSION_PROFILES, STORAGE from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, SubmissionParams from assemblyline.odm.models.user_settings import UserSettings @@ -8,6 +9,40 @@ USER_SETTINGS_FIELDS = list(UserSettings.fields().keys()) SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) +def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): + out = {} + for profile in SUBMISSION_PROFILES.values(): + if CLASSIFICATION.is_accessible(classification, profile.classification): + user_default_profile = user_default_values.get(profile.name, {}) + + params = copy(profile.params.as_primitives(strip_null=True)) + out[profile.name] = recursive_update(params, user_default_values.get(profile.name, {})) + + service_spec = [] + # If there are any editable service parameters that haven't been set, then assign their default values + for service, editable_params in profile.editable_params.items(): + service_obj = STORAGE.get_service_with_delta(service, as_obj=False) + if not service_obj: + continue + param_object = {'name': service, "params": []} + profile_service_spec = user_default_profile.get('service_spec', {}).get(service, {}) + for param in service_obj['submission_params']: + if param['name'] not in editable_params: + # Service parameter isn't allowed to be overridden in this profile + continue + + new_param = copy(param) + if profile_service_spec.get(param['name']): + # Overwrite with user-specific value for profile + new_param['value'] = profile_service_spec[param['name']] + new_param["hide"] = False + param_object["params"].append(new_param) + service_spec.append(param_object) + + # Overwrite 'service_spec' value to be compliant with frontend rendering + out[profile.name]['service_spec'] = service_spec + return out + def get_default_service_spec(srv_list=None, user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): if not srv_list: srv_list = SERVICE_LIST diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 53c3aeb5..651bb19c 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -7,7 +7,7 @@ from assemblyline.odm.models.user_settings import UserSettings from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, SUBMISSION_TRACKER, \ config, CLASSIFICATION as Classification, SERVICE_LIST, DOWNLOAD_ENCODING, DEFAULT_ZIP_PASSWORD -from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services +from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services, get_default_submission_profiles from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException, AuthenticationException ACCOUNT_USER_MODIFIABLE = ["name", "avatar", "password"] @@ -290,6 +290,10 @@ def load_user_settings(user): # Only display services that a user is allowed to see settings['service_spec'] = get_default_service_spec(srv_list, settings.get('service_spec', {}), user_classfication) settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) + settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], + user_classfication) + settings['preferred_submission_profile'] = user.get('preferred_submission_profile') or \ + list(settings['submission_profiles'].keys())[0] settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) # Normalize the user's classification @@ -301,4 +305,12 @@ def load_user_settings(user): def save_user_settings(username, data): data["services"] = {'selected': simplify_services(data["services"])} + # Ensure submission profile changes are valid + for profile_name, profile_spec in data["submission_profiles"].items(): + saved_spec = {} + for spec in profile_spec["service_spec"]: + saved_spec[spec["name"]] = {param['name']: param['value'] for param in spec["params"] if param['name'] in config.submission.profiles[profile_name].editable_params} + data["submission_profiles"][profile_name]["service_spec"] = saved_spec + + return STORAGE.user_settings.save(username, data) From 9aa736bfade6e4f222ea8f7b7ee30e5498c90aaa Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 01:30:27 +0000 Subject: [PATCH 28/47] Changed the submission profile's loading and setting methods --- assemblyline_ui/api/v4/user.py | 73 +++++++++++++++++++------- assemblyline_ui/helper/service.py | 36 +++---------- assemblyline_ui/helper/user.py | 85 +++++++++++++++++++++++++------ 3 files changed, 132 insertions(+), 62 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 707d3879..37cc97e5 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -1,29 +1,55 @@ from copy import deepcopy from typing import List -from assemblyline.odm.models.config import ExternalLinks -from flask import request, session as flsk_session from assemblyline.common.comms import send_activated_email, send_authorize_email from assemblyline.common.isotime import now_as_iso -from assemblyline.common.security import (check_password_requirements, get_password_hash, - get_password_requirement_message) +from assemblyline.common.security import ( + check_password_requirements, + get_password_hash, + get_password_requirement_message, +) from assemblyline.datastore.exceptions import SearchException -from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES -from assemblyline.odm.models.user import (ACL_MAP, ROLES, USER_ROLES, USER_TYPE_DEP, USER_TYPES, User, load_roles, - load_roles_form_acls) +from assemblyline.odm.models.config import HASH_PATTERN_MAP, ExternalLinks +from assemblyline.odm.models.user import ( + ACL_MAP, + ROLES, + USER_ROLES, + USER_TYPE_DEP, + USER_TYPES, + User, + load_roles, + load_roles_form_acls, +) from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint -from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, UI_MESSAGING, \ - VERSION, config, AI_AGENT, UI_METADATA_VALIDATION, SUBMISSION_PROFILES +from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names +from assemblyline_ui.config import ( + AI_AGENT, + APPS_LIST, + CLASSIFICATION, + CLASSIFICATION_ALIASES, + DAILY_QUOTA_TRACKER, + LOGGER, + STORAGE, + SUBMISSION_PROFILES, + UI_MESSAGING, + UI_METADATA_VALIDATION, + VERSION, + config, +) from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( - get_default_user_quotas, get_dynamic_classification, load_user_settings, save_user_account, save_user_settings, - API_PRIV_MAP) + API_PRIV_MAP, + get_default_user_quotas, + get_dynamic_classification, + load_user_settings, + save_user_account, + save_user_settings, +) from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException - -from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names - +from flask import request +from flask import session as flsk_session SUB_API = 'user' user_api = make_subapi_blueprint(SUB_API, api_version=4) @@ -211,7 +237,8 @@ def who_am_i(**kwargs): expanded_services = list() for srv in services: if srv in service_categories: - expanded_services.extend([i['name'] for i in STORAGE.service.search(f"category:{srv}", as_obj=False, fl="name")['items']]) + expanded_services.extend([i['name'] for i in STORAGE.service.search( + f"category:{srv}", as_obj=False, fl="name")['items']]) profile['services'][key] = list(set(services).union(set(expanded_services))) user_data['configuration'] = { @@ -963,8 +990,6 @@ def set_user_settings(username, **kwargs): } """ user = kwargs['user'] - if username != user['uname'] and ROLES.administration not in user['roles']: - raise AccessDeniedException("You are not allowed to set settings for another user then yourself.") try: data = request.json @@ -972,7 +997,19 @@ def set_user_settings(username, **kwargs): if not data.get('default_zip_password', ''): return make_api_response({"success": False}, "Encryption password can't be empty.", 403) - if save_user_settings(username, data): + # Changing your own settings + if username == user['uname']: + if ROLES.administration not in user['roles'] or ROLES.self_manage not in user['roles']: + raise AccessDeniedException("You are not allowed to modify your own settings.") + edit_user = user + + # Changing someone else's settings + else: + if ROLES.administration not in user['roles']: + raise AccessDeniedException("You are not allowed to set settings for another user then yourself.") + edit_user = STORAGE.user.get(username, as_obj=False) + + if save_user_settings(edit_user, data): return make_api_response({"success": True}) else: return make_api_response({"success": False}, "Failed to save user's settings", 500) diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 9116dcb7..5a1a279a 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -9,40 +9,20 @@ USER_SETTINGS_FIELDS = list(UserSettings.fields().keys()) SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) + def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): out = {} for profile in SUBMISSION_PROFILES.values(): if CLASSIFICATION.is_accessible(classification, profile.classification): - user_default_profile = user_default_values.get(profile.name, {}) - - params = copy(profile.params.as_primitives(strip_null=True)) - out[profile.name] = recursive_update(params, user_default_values.get(profile.name, {})) - - service_spec = [] - # If there are any editable service parameters that haven't been set, then assign their default values - for service, editable_params in profile.editable_params.items(): - service_obj = STORAGE.get_service_with_delta(service, as_obj=False) - if not service_obj: - continue - param_object = {'name': service, "params": []} - profile_service_spec = user_default_profile.get('service_spec', {}).get(service, {}) - for param in service_obj['submission_params']: - if param['name'] not in editable_params: - # Service parameter isn't allowed to be overridden in this profile - continue - - new_param = copy(param) - if profile_service_spec.get(param['name']): - # Overwrite with user-specific value for profile - new_param['value'] = profile_service_spec[param['name']] - new_param["hide"] = False - param_object["params"].append(new_param) - service_spec.append(param_object) - - # Overwrite 'service_spec' value to be compliant with frontend rendering - out[profile.name]['service_spec'] = service_spec + profile_values = copy(profile.as_primitives(strip_null=True)) + out[profile.name] = recursive_update(profile_values, user_default_values.get(profile.name, {})) + + out[profile.name].pop("name") + out[profile.name].pop("classification") + return out + def get_default_service_spec(srv_list=None, user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): if not srv_list: srv_list = SERVICE_LIST diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 651bb19c..8462723d 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,14 +1,30 @@ from typing import Optional -from flask import session as flsk_session - from assemblyline.common.str_utils import safe_str -from assemblyline.odm.models.user import User, load_roles, ROLES +from assemblyline.odm.models.config import SubmissionProfileParams +from assemblyline.odm.models.user import ROLES, User, load_roles from assemblyline.odm.models.user_settings import UserSettings -from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, SUBMISSION_TRACKER, \ - config, CLASSIFICATION as Classification, SERVICE_LIST, DOWNLOAD_ENCODING, DEFAULT_ZIP_PASSWORD -from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services, get_default_submission_profiles -from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException, AuthenticationException +from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER +from assemblyline_ui.config import CLASSIFICATION as Classification +from assemblyline_ui.config import ( + DAILY_QUOTA_TRACKER, + DEFAULT_ZIP_PASSWORD, + DOWNLOAD_ENCODING, + LOGGER, + SERVICE_LIST, + STORAGE, + SUBMISSION_PROFILES, + SUBMISSION_TRACKER, + config, +) +from assemblyline_ui.helper.service import ( + get_default_service_list, + get_default_service_spec, + get_default_submission_profiles, + simplify_services, +) +from assemblyline_ui.http_exceptions import AccessDeniedException, AuthenticationException, InvalidDataException +from flask import session as flsk_session ACCOUNT_USER_MODIFIABLE = ["name", "avatar", "password"] @@ -302,15 +318,52 @@ def load_user_settings(user): return settings -def save_user_settings(username, data): - data["services"] = {'selected': simplify_services(data["services"])} +def save_user_settings(user, data): + username = user.get('uname', None) + if username == None: + raise Exception("Invalid username") + + out = STORAGE.user_settings.get(username).as_primitives() + for key in out.keys(): + if key in data and key not in ["services", "service_spec", "submission_profiles"]: + out[key] = data.get(key, None) + + out["services"] = {'selected': simplify_services(data["services"])} + + classification = user.get("classification", None) + submission_customize = ROLES.submission_customize in user['roles'] + srv_list = [x['name'] for x in SERVICE_LIST if x['enabled']] + srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] + srv_list = list(set(srv_list)) + + submission_profiles = {} + for name, profile in SUBMISSION_PROFILES.items(): + if Classification.is_accessible(classification, profile.classification): + user_params = data.get('submission_profiles', {}).get(profile.name, {}).get('params', {}) + + # Applying the submission params + profile_defaults = SubmissionProfileParams().as_primitives() + default_keys = profile["editable_params"].get("submit", []) + for key in profile_defaults.keys(): + if key in user_params and key not in ["services", "service_spec"] and \ + (submission_customize or key in default_keys): + profile["params"][key] = user_params.get(key, None) + + # Applying the selected services + if submission_customize: + profile["params"]["services"]["selected"] = [x for x in user_params.get( + 'services', {}).get('selected', []) if x in srv_list] + + # Applying the service specs + profile["params"]["service_spec"] = {} + for svr_name, spec in user_params.get("service_spec", {}).items(): + for p_name, p_value in spec.items(): + if (p_name in profile["editable_params"].get(svr_name, []) or submission_customize) and \ + svr_name in srv_list: + profile["params"]["service_spec"].setdefault(svr_name, {}).setdefault(p_name, p_value) - # Ensure submission profile changes are valid - for profile_name, profile_spec in data["submission_profiles"].items(): - saved_spec = {} - for spec in profile_spec["service_spec"]: - saved_spec[spec["name"]] = {param['name']: param['value'] for param in spec["params"] if param['name'] in config.submission.profiles[profile_name].editable_params} - data["submission_profiles"][profile_name]["service_spec"] = saved_spec + submission_profiles[name] = profile.params.as_primitives(strip_null=True) + out["submission_profiles"] = submission_profiles - return STORAGE.user_settings.save(username, data) + return STORAGE.user_settings.save(username, out) From 8eb4c04442ca4bc7ed1ed791130efbbfe8b4a812 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 02:10:26 +0000 Subject: [PATCH 29/47] Set the preferred_submission_profile if it doesn't exist in the existing settings --- assemblyline_ui/helper/user.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 8462723d..c05a9682 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -336,6 +336,11 @@ def save_user_settings(user, data): srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] srv_list = list(set(srv_list)) + if data.get('preferred_submission_profile', None) not in SUBMISSION_PROFILES.keys(): + out['preferred_submission_profile'] = SUBMISSION_PROFILES.keys()[0] + else: + out['preferred_submission_profile'] = data['preferred_submission_profile'] + submission_profiles = {} for name, profile in SUBMISSION_PROFILES.items(): if Classification.is_accessible(classification, profile.classification): From cba0c6dc61bb8ac11c71f78c99590eec67ff661d Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Tue, 17 Dec 2024 02:26:44 +0000 Subject: [PATCH 30/47] Fixed the loading of the preferred_submission_profile --- assemblyline_ui/helper/user.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index c05a9682..87b30b87 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -308,13 +308,14 @@ def load_user_settings(user): settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], user_classfication) - settings['preferred_submission_profile'] = user.get('preferred_submission_profile') or \ - list(settings['submission_profiles'].keys())[0] settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) # Normalize the user's classification settings['classification'] = Classification.normalize_classification(settings['classification']) + if settings.get('preferred_submission_profile', None) not in list(settings['submission_profiles'].keys()): + settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] + return settings From 5e49153e489dd7304e28590cb1d8a5bb9fe4b205 Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Fri, 3 Jan 2025 11:49:41 +0000 Subject: [PATCH 31/47] Added the max file size to the /whoami path --- assemblyline_ui/api/v4/user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index 37cc97e5..f2642847 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -121,6 +121,7 @@ def who_am_i(**kwargs): "configurable_params": [], # Submission parameters that are configurable when using profiles "dtl": 10, # Default number of days submission stay in the system "max_dtl": 30, # Maximum number of days submission stay in the system + "max_file_size": 104857600, # Maximum size for files submitted in the system "file_sources": [], # List of file sources to perform remote submission into the system "metadata": {}, # Metadata compliance policy to submit to the system "profiles": {}, # Submission profiles @@ -266,6 +267,7 @@ def who_am_i(**kwargs): "submission": { "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, + "max_file_size": config.submission.max_file_size, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, "profiles": submission_profiles, From dcfe3984e7ab75d3e24cc7aab1b07b4c76e7f13f Mon Sep 17 00:00:00 2001 From: cccs-nr Date: Fri, 17 Jan 2025 18:27:32 +0000 Subject: [PATCH 32/47] minor change to the load_user_settings --- assemblyline_ui/helper/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 87b30b87..41b67bb2 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -285,7 +285,7 @@ def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - settings = STORAGE.user_settings.get_if_exists(user['uname'], as_obj=False) + settings = STORAGE.user_settings.get_if_exists(user['uname']).as_primitives(strip_null=True) srv_list = [x for x in SERVICE_LIST if x['enabled']] if not settings: def_srv_list = None From 2b3db821b0b6540600e22265766450c85e4911b8 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 23 Jan 2025 18:19:52 +0000 Subject: [PATCH 33/47] Bugfix: New users should use default settings --- assemblyline_ui/helper/user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 41b67bb2..727c5e08 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -285,12 +285,13 @@ def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - settings = STORAGE.user_settings.get_if_exists(user['uname']).as_primitives(strip_null=True) + settings = STORAGE.user_settings.get_if_exists(user['uname']) srv_list = [x for x in SERVICE_LIST if x['enabled']] if not settings: def_srv_list = None settings = default_settings else: + settings = settings.as_primitives(strip_null=True) # Make sure all defaults are there for key, item in default_settings.items(): if key not in settings: From fbfd2fed02c54c5bd9881b58f538c30e4e041470 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 1 Feb 2025 07:50:38 +0000 Subject: [PATCH 34/47] Update APIs to handle changes to submission profiles --- assemblyline_ui/api/v4/user.py | 29 ++++---- assemblyline_ui/helper/service.py | 12 ++-- assemblyline_ui/helper/submission.py | 30 ++++++-- assemblyline_ui/helper/user.py | 103 +++++++++++++++------------ 4 files changed, 103 insertions(+), 71 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index f2642847..69889569 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -228,8 +228,12 @@ def who_am_i(**kwargs): for name, profile in SUBMISSION_PROFILES.items(): if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): # We want to pass forward the configurations that have been explicitly set as a configuration - submission_profiles[name] = profile.params.as_primitives(strip_null=True) - submission_profiles[name]["editable_params"] = profile.editable_params + submission_profiles[name] = profile.as_primitives(strip_null=True) + + # Remove unwanted/redundant data from being shared + for field in ['name', 'classification']: + submission_profiles[name].pop(field, None) + # Expand service categories if used in submission profiles (assists with the UI locking down service selection) service_categories = list(STORAGE.service.facet('category').keys()) @@ -935,17 +939,14 @@ def get_user_settings(username, **kwargs): Result example: { - "classification": "", # Default classification for this user sumbissions - "description": "", # Default description for this user's submissions - "download_encoding": "blah", # Default encoding for downloaded files - "expand_min_score": 100, # Default minimum score to auto-expand sections - "priority": 1000, # Default submission priority - "service_spec": [], # Default Service specific parameters - "ignore_cache": true, # Should file be reprocessed even if there are cached results - "groups": [ ... ], # Default groups selection for the user scans - "ttl": 30, # Default time to live in days of the users submissions - "services": [ ... ], # Default list of selected services - "ignore_filtering": false # Should filtering services by ignored? + "default_external_sources": [], # Default file sources for this user + "default_zip_password": "infected", # Default password for password protected ZIP + "download_encoding": "blah", # Default encoding for downloaded files, + "executive_summary": false, # Should the executive summary be shown by default + "expand_min_score": 100, # Default minimum score to auto-expand sections + "preferred_submission_profile": "default", # Default submission profile + "submission_profiles": [], # List of submission profiles + "submission_view": "report", # Default submission view } """ user = kwargs['user'] @@ -1001,7 +1002,7 @@ def set_user_settings(username, **kwargs): # Changing your own settings if username == user['uname']: - if ROLES.administration not in user['roles'] or ROLES.self_manage not in user['roles']: + if ROLES.administration not in user['roles'] and ROLES.self_manage not in user['roles']: raise AccessDeniedException("You are not allowed to modify your own settings.") edit_user = user diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 5a1a279a..ab269a9a 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -10,16 +10,16 @@ SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) -def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): +def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED, + include_default=False): out = {} + if include_default: + out['default'] = user_default_values.get('default', {}) + for profile in SUBMISSION_PROFILES.values(): if CLASSIFICATION.is_accessible(classification, profile.classification): - profile_values = copy(profile.as_primitives(strip_null=True)) + profile_values = copy(profile.params.as_primitives(strip_null=True)) out[profile.name] = recursive_update(profile_values, user_default_values.get(profile.name, {})) - - out[profile.name].pop("name") - out[profile.name].pop("classification") - return out diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 357d519d..512c399e 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -14,8 +14,9 @@ from assemblyline.common.isotime import now_as_iso from assemblyline.common.str_utils import safe_str from assemblyline.common.iprange import is_ip_reserved -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, SubmissionProfile from assemblyline.odm.messages.submission import SubmissionMessage + from assemblyline.odm.models.user import ROLES from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES @@ -52,6 +53,26 @@ class InvalidUrlException(Exception): class ForbiddenLocation(Exception): pass +def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submission_customize=False) -> dict: + validated_profile = profile.params.as_primitives(strip_null=True) + + for param_type, list_of_params in profile.editable_params.items(): + if param_type == "submission": + # Submission-level parameters + for p in list(updates.keys()): + if p not in ['services', 'service_spec'] and \ + (p not in list_of_params and not submission_customize): + # Submission parameter isn't allowed to be modified based on profile configuration + updates.pop(p) + else: + # Service-level parameters + service_spec = updates.get('service_spec', {}).get(param_type, {}) + for key in list(service_spec.keys()): + if key not in list_of_params and not submission_customize: + # Service parameter isn't allowed to be changed + service_spec.pop(key) + + return recursive_update(validated_profile, updates) def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: dict, out_file: str, default_external_sources: List[str]): @@ -185,8 +206,9 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di def update_submission_parameters(s_params: dict, data: dict, user: dict): s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) + submission_customize = ROLES.submission_customize in user['roles'] # Apply provided params (if the user is allowed to) - if ROLES.submission_customize in user['roles']: + if submission_customize: s_params.update(data.get("params", {})) elif s_profile: if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification): @@ -194,14 +216,12 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params = recursive_update(s_params, data.get("params", {})) - s_params = recursive_update(s_params, s_profile.params.as_primitives(strip_null=True)) + s_params = apply_changes_to_profile(s_params, s_profile, submission_customize) else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") - - def refang_url(url): ''' Refangs a url of text. Based on source of: https://pypi.org/project/defang/ diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 727c5e08..c5877b6b 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,9 +1,9 @@ from typing import Optional from assemblyline.common.str_utils import safe_str -from assemblyline.odm.models.config import SubmissionProfileParams +from assemblyline.odm.models.config import SubmissionProfileParams, SubmissionProfile from assemblyline.odm.models.user import ROLES, User, load_roles -from assemblyline.odm.models.user_settings import UserSettings +from assemblyline.odm.models.user_settings import UserSettings, DEFAULT_USER_PROFILE_SETTINGS from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER from assemblyline_ui.config import CLASSIFICATION as Classification from assemblyline_ui.config import ( @@ -23,6 +23,7 @@ get_default_submission_profiles, simplify_services, ) +from assemblyline_ui.helper.submission import apply_changes_to_profile from assemblyline_ui.http_exceptions import AccessDeniedException, AuthenticationException, InvalidDataException from flask import session as flsk_session @@ -274,19 +275,20 @@ def get_dynamic_classification(current_c12n, user_info): return new_c12n -def get_default_user_settings(user): - return UserSettings({"classification": Classification.default_user_classification(user), - "ttl": config.submission.dtl, - "default_zip_password": DEFAULT_ZIP_PASSWORD, - "download_encoding": DOWNLOAD_ENCODING}).as_primitives() +def get_default_user_settings(user: dict) -> dict: + settings = DEFAULT_USER_PROFILE_SETTINGS + settings.update({"classification": Classification.default_user_classification(user), + "ttl": config.submission.dtl, + "default_zip_password": DEFAULT_ZIP_PASSWORD, + "download_encoding": DOWNLOAD_ENCODING}) + return UserSettings(settings).as_primitives() def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - + submission_customize = ROLES.submission_customize in user['roles'] settings = STORAGE.user_settings.get_if_exists(user['uname']) - srv_list = [x for x in SERVICE_LIST if x['enabled']] if not settings: def_srv_list = None settings = default_settings @@ -304,18 +306,31 @@ def load_user_settings(user): def_srv_list = settings.get('services', {}).get('selected', None) + srv_list = [x for x in SERVICE_LIST if x['enabled']] + settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) + # Normalize the user's classification + settings['classification'] = Classification.normalize_classification(settings['classification']) + + # Check if the user has instantiated their default submission profile + if submission_customize and not settings['submission_profiles'].get('default'): + settings['submission_profiles']['default'] = SubmissionProfileParams({key: value for key, value in settings.items() if key in SubmissionProfileParams.fields()}).as_primitives() + # Only display services that a user is allowed to see settings['service_spec'] = get_default_service_spec(srv_list, settings.get('service_spec', {}), user_classfication) settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], - user_classfication) - settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) + user_classfication, include_default=submission_customize) - # Normalize the user's classification - settings['classification'] = Classification.normalize_classification(settings['classification']) - if settings.get('preferred_submission_profile', None) not in list(settings['submission_profiles'].keys()): - settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] + # Check if the user has a preferred submission profile + if not settings.get('preferred_submission_profile'): + # No preferred submission profile, set one based on the user's roles + if submission_customize: + # User can customize their submission, set the preferred profile to the legacy default + settings['preferred_submission_profile'] = 'default' + else: + # User cannot customize their submission, set the preferred profile to first one on the list + settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] return settings @@ -330,7 +345,7 @@ def save_user_settings(user, data): if key in data and key not in ["services", "service_spec", "submission_profiles"]: out[key] = data.get(key, None) - out["services"] = {'selected': simplify_services(data["services"])} + out["services"] = {'selected': simplify_services(data.get("services", []))} classification = user.get("classification", None) submission_customize = ROLES.submission_customize in user['roles'] @@ -338,38 +353,34 @@ def save_user_settings(user, data): srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] srv_list = list(set(srv_list)) - if data.get('preferred_submission_profile', None) not in SUBMISSION_PROFILES.keys(): - out['preferred_submission_profile'] = SUBMISSION_PROFILES.keys()[0] - else: - out['preferred_submission_profile'] = data['preferred_submission_profile'] + accessible_profiles = [name for name, profile in SUBMISSION_PROFILES.items() \ + if Classification.is_accessible(classification, profile.classification)] + + # Check submission profile preference selection + preferred_submission_profile = data.get('preferred_submission_profile', None) + if submission_customize: + # User is allowed to customize their own default profile + accessible_profiles += ['default'] + + if preferred_submission_profile in accessible_profiles: + out['preferred_submission_profile'] = preferred_submission_profile submission_profiles = {} - for name, profile in SUBMISSION_PROFILES.items(): - if Classification.is_accessible(classification, profile.classification): - user_params = data.get('submission_profiles', {}).get(profile.name, {}).get('params', {}) - - # Applying the submission params - profile_defaults = SubmissionProfileParams().as_primitives() - default_keys = profile["editable_params"].get("submit", []) - for key in profile_defaults.keys(): - if key in user_params and key not in ["services", "service_spec"] and \ - (submission_customize or key in default_keys): - profile["params"][key] = user_params.get(key, None) - - # Applying the selected services - if submission_customize: - profile["params"]["services"]["selected"] = [x for x in user_params.get( - 'services', {}).get('selected', []) if x in srv_list] - - # Applying the service specs - profile["params"]["service_spec"] = {} - for svr_name, spec in user_params.get("service_spec", {}).items(): - for p_name, p_value in spec.items(): - if (p_name in profile["editable_params"].get(svr_name, []) or submission_customize) and \ - svr_name in srv_list: - profile["params"]["service_spec"].setdefault(svr_name, {}).setdefault(p_name, p_value) - - submission_profiles[name] = profile.params.as_primitives(strip_null=True) + for name in accessible_profiles: + user_params = data.get('submission_profiles', {}).get(name, {}) + profile_config: Optional[SubmissionProfile] = SUBMISSION_PROFILES.get(name) + + if profile_config == None: + if name == "default": + # There is no restriction on what you can set for your default submission profile + # Set profile based on preferences set at the root-level + data["services"] = out['services'] + submission_profiles[name] = SubmissionProfileParams({key: value for key, value in data.items() + if key in SubmissionProfileParams.fields()}).as_primitives() + + else: + # Apply changes to the profile relative to what's allowed to be changed based on configuration + submission_profiles[name] = apply_changes_to_profile(profile_config, user_params, submission_customize) out["submission_profiles"] = submission_profiles From 5e819da95d0dbfb40c16b2907afb408ef807a35d Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 4 Feb 2025 16:29:44 +0000 Subject: [PATCH 35/47] Remove unused imports --- assemblyline_ui/api/v4/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 2de06863..1ab61ee1 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -18,7 +18,7 @@ from assemblyline_core.submission_client import SubmissionClient, SubmissionException from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint from assemblyline_ui.config import ARCHIVESTORE, STORAGE, TEMP_SUBMIT_DIR, FILESTORE, config, \ - CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER, SUBMISSION_PROFILES, USER_CONFIGURABLE_SUBMISSION_PARAMS + CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ FETCH_METHODS, URL_GENERATORS, update_submission_parameters From 00b42084c1a544c0823f8f7e311ed55cf535b248 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:01:56 +0000 Subject: [PATCH 36/47] Fix pre-existing tests --- assemblyline_ui/api/v4/ingest.py | 2 +- assemblyline_ui/api/v4/submit.py | 2 +- assemblyline_ui/api/v4/ui.py | 12 ++++++------ assemblyline_ui/helper/submission.py | 8 ++++++-- test/test_ingest.py | 8 ++++---- test/test_submit.py | 14 ++++++++------ test/test_user.py | 7 +++++-- 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index eb9bc63f..d23a338b 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -275,7 +275,7 @@ def ingest_single_file(**kwargs): # Update submission parameters as specified by the user try: - update_submission_parameters(s_params, data, user) + s_params = update_submission_parameters(s_params, data, user) except Exception as e: return make_api_response({}, str(e), 400) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 1ab61ee1..fabd143f 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -345,7 +345,7 @@ def submit(**kwargs): # Update submission parameters as specified by the user try: - update_submission_parameters(s_params, data, user) + s_params = update_submission_parameters(s_params, data, user) except Exception as e: return make_api_response({}, str(e), 400) diff --git a/assemblyline_ui/api/v4/ui.py b/assemblyline_ui/api/v4/ui.py index e0dc1a7b..2806f67d 100644 --- a/assemblyline_ui/api/v4/ui.py +++ b/assemblyline_ui/api/v4/ui.py @@ -252,21 +252,21 @@ def start_ui_submission(ui_sid, **kwargs): return make_api_response("", err=str(e), status_code=400) return make_api_response({"started": True, "sid": submission['sid']}) - allow_description_overwrite = False - if not ui_params['description']: - allow_description_overwrite = True - ui_params['description'] = f"Inspection of file: {fname}" - # Submit to dispatcher try: params = ui_to_submission_params(ui_params) # Update submission parameters as specified by the user try: - update_submission_parameters(params, ui_params, user) + params = update_submission_parameters(params, ui_params, user) except Exception as e: return make_api_response({}, str(e), 400) + allow_description_overwrite = False + if not ui_params['description']: + allow_description_overwrite = True + ui_params['description'] = f"Inspection of file: {fname}" + metadata = params.pop("metadata", {}) # Enforce maximum DTL diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 512c399e..8e38bd44 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -204,7 +204,7 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di return found, fileinfo -def update_submission_parameters(s_params: dict, data: dict, user: dict): +def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict: s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) submission_customize = ROLES.submission_customize in user['roles'] # Apply provided params (if the user is allowed to) @@ -216,11 +216,15 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict): raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params = recursive_update(s_params, data.get("params", {})) - s_params = apply_changes_to_profile(s_params, s_profile, submission_customize) + s_params = apply_changes_to_profile(s_profile, s_params, submission_customize) else: # No profile specified, raise an exception back to the user raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") + # Ensure the description key exists in the resulting submission params + s_params.setdefault("description", "") + return s_params + def refang_url(url): ''' diff --git a/test/test_ingest.py b/test/test_ingest.py index 785d9a50..c321f754 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -362,8 +362,9 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): iq.delete() # Make the user a simple user and try to submit - datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'admin'), - (datastore.user.UPDATE_APPEND, 'type', 'user')]) + datastore.user.update('admin', [ + (datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'roles', 'submission_create')]) byte_str = get_random_phrase(wmin=30, wmax=75).encode() sha256 = hashlib.sha256(byte_str).hexdigest() data = { @@ -387,5 +388,4 @@ def test_ingest_submission_profile(datastore, login_session, scheduler): get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) # Restore original roles for later tests - datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), - (datastore.user.UPDATE_APPEND, 'type', 'admin')]) + datastore.user.update('admin', [(datastore.user.UPDATE_APPEND, 'type', 'admin'),]) diff --git a/test/test_submit.py b/test/test_submit.py index 47c71c54..f54c3d81 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -10,7 +10,7 @@ from assemblyline.common import forge from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES -from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions +from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions, create_services, wipe_services from assemblyline.odm.randomizer import get_random_phrase from assemblyline.remote.datatypes.queues.named import NamedQueue from assemblyline_core.dispatching.dispatcher import SubmissionTask @@ -27,10 +27,12 @@ def datastore(datastore_connection, filestore): try: create_users(datastore_connection) submission = create_submission(datastore_connection, filestore) + create_services(datastore_connection) yield datastore_connection finally: wipe_users(datastore_connection) wipe_submissions(datastore_connection, filestore) + wipe_services(datastore_connection) sq.delete() @@ -288,8 +290,9 @@ def test_submit_submission_profile(datastore, login_session, scheduler): sq.delete() # Make the user a simple user and try to submit - datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'admin'), - (datastore.user.UPDATE_APPEND, 'type', 'user')]) + datastore.user.update('admin', [ + (datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'roles', 'submission_create')]) byte_str = get_random_phrase(wmin=30, wmax=75).encode() sha256 = hashlib.sha256(byte_str).hexdigest() data = { @@ -311,9 +314,8 @@ def test_submit_submission_profile(datastore, login_session, scheduler): # But also try setting a parameter that you are allowed to set data['params'] = {'deep_scan': True} resp = get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) - assert resp['params']['services']['selected'] == profile['params']['services']['selected'] + assert set(resp['params']['services']['selected']) == set(profile['params']['services']['selected']) assert resp['params']['deep_scan'] == True # Restore original roles for later tests - datastore.user.update('admin', [(datastore.user.UPDATE_REMOVE, 'type', 'user'), - (datastore.user.UPDATE_APPEND, 'type', 'admin')]) + datastore.user.update('admin', [(datastore.user.UPDATE_APPEND, 'type', 'admin'),]) diff --git a/test/test_user.py b/test/test_user.py index 6e3ef87e..69d3b4c2 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -8,6 +8,7 @@ from assemblyline_ui.helper.user import load_user_settings from assemblyline.odm.models.user import User from assemblyline.odm.models.user_favorites import Favorite, UserFavorites +from assemblyline.odm.models.user_settings import UserSettings from assemblyline.odm.randomizer import random_model_obj from assemblyline.odm.random_data import create_users, wipe_users @@ -48,6 +49,7 @@ def datastore(datastore_connection): ds.user.save(u.uname, u) ds.user_favorites.save(u.uname, data) ds.user_avatar.save(u.uname, AVATAR) + ds.user_settings.save(u.uname, random_model_obj(UserSettings)) user_list.append(u.uname) yield ds @@ -279,7 +281,8 @@ def test_set_user_settings(datastore, login_session): _, session, host = login_session username = random.choice(user_list) - uset = load_user_settings({'uname': username}) + user = datastore.user.get(username, as_obj=False) + uset = load_user_settings(user) uset['expand_min_score'] = 111 uset['priority'] = 111 @@ -287,4 +290,4 @@ def test_set_user_settings(datastore, login_session): assert resp['success'] datastore.user_settings.commit() - assert uset == load_user_settings({'uname': username}) + assert uset == load_user_settings(user) From d91a1122b5331c09fddcca6f61a3f9c69d22979c Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 5 Feb 2025 20:40:44 +0000 Subject: [PATCH 37/47] Update testing for setting user's settings --- assemblyline_ui/api/v4/user.py | 2 ++ assemblyline_ui/helper/submission.py | 10 +++++-- test/test_user.py | 43 ++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index aded3f81..004c5e88 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -1024,6 +1024,8 @@ def set_user_settings(username, **kwargs): return make_api_response({"success": True}) else: return make_api_response({"success": False}, "Failed to save user's settings", 500) + except PermissionError as e: + return make_api_response({"success": False}, str(e), 401) except ValueError as e: return make_api_response({"success": False}, str(e), 400) diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 8e38bd44..8a332b3c 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -56,6 +56,12 @@ class ForbiddenLocation(Exception): def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submission_customize=False) -> dict: validated_profile = profile.params.as_primitives(strip_null=True) + # Check to see if user is trying to modify the service params of a service not explicitly defined in the profile + unrecognized_services = set(updates.get('service_spec', {}).keys()) - set(list(profile.editable_params.keys())) + if unrecognized_services and not submission_customize: + # User isn't allowed to change/set parameters of services not explicitly defined in the profile + raise PermissionError(f"User isn't allowed to modify the following services: {list(unrecognized_services)}") + for param_type, list_of_params in profile.editable_params.items(): if param_type == "submission": # Submission-level parameters @@ -63,14 +69,14 @@ def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submissi if p not in ['services', 'service_spec'] and \ (p not in list_of_params and not submission_customize): # Submission parameter isn't allowed to be modified based on profile configuration - updates.pop(p) + raise PermissionError(f"User isn't allowed to modify the \"{p}\" parameter of {profile.display_name} profile") else: # Service-level parameters service_spec = updates.get('service_spec', {}).get(param_type, {}) for key in list(service_spec.keys()): if key not in list_of_params and not submission_customize: # Service parameter isn't allowed to be changed - service_spec.pop(key) + raise PermissionError(f"User isn't allowed to modify the \"{p}\" parameter of \"{param_type}\" service in \"{profile.display_name}\" profile") return recursive_update(validated_profile, updates) diff --git a/test/test_user.py b/test/test_user.py index 69d3b4c2..5eff5294 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -277,17 +277,44 @@ def test_set_user_favorites(datastore, login_session): # noinspection PyUnusedLocal -def test_set_user_settings(datastore, login_session): +@pytest.mark.parametrize("allow_submission_customize", [True, False], ids=["submission_customize=true", "submission_customize=false"]) +def test_set_user_settings(datastore, login_session, allow_submission_customize): _, session, host = login_session username = random.choice(user_list) - user = datastore.user.get(username, as_obj=False) + + if allow_submission_customize: + # User is allowed to customize their submission profiles + datastore.user.update(username, [(datastore.user.UPDATE_APPEND, 'roles', 'submission_customize')]) + if 'submission_customize' not in user['roles']: + user['roles'].append('submission_customize') + else: + # Users that aren't allow to customize submissions shouldn't be able to customize submission profiles parameters if the configuration doesn't allow it + datastore.user.update(username, [(datastore.user.UPDATE_REMOVE, 'roles', 'submission_customize')]) + if 'submission_customize' in user['roles']: + user['roles'].remove('submission_customize') + uset = load_user_settings(user) uset['expand_min_score'] = 111 uset['priority'] = 111 - - resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset)) - assert resp['success'] - - datastore.user_settings.commit() - assert uset == load_user_settings(user) + uset['submission_profiles']['static']['service_spec'] = { + "test": { + "p": True + } + } + + if allow_submission_customize: + # User is allowed to customize their submission profiles as they see fit + resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset)) + assert resp['success'] + + datastore.user_settings.commit() + new_user_settings = load_user_settings(user) + + # Ensure the changes are applied in the right places + assert new_user_settings['submission_profiles']['default']['priority'] == uset['priority'] + assert new_user_settings['submission_profiles']['static']['service_spec'] == uset['submission_profiles']['static']['service_spec'] + else: + with pytest.raises(APIError): + # User isn't allowed to customize their submission profiles, API should return an exception + resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset)) From cc4ea6efd704e257a1f15e2399781cfd9885dbf2 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 5 Feb 2025 21:08:05 +0000 Subject: [PATCH 38/47] More fixes --- assemblyline_ui/api/v4/ingest.py | 32 +++++++++++++++++----------- assemblyline_ui/api/v4/submit.py | 5 ++++- assemblyline_ui/helper/submission.py | 4 ++++ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index d23a338b..8df7584e 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -38,6 +38,16 @@ port=config.core.redis.persistent.port) MAX_SIZE = config.submission.max_file_size +DEFAULT_INGEST_PARAMS = { + 'deep_scan': False, + "priority": 150, + "ignore_cache": False, + # the following one line can be removed after assemblyline 4.6+ + "ignore_dynamic_recursion_prevention": False, + "ignore_recursion_prevention": False, + "ignore_filtering": False, + "type": "INGEST" +} # noinspection PyUnusedLocal @ingest_api.route("/get_message//", methods=["GET"]) @@ -259,19 +269,10 @@ def ingest_single_file(**kwargs): default_external_sources = user_settings.pop('default_external_sources', []) # Load default user params from user settings - s_params = ui_to_submission_params(user_settings) - - # Reset dangerous user settings to safe values - s_params.update({ - 'deep_scan': False, - "priority": 150, - "ignore_cache": False, - # the following one line can be removed after assemblyline 4.6+ - "ignore_dynamic_recursion_prevention": False, - "ignore_recursion_prevention": False, - "ignore_filtering": False, - "type": "INGEST" - }) + if ROLES.submission_customize in user['roles']: + s_params = ui_to_submission_params(user_settings) + else: + s_params = {} # Update submission parameters as specified by the user try: @@ -279,6 +280,11 @@ def ingest_single_file(**kwargs): except Exception as e: return make_api_response({}, str(e), 400) + # Set any dangerous user settings to safe values (if wasn't set in request) + for k, v in DEFAULT_INGEST_PARAMS.items(): + if k not in s_params: + s_params[k] = v + # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index fabd143f..7b08f0f8 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -341,7 +341,10 @@ def submit(**kwargs): default_external_sources = user_settings.pop('default_external_sources', []) # Create task object - s_params = ui_to_submission_params(user_settings) + if ROLES.submission_customize in user['roles']: + s_params = ui_to_submission_params(user_settings) + else: + s_params = {} # Update submission parameters as specified by the user try: diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 8a332b3c..2358d656 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -213,6 +213,10 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict: s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) submission_customize = ROLES.submission_customize in user['roles'] + + # Ensure classification is set based on the user before applying updates + s_params.setdefault("classification", user['classification']) + # Apply provided params (if the user is allowed to) if submission_customize: s_params.update(data.get("params", {})) From 64c9b8f1297f659647debfaf73e94ec152927b0c Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Wed, 5 Feb 2025 21:47:28 +0000 Subject: [PATCH 39/47] Add testing for submission profiles with a preset ingestion type for metadata validation --- assemblyline_ui/helper/submission.py | 9 +++++---- test/config/config.yml | 6 ++++++ test/test_ingest.py | 14 +++++++++++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 2358d656..16003dfd 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -220,16 +220,17 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict # Apply provided params (if the user is allowed to) if submission_customize: s_params.update(data.get("params", {})) - elif s_profile: + elif not s_profile: + # No profile specified, raise an exception back to the user + raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") + + if s_profile: if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification): # User isn't allowed to use the submission profile specified raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params = recursive_update(s_params, data.get("params", {})) s_params = apply_changes_to_profile(s_profile, s_params, submission_customize) - else: - # No profile specified, raise an exception back to the user - raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") # Ensure the description key exists in the resulting submission params s_params.setdefault("description", "") diff --git a/test/config/config.yml b/test/config/config.yml index 66e52072..e1d24619 100644 --- a/test/config/config.yml +++ b/test/config/config.yml @@ -45,6 +45,12 @@ submission: validator_type: keyword strict_schemes: - strict_ingest + profiles: + # Profile that will enforce metadata validation + - name: "static" + display_name: "Static Analysis" + params: + type: strict_ingest ui: enforce_quota: false diff --git a/test/test_ingest.py b/test/test_ingest.py index c321f754..f2b9b249 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -293,7 +293,6 @@ def test_ingest_metadata_validation(datastore, login_session): 'base64': base64.b64encode(byte_str).decode('ascii'), 'metadata': {'test': 'ingest_base64_nameless'}, "params": {'type': 'strict_ingest'} - } resp = get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) assert isinstance(resp['ingest_id'], str) @@ -309,6 +308,19 @@ def test_ingest_metadata_validation(datastore, login_session): resp = get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) assert isinstance(resp['ingest_id'], str) + # Test submitting with a submission profile that has the ingest type preset + # With currently set metadata, this should raise an API error + with pytest.raises(APIError, match="Extra metadata found from submission"): + data.pop('params') + data['submission_profile'] = "static" + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Fix metadata and resubmit (still using a submission profile) + data['metadata'].pop('blah') + resp = get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + assert isinstance(resp['ingest_id'], str) + + # noinspection PyUnusedLocal def test_get_message(datastore, login_session): From 7c11f9aa7a00edc0446cfac40c9b57c3b316e949 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 6 Feb 2025 01:55:10 +0000 Subject: [PATCH 40/47] Add profile to testing pipeline configuration --- pipelines/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pipelines/config.yml b/pipelines/config.yml index cfa82dfc..7a57fde4 100644 --- a/pipelines/config.yml +++ b/pipelines/config.yml @@ -41,6 +41,12 @@ submission: validator_type: keyword strict_schemes: - strict_ingest + profiles: + # Profile that will enforce metadata validation + - name: "static" + display_name: "Static Analysis" + params: + type: strict_ingest logging: log_level: DISABLED From 42018c7caaf247e0bdb4df29095dc6006b97c055 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 6 Feb 2025 02:24:37 +0000 Subject: [PATCH 41/47] Retry random testing errors with Badlist --- assemblyline_ui/api/v4/badlist.py | 2 +- test/test_badlist.py | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/assemblyline_ui/api/v4/badlist.py b/assemblyline_ui/api/v4/badlist.py index ea396b2e..9efae686 100644 --- a/assemblyline_ui/api/v4/badlist.py +++ b/assemblyline_ui/api/v4/badlist.py @@ -720,7 +720,7 @@ def remove_attribution(qhash, attrib_type, value, **_): while True: current_badlist, version = STORAGE.badlist.get_if_exists(qhash, as_obj=False, version=True) if not current_badlist: - return make_api_response({}, "The badlist ietm your are trying to modify does not exists", 404) + return make_api_response({}, "The badlist item you are trying to modify does not exists", 404) if 'attribution' not in current_badlist: return make_api_response({'success': False}) diff --git a/test/test_badlist.py b/test/test_badlist.py index 618c889e..2338ff27 100644 --- a/test/test_badlist.py +++ b/test/test_badlist.py @@ -354,14 +354,22 @@ def test_badlist_delete_hash(datastore, login_session): def test_badlist_attribution(datastore, login_session): _, session, host = login_session + badlist_items = datastore.badlist.search("attribution.actor:*", + fl="id,attribution.actor", rows=100, as_obj=False)['items'] + while True: + item = random.choice(badlist_items) + + actor = random.choice(item['attribution']['actor']) + hash = item['id'] + + try: + # Test removing attribution from the hash in the Badlist + resp = get_api_data(session, f"{host}/api/v4/badlist/attribution/{hash}/actor/{actor}/", method="DELETE") + break + except APIError: + # TODO: Investigate why in some cases the API thinks the hash doesn't exist + pass - item = random.choice(datastore.badlist.search("attribution.actor:*", - fl="id,attribution.actor", rows=100, as_obj=False)['items']) - - actor = random.choice(item['attribution']['actor']) - hash = item['id'] - - resp = get_api_data(session, f"{host}/api/v4/badlist/attribution/{hash}/actor/{actor}/", method="DELETE") assert resp['success'] assert actor not in datastore.badlist.get_if_exists(hash).attribution.actor From c8e50bd9cd5b6eb457a3e9816459e4063e1f9b34 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 7 Feb 2025 03:11:52 +0000 Subject: [PATCH 42/47] Patch save_user_settings API to account for actual changes made to submission profiles --- assemblyline_ui/helper/user.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index c5877b6b..10cbbac1 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,6 +1,7 @@ from typing import Optional from assemblyline.common.str_utils import safe_str +from assemblyline.common.dict_utils import recursive_update, get_recursive_delta from assemblyline.odm.models.config import SubmissionProfileParams, SubmissionProfile from assemblyline.odm.models.user import ROLES, User, load_roles from assemblyline.odm.models.user_settings import UserSettings, DEFAULT_USER_PROFILE_SETTINGS @@ -340,12 +341,17 @@ def save_user_settings(user, data): if username == None: raise Exception("Invalid username") - out = STORAGE.user_settings.get(username).as_primitives() - for key in out.keys(): + user_settings = STORAGE.user_settings.get(username) + if user_settings: + user_settings = user_settings.as_primitives() + else: + user_settings = {} + + for key in user_settings.keys(): if key in data and key not in ["services", "service_spec", "submission_profiles"]: - out[key] = data.get(key, None) + user_settings[key] = data.get(key, None) - out["services"] = {'selected': simplify_services(data.get("services", []))} + user_settings["services"] = {'selected': simplify_services(data.get("services", []))} classification = user.get("classification", None) submission_customize = ROLES.submission_customize in user['roles'] @@ -363,7 +369,7 @@ def save_user_settings(user, data): accessible_profiles += ['default'] if preferred_submission_profile in accessible_profiles: - out['preferred_submission_profile'] = preferred_submission_profile + user_settings['preferred_submission_profile'] = preferred_submission_profile submission_profiles = {} for name in accessible_profiles: @@ -374,14 +380,17 @@ def save_user_settings(user, data): if name == "default": # There is no restriction on what you can set for your default submission profile # Set profile based on preferences set at the root-level - data["services"] = out['services'] + data["services"] = user_settings['services'] submission_profiles[name] = SubmissionProfileParams({key: value for key, value in data.items() if key in SubmissionProfileParams.fields()}).as_primitives() else: + # Calculate what the profiles updates are based on default profile settings and the user-submitted changes + profile_updates = get_recursive_delta(DEFAULT_USER_PROFILE_SETTINGS, user_params) + # Apply changes to the profile relative to what's allowed to be changed based on configuration - submission_profiles[name] = apply_changes_to_profile(profile_config, user_params, submission_customize) + submission_profiles[name] = apply_changes_to_profile(profile_config, profile_updates, submission_customize) - out["submission_profiles"] = submission_profiles + user_settings["submission_profiles"] = submission_profiles - return STORAGE.user_settings.save(username, out) + return STORAGE.user_settings.save(username, user_settings) From 9697be51bf3e3e15e3da18cbee52b98421a42a7c Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Sat, 8 Feb 2025 07:04:33 +0000 Subject: [PATCH 43/47] Update API for UI submissions --- assemblyline_ui/api/v4/submit.py | 2 +- assemblyline_ui/helper/submission.py | 18 +++++++++++++----- assemblyline_ui/helper/user.py | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 7b08f0f8..7619b3fd 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -341,7 +341,7 @@ def submit(**kwargs): default_external_sources = user_settings.pop('default_external_sources', []) # Create task object - if ROLES.submission_customize in user['roles']: + if (ROLES.submission_customize in user['roles']) or "ui_params" in data: s_params = ui_to_submission_params(user_settings) else: s_params = {} diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 16003dfd..678ae5b2 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -9,12 +9,13 @@ from typing import List from urllib.parse import urlparse -from assemblyline.common.dict_utils import recursive_update +from assemblyline.common.dict_utils import recursive_update, get_recursive_delta from assemblyline.common.file import make_uri_file from assemblyline.common.isotime import now_as_iso from assemblyline.common.str_utils import safe_str from assemblyline.common.iprange import is_ip_reserved -from assemblyline.odm.models.config import HASH_PATTERN_MAP, SubmissionProfile +from assemblyline.odm.models.config import HASH_PATTERN_MAP, SubmissionProfile, SubmissionProfileParams +from assemblyline.odm.models.user_settings import DEFAULT_USER_PROFILE_SETTINGS from assemblyline.odm.messages.submission import SubmissionMessage from assemblyline.odm.models.user import ROLES @@ -66,8 +67,13 @@ def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submissi if param_type == "submission": # Submission-level parameters for p in list(updates.keys()): - if p not in ['services', 'service_spec'] and \ - (p not in list_of_params and not submission_customize): + if not hasattr(SubmissionProfileParams, p): + # Don't need to be concerned about a parameter that has no limitation ie. description + continue + elif p in ['services', 'service_spec']: + # These will be checked later + continue + elif p not in list_of_params and not submission_customize: # Submission parameter isn't allowed to be modified based on profile configuration raise PermissionError(f"User isn't allowed to modify the \"{p}\" parameter of {profile.display_name} profile") else: @@ -215,7 +221,7 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict submission_customize = ROLES.submission_customize in user['roles'] # Ensure classification is set based on the user before applying updates - s_params.setdefault("classification", user['classification']) + classification = s_params.get("classification", user['classification']) # Apply provided params (if the user is allowed to) if submission_customize: @@ -230,10 +236,12 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") # Apply the profile (but allow the user to change some properties) s_params = recursive_update(s_params, data.get("params", {})) + s_params = get_recursive_delta(DEFAULT_USER_PROFILE_SETTINGS, s_params) s_params = apply_changes_to_profile(s_profile, s_params, submission_customize) # Ensure the description key exists in the resulting submission params s_params.setdefault("description", "") + s_params.setdefault("classification", classification) return s_params diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 10cbbac1..3bd1f6e7 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,7 +1,7 @@ from typing import Optional from assemblyline.common.str_utils import safe_str -from assemblyline.common.dict_utils import recursive_update, get_recursive_delta +from assemblyline.common.dict_utils import get_recursive_delta from assemblyline.odm.models.config import SubmissionProfileParams, SubmissionProfile from assemblyline.odm.models.user import ROLES, User, load_roles from assemblyline.odm.models.user_settings import UserSettings, DEFAULT_USER_PROFILE_SETTINGS From 416f4235ebed0282a84808e90a83c0e96287e55a Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Tue, 11 Feb 2025 21:16:57 +0000 Subject: [PATCH 44/47] Add API for resubmitting file/submission with a submission profile --- assemblyline_ui/api/v4/submit.py | 219 ++++++++++++++++++++----------- test/test_submit.py | 22 +++- 2 files changed, 162 insertions(+), 79 deletions(-) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 7619b3fd..d2669243 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -7,18 +7,18 @@ import tempfile from flask import request +from typing import Tuple, Union from assemblyline.common.constants import MAX_PRIORITY, PRIORITIES from assemblyline.common.dict_utils import flatten from assemblyline.common.str_utils import safe_str from assemblyline.common.uid import get_random_id -from assemblyline.odm.models.config import SubmissionProfile from assemblyline.odm.messages.submission import Submission from assemblyline.odm.models.user import ROLES from assemblyline_core.submission_client import SubmissionClient, SubmissionException -from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint +from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint, Response from assemblyline_ui.config import ARCHIVESTORE, STORAGE, TEMP_SUBMIT_DIR, FILESTORE, config, \ - CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER + CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER, SUBMISSION_PROFILES from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ FETCH_METHODS, URL_GENERATORS, update_submission_parameters @@ -34,28 +34,7 @@ submission_client = SubmissionClient(datastore=STORAGE, filestore=FILESTORE, config=config, identify=IDENTIFY) -# noinspection PyUnusedLocal -@submit_api.route("/dynamic//", methods=["GET"]) -@api_login(allow_readonly=False, require_role=[ROLES.submission_create], count_toward_quota=False) -def resubmit_for_dynamic(sha256, *args, **kwargs): - """ - Resubmit a file for dynamic analysis - - Variables: - sha256 => Resource locator (SHA256) - - Arguments (Optional): - copy_sid => Mimic the attributes of this SID. - name => Name of the file for the submission - - Data Block: - None - - Result example: - # Submission message object as a json dictionary - """ - user = kwargs['user'] - +def create_resubmission_task(sha256: str, user: dict, copy_sid: str = None, name: str = None, profile: str = None, **kwargs) ->Union[Tuple[Submission, int] | Response]: # Check if we've reached the quotas quota_error = check_submission_quota(user) if quota_error: @@ -69,58 +48,70 @@ def resubmit_for_dynamic(sha256, *args, **kwargs): if not Classification.is_accessible(user['classification'], file_info['classification']): return make_api_response("", "You are not allowed to re-submit a file that you don't have access to", 403) - submit_result = None metadata = {} - try: - copy_sid = request.args.get('copy_sid', None) - if copy_sid: - submission = STORAGE.submission.get(copy_sid, as_obj=False) - else: - submission = None - - if submission: - if not Classification.is_accessible(user['classification'], submission['classification']): - return make_api_response("", - "You are not allowed to re-submit a submission that you don't have access to", - 403) - - submission_params = submission['params'] - submission_params['classification'] = submission['classification'] - expiry = submission['expiry_ts'] - metadata = submission['metadata'] + copy_sid = request.args.get('copy_sid', None) + if copy_sid: + submission = STORAGE.submission.get(copy_sid, as_obj=False) + else: + submission = None + + if submission: + if not Classification.is_accessible(user['classification'], submission['classification']): + return make_api_response("", + "You are not allowed to re-submit a submission that you don't have access to", + 403) + + submission_params = submission['params'] + submission_params['classification'] = submission['classification'] + expiry = submission['expiry_ts'] + metadata = submission['metadata'] + + else: + submission_params = ui_to_submission_params(load_user_settings(user)) + submission_params['classification'] = file_info['classification'] + expiry = file_info['expiry_ts'] + + # Ignore external sources + submission_params.pop('default_external_sources', None) + + if not FILESTORE.exists(sha256): + if ARCHIVESTORE and ARCHIVESTORE != FILESTORE and \ + ROLES.archive_download in user['roles'] and ARCHIVESTORE.exists(sha256): + + # File exists in the archivestore, copying it to the filestore + with tempfile.NamedTemporaryFile() as buf: + ARCHIVESTORE.download(sha256, buf.name) + FILESTORE.upload(buf.name, sha256, location='far') else: - submission_params = ui_to_submission_params(load_user_settings(user)) - submission_params['classification'] = file_info['classification'] - expiry = file_info['expiry_ts'] + return make_api_response({}, "File %s cannot be found on the server therefore it cannot be resubmitted." + % sha256, status_code=404) - # Ignore external sources - submission_params.pop('default_external_sources', None) - if not FILESTORE.exists(sha256): - if ARCHIVESTORE and ARCHIVESTORE != FILESTORE and \ - ROLES.archive_download in user['roles'] and ARCHIVESTORE.exists(sha256): + if (file_info["type"].startswith("uri/") and "uri_info" in file_info and "uri" in file_info["uri_info"]): + name = safe_str(file_info["uri_info"]["uri"]) + else: + name = safe_str(request.args.get('name', sha256)) + description_prefix = f"Resubmit {name}" - # File exists in the archivestore, copying it to the filestore - with tempfile.NamedTemporaryFile() as buf: - ARCHIVESTORE.download(sha256, buf.name) - FILESTORE.upload(buf.name, sha256, location='far') + files = [{'name': name, 'sha256': sha256, 'size': file_info['size']}] - else: - return make_api_response({}, "File %s cannot be found on the server therefore it cannot be resubmitted." - % sha256, status_code=404) - - if (file_info["type"].startswith("uri/") and "uri_info" in file_info and "uri" in file_info["uri_info"]): - name = safe_str(file_info["uri_info"]["uri"]) - submission_params['description'] = f"Resubmit {file_info['uri_info']['uri']} for Dynamic Analysis" + if profile: + # Obtain any settings from the user and apply them to the submission + user_settings = STORAGE.user_settings.get(user['uname'], as_obj=False) + if user_settings: + # Reuse existing settings for specified profile + profile_params = user_settings['submission_profiles'].get(profile, {}) else: - name = safe_str(request.args.get('name', sha256)) - submission_params['description'] = f"Resubmit {name} for Dynamic Analysis" + # Otherwise default to what's set for the profile at the configuration-level + profile_params = {} + profile_params['submission_profile'] = profile - files = [{'name': name, 'sha256': sha256, 'size': file_info['size']}] + submission_params = update_submission_parameters(submission_params, profile_params, user) + submission_params['description'] = f"{description_prefix} with {SUBMISSION_PROFILES[profile].display_name}" - submission_params['submitter'] = user['uname'] - submission_params['quota_item'] = True + else: + # Only append Dynamic Analysis as a selected service and set the priority if 'priority' not in submission_params: submission_params['priority'] = 500 if "Dynamic Analysis" not in submission_params['services']['selected']: @@ -128,20 +119,92 @@ def resubmit_for_dynamic(sha256, *args, **kwargs): # Ensure submission priority stays within the range of user priorities submission_params['priority'] = max(min(submission_params['priority'], MAX_PRIORITY), PRIORITIES['user-low']) + submission_params['description'] = f"{description_prefix} for Dynamic Analysis" - try: - submission_obj = Submission({ - "files": files, - "params": submission_params, - "metadata": metadata, - }) - except (ValueError, KeyError) as e: - return make_api_response("", err=str(e), status_code=400) + submission_params['submitter'] = user['uname'] + submission_params['quota_item'] = True - submit_result = submission_client.submit(submission_obj, expiry=expiry) - submission_received(submission_obj) - return make_api_response(submit_result.as_primitives()) + try: + return Submission({ "files": files, "params": submission_params, "metadata": metadata}), expiry + except (ValueError, KeyError) as e: + return make_api_response("", err=str(e), status_code=400) + +# noinspection PyUnusedLocal +@submit_api.route("///", methods=["GET"]) +@api_login(allow_readonly=False, require_role=[ROLES.submission_create], count_toward_quota=False) +def resubmit_with_profile(profile, sha256, *args, **kwargs): + """ + Resubmit a file using a submission profile + Variables: + profile => Submission profile to be used in new submission + sha256 => Resource locator (SHA256) + + Arguments (Optional): + copy_sid => Mimic the attributes of this SID. + name => Name of the file for the submission + + Data Block: + None + + Result example: + # Submission message object as a json dictionary + """ + user = kwargs['user'] + + submit_result = None + try: + ret_value = create_resubmission_task(sha256=sha256, profile=profile, user=user, **request.args) + if isinstance(ret_value, Response): + # Forward error response back to user + return ret_value + else: + # Otherwise we got submission object with an expiry + submission_obj, expiry = ret_value + submit_result = submission_client.submit(submission_obj, expiry=expiry) + submission_received(submission_obj) + return make_api_response(submit_result.as_primitives()) + except SubmissionException as e: + return make_api_response("", err=str(e), status_code=400) + finally: + if submit_result is None: + # We had an error during the submission, release the quotas for the user + decrement_submission_quota(user) + + +# noinspection PyUnusedLocal +@submit_api.route("/dynamic//", methods=["GET"]) +@api_login(allow_readonly=False, require_role=[ROLES.submission_create], count_toward_quota=False) +def resubmit_for_dynamic(sha256, *args, **kwargs): + """ + Resubmit a file for dynamic analysis + + Variables: + sha256 => Resource locator (SHA256) + + Arguments (Optional): + copy_sid => Mimic the attributes of this SID. + name => Name of the file for the submission + + Data Block: + None + + Result example: + # Submission message object as a json dictionary + """ + user = kwargs['user'] + submit_result = None + try: + ret_value = create_resubmission_task(sha256=sha256, user=user, **request.args) + if isinstance(ret_value, Response): + # Forward error response back to user + return ret_value + else: + # Otherwise we got submission object with an expiry + submission_obj, expiry = ret_value + submit_result = submission_client.submit(submission_obj, expiry=expiry) + submission_received(submission_obj) + return make_api_response(submit_result.as_primitives()) except SubmissionException as e: return make_api_response("", err=str(e), status_code=400) finally: diff --git a/test/test_submit.py b/test/test_submit.py index f54c3d81..912f9da4 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -9,7 +9,7 @@ from conftest import get_api_data, APIError from assemblyline.common import forge -from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES, DEFAULT_SRV_SEL from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions, create_services, wipe_services from assemblyline.odm.randomizer import get_random_phrase from assemblyline.remote.datatypes.queues.named import NamedQueue @@ -51,6 +51,26 @@ def test_resubmit(datastore, login_session, scheduler): msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) assert msg.submission.sid == resp['sid'] +# noinspection PyUnusedLocal +def test_resubmit_profile(datastore, login_session, scheduler): + _, session, host = login_session + + sq.delete() + sha256 = random.choice(submission.results)[:64] + + # Submit file for resubmission with a profile selected + resp = get_api_data(session, f"{host}/api/v4/submit/static/{sha256}/") + assert resp['params']['description'].startswith('Resubmit') + assert resp['params']['description'].endswith('Static Analysis') + assert resp['sid'] != submission.sid + for f in resp['files']: + assert f['sha256'] == sha256 + assert set(resp['params']['services']['selected']) == set(DEFAULT_SRV_SEL) + + msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) + assert msg.submission.sid == resp['sid'] + + # noinspection PyUnusedLocal def test_resubmit_dynamic(datastore, login_session, scheduler): From cd312130cecbe65f7a7474ab3a2070fbb7a620e7 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 13 Feb 2025 18:15:55 +0000 Subject: [PATCH 45/47] Fix for 3.9 compatibility --- assemblyline_ui/api/v4/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index d2669243..8de90786 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -34,7 +34,7 @@ submission_client = SubmissionClient(datastore=STORAGE, filestore=FILESTORE, config=config, identify=IDENTIFY) -def create_resubmission_task(sha256: str, user: dict, copy_sid: str = None, name: str = None, profile: str = None, **kwargs) ->Union[Tuple[Submission, int] | Response]: +def create_resubmission_task(sha256: str, user: dict, copy_sid: str = None, name: str = None, profile: str = None, **kwargs) ->Union[Tuple[Submission, int], Response]: # Check if we've reached the quotas quota_error = check_submission_quota(user) if quota_error: From ba7a49164b06087a579e34f10a3e0d35fe933274 Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 14 Feb 2025 17:04:31 +0000 Subject: [PATCH 46/47] Allow the API to correct the name of the file if the downloaded content doesn't match the hash given --- assemblyline_ui/api/v4/ingest.py | 4 ++-- assemblyline_ui/api/v4/submit.py | 4 ++-- assemblyline_ui/helper/submission.py | 13 ++++++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index 8df7584e..1df82871 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -295,8 +295,8 @@ def ingest_single_file(**kwargs): if not binary: if string_type: try: - found, fileinfo = fetch_file(string_type, string_value, user, s_params, metadata, out_file, - default_external_sources) + found, fileinfo, name = fetch_file(string_type, string_value, user, s_params, metadata, out_file, + default_external_sources, name) if not found: raise FileNotFoundError( f"{string_type.upper()} does not exist in Assemblyline or any of the selected sources") diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 8de90786..c939a0c8 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -443,8 +443,8 @@ def submit(**kwargs): if not binary: if string_type: try: - found, _ = fetch_file(string_type, string_value, user, s_params, metadata, out_file, - default_external_sources) + found, _, name = fetch_file(string_type, string_value, user, s_params, metadata, out_file, + default_external_sources, name) if not found: raise FileNotFoundError( f"{string_type.upper()} does not exist in Assemblyline or any of the selected sources") diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 678ae5b2..3383b96e 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -19,7 +19,7 @@ from assemblyline.odm.messages.submission import SubmissionMessage from assemblyline.odm.models.user import ROLES -from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES +from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES, IDENTIFY # Baseline fetch methods FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url']) @@ -87,7 +87,7 @@ def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submissi return recursive_update(validated_profile, updates) def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: dict, out_file: str, - default_external_sources: List[str]): + default_external_sources: List[str], name: str): sha256 = None fileinfo = None # If the method is by SHA256 hash, check to see if we already have that file @@ -210,11 +210,18 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di if service not in s_params['services']['selected']: s_params['services']['selected'].append(service) + # Check if the downloaded content has the same hash as the fetch method + if method in HASH_PATTERN_MAP and name == input: + hash = IDENTIFY.fileinfo(out_file)[method] + if hash != input: + # Rename the file to the hash of the downloaded content to avoid confusion + name = hash + # A source suited for the task was found, skip the rest break - return found, fileinfo + return found, fileinfo, name def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict: s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) From 3e5071a2a32335c41e0091e692201cf6c0873f1c Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:25:07 +0000 Subject: [PATCH 47/47] Re-apply changes to the default profile settings --- assemblyline_ui/helper/submission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 3383b96e..43c34cdc 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -245,6 +245,7 @@ def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict s_params = recursive_update(s_params, data.get("params", {})) s_params = get_recursive_delta(DEFAULT_USER_PROFILE_SETTINGS, s_params) s_params = apply_changes_to_profile(s_profile, s_params, submission_customize) + s_params = recursive_update(DEFAULT_USER_PROFILE_SETTINGS, s_params) # Ensure the description key exists in the resulting submission params s_params.setdefault("description", "")