diff --git a/assemblyline_ui/api/v4/badlist.py b/assemblyline_ui/api/v4/badlist.py index ea396b2e..9efae686 100644 --- a/assemblyline_ui/api/v4/badlist.py +++ b/assemblyline_ui/api/v4/badlist.py @@ -720,7 +720,7 @@ def remove_attribution(qhash, attrib_type, value, **_): while True: current_badlist, version = STORAGE.badlist.get_if_exists(qhash, as_obj=False, version=True) if not current_badlist: - return make_api_response({}, "The badlist ietm your are trying to modify does not exists", 404) + return make_api_response({}, "The badlist item you are trying to modify does not exists", 404) if 'attribution' not in current_badlist: return make_api_response({'success': False}) diff --git a/assemblyline_ui/api/v4/ingest.py b/assemblyline_ui/api/v4/ingest.py index ff611f18..1df82871 100644 --- a/assemblyline_ui/api/v4/ingest.py +++ b/assemblyline_ui/api/v4/ingest.py @@ -22,7 +22,8 @@ STORAGE, config, FILESTORE, metadata_validator, LOGGER from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ - FETCH_METHODS, URL_GENERATORS + FETCH_METHODS, URL_GENERATORS, update_submission_parameters + from assemblyline_ui.helper.user import check_async_submission_quota, decrement_submission_ingest_quota, \ load_user_settings @@ -37,6 +38,16 @@ port=config.core.redis.persistent.port) MAX_SIZE = config.submission.max_file_size +DEFAULT_INGEST_PARAMS = { + 'deep_scan': False, + "priority": 150, + "ignore_cache": False, + # the following one line can be removed after assemblyline 4.6+ + "ignore_dynamic_recursion_prevention": False, + "ignore_recursion_prevention": False, + "ignore_filtering": False, + "type": "INGEST" +} # noinspection PyUnusedLocal @ingest_api.route("/get_message//", methods=["GET"]) @@ -151,15 +162,17 @@ def ingest_single_file(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + + "submission_profile": "Static Analysis", # Name of submission profile to use - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v3/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v3/user/submission_params// "generate_alert": False, # Generate an alert in our alerting system or not "notification_queue": None, # Name of the notification queue @@ -256,22 +269,21 @@ def ingest_single_file(**kwargs): default_external_sources = user_settings.pop('default_external_sources', []) # Load default user params from user settings - s_params = ui_to_submission_params(user_settings) + if ROLES.submission_customize in user['roles']: + s_params = ui_to_submission_params(user_settings) + else: + s_params = {} - # Reset dangerous user settings to safe values - s_params.update({ - 'deep_scan': False, - "priority": 150, - "ignore_cache": False, - # the following one line can be removed after assemblyline 4.6+ - "ignore_dynamic_recursion_prevention": False, - "ignore_recursion_prevention": False, - "ignore_filtering": False, - "type": "INGEST" - }) + # Update submission parameters as specified by the user + try: + s_params = update_submission_parameters(s_params, data, user) + except Exception as e: + return make_api_response({}, str(e), 400) - # Apply provided params - s_params.update(data.get("params", {})) + # Set any dangerous user settings to safe values (if wasn't set in request) + for k, v in DEFAULT_INGEST_PARAMS.items(): + if k not in s_params: + s_params[k] = v # Use the `default_external_sources` if specified as a param in request otherwise default to user's settings default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources @@ -283,8 +295,8 @@ def ingest_single_file(**kwargs): if not binary: if string_type: try: - found, fileinfo = fetch_file(string_type, string_value, user, s_params, metadata, out_file, - default_external_sources) + found, fileinfo, name = fetch_file(string_type, string_value, user, s_params, metadata, out_file, + default_external_sources, name) if not found: raise FileNotFoundError( f"{string_type.upper()} does not exist in Assemblyline or any of the selected sources") diff --git a/assemblyline_ui/api/v4/submission.py b/assemblyline_ui/api/v4/submission.py index 5f8e68a2..3a74e4e8 100644 --- a/assemblyline_ui/api/v4/submission.py +++ b/assemblyline_ui/api/v4/submission.py @@ -312,7 +312,6 @@ def get_full_results(sid, **kwargs): }, "state": "completed", # State of the submission "submission": { # Submission Block - "profile": true, # Should keep stats about execution? "description": "", # Submission description "ttl": 30, # Submission days to live "ignore_filtering": false, # Ignore filtering services? @@ -451,7 +450,6 @@ def get_submission(sid, **kwargs): ["FNAME", "sha256"], ...], # Each file = List of name/sha256 "errors": [], # List of error keys (sha256.ServiceName) "submission": { # Submission Block - "profile": true, # Should keep stats about execution? "description": "", # Submission description "ttl": 30, # Submission days to live "ignore_filtering": false, # Ignore filtering services? diff --git a/assemblyline_ui/api/v4/submit.py b/assemblyline_ui/api/v4/submit.py index 43149454..c939a0c8 100644 --- a/assemblyline_ui/api/v4/submit.py +++ b/assemblyline_ui/api/v4/submit.py @@ -7,6 +7,7 @@ import tempfile from flask import request +from typing import Tuple, Union from assemblyline.common.constants import MAX_PRIORITY, PRIORITIES from assemblyline.common.dict_utils import flatten @@ -15,12 +16,12 @@ from assemblyline.odm.messages.submission import Submission from assemblyline.odm.models.user import ROLES from assemblyline_core.submission_client import SubmissionClient, SubmissionException -from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint +from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint, Response from assemblyline_ui.config import ARCHIVESTORE, STORAGE, TEMP_SUBMIT_DIR, FILESTORE, config, \ - CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER + CLASSIFICATION as Classification, IDENTIFY, metadata_validator, LOGGER, SUBMISSION_PROFILES from assemblyline_ui.helper.service import ui_to_submission_params from assemblyline_ui.helper.submission import FileTooBigException, submission_received, refang_url, fetch_file, \ - FETCH_METHODS, URL_GENERATORS + FETCH_METHODS, URL_GENERATORS, update_submission_parameters from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota, load_user_settings SUB_API = 'submit' @@ -33,28 +34,7 @@ submission_client = SubmissionClient(datastore=STORAGE, filestore=FILESTORE, config=config, identify=IDENTIFY) -# noinspection PyUnusedLocal -@submit_api.route("/dynamic//", methods=["GET"]) -@api_login(allow_readonly=False, require_role=[ROLES.submission_create], count_toward_quota=False) -def resubmit_for_dynamic(sha256, *args, **kwargs): - """ - Resubmit a file for dynamic analysis - - Variables: - sha256 => Resource locator (SHA256) - - Arguments (Optional): - copy_sid => Mimic the attributes of this SID. - name => Name of the file for the submission - - Data Block: - None - - Result example: - # Submission message object as a json dictionary - """ - user = kwargs['user'] - +def create_resubmission_task(sha256: str, user: dict, copy_sid: str = None, name: str = None, profile: str = None, **kwargs) ->Union[Tuple[Submission, int], Response]: # Check if we've reached the quotas quota_error = check_submission_quota(user) if quota_error: @@ -68,58 +48,70 @@ def resubmit_for_dynamic(sha256, *args, **kwargs): if not Classification.is_accessible(user['classification'], file_info['classification']): return make_api_response("", "You are not allowed to re-submit a file that you don't have access to", 403) - submit_result = None metadata = {} - try: - copy_sid = request.args.get('copy_sid', None) - if copy_sid: - submission = STORAGE.submission.get(copy_sid, as_obj=False) - else: - submission = None - - if submission: - if not Classification.is_accessible(user['classification'], submission['classification']): - return make_api_response("", - "You are not allowed to re-submit a submission that you don't have access to", - 403) - - submission_params = submission['params'] - submission_params['classification'] = submission['classification'] - expiry = submission['expiry_ts'] - metadata = submission['metadata'] + copy_sid = request.args.get('copy_sid', None) + if copy_sid: + submission = STORAGE.submission.get(copy_sid, as_obj=False) + else: + submission = None + + if submission: + if not Classification.is_accessible(user['classification'], submission['classification']): + return make_api_response("", + "You are not allowed to re-submit a submission that you don't have access to", + 403) + + submission_params = submission['params'] + submission_params['classification'] = submission['classification'] + expiry = submission['expiry_ts'] + metadata = submission['metadata'] + + else: + submission_params = ui_to_submission_params(load_user_settings(user)) + submission_params['classification'] = file_info['classification'] + expiry = file_info['expiry_ts'] + + # Ignore external sources + submission_params.pop('default_external_sources', None) + + if not FILESTORE.exists(sha256): + if ARCHIVESTORE and ARCHIVESTORE != FILESTORE and \ + ROLES.archive_download in user['roles'] and ARCHIVESTORE.exists(sha256): + + # File exists in the archivestore, copying it to the filestore + with tempfile.NamedTemporaryFile() as buf: + ARCHIVESTORE.download(sha256, buf.name) + FILESTORE.upload(buf.name, sha256, location='far') else: - submission_params = ui_to_submission_params(load_user_settings(user)) - submission_params['classification'] = file_info['classification'] - expiry = file_info['expiry_ts'] - - # Ignore external sources - submission_params.pop('default_external_sources', None) + return make_api_response({}, "File %s cannot be found on the server therefore it cannot be resubmitted." + % sha256, status_code=404) - if not FILESTORE.exists(sha256): - if ARCHIVESTORE and ARCHIVESTORE != FILESTORE and \ - ROLES.archive_download in user['roles'] and ARCHIVESTORE.exists(sha256): - # File exists in the archivestore, copying it to the filestore - with tempfile.NamedTemporaryFile() as buf: - ARCHIVESTORE.download(sha256, buf.name) - FILESTORE.upload(buf.name, sha256, location='far') + if (file_info["type"].startswith("uri/") and "uri_info" in file_info and "uri" in file_info["uri_info"]): + name = safe_str(file_info["uri_info"]["uri"]) + else: + name = safe_str(request.args.get('name', sha256)) + description_prefix = f"Resubmit {name}" - else: - return make_api_response({}, "File %s cannot be found on the server therefore it cannot be resubmitted." - % sha256, status_code=404) + files = [{'name': name, 'sha256': sha256, 'size': file_info['size']}] - if (file_info["type"].startswith("uri/") and "uri_info" in file_info and "uri" in file_info["uri_info"]): - name = safe_str(file_info["uri_info"]["uri"]) - submission_params['description'] = f"Resubmit {file_info['uri_info']['uri']} for Dynamic Analysis" + if profile: + # Obtain any settings from the user and apply them to the submission + user_settings = STORAGE.user_settings.get(user['uname'], as_obj=False) + if user_settings: + # Reuse existing settings for specified profile + profile_params = user_settings['submission_profiles'].get(profile, {}) else: - name = safe_str(request.args.get('name', sha256)) - submission_params['description'] = f"Resubmit {name} for Dynamic Analysis" + # Otherwise default to what's set for the profile at the configuration-level + profile_params = {} + profile_params['submission_profile'] = profile - files = [{'name': name, 'sha256': sha256, 'size': file_info['size']}] + submission_params = update_submission_parameters(submission_params, profile_params, user) + submission_params['description'] = f"{description_prefix} with {SUBMISSION_PROFILES[profile].display_name}" - submission_params['submitter'] = user['uname'] - submission_params['quota_item'] = True + else: + # Only append Dynamic Analysis as a selected service and set the priority if 'priority' not in submission_params: submission_params['priority'] = 500 if "Dynamic Analysis" not in submission_params['services']['selected']: @@ -127,20 +119,92 @@ def resubmit_for_dynamic(sha256, *args, **kwargs): # Ensure submission priority stays within the range of user priorities submission_params['priority'] = max(min(submission_params['priority'], MAX_PRIORITY), PRIORITIES['user-low']) + submission_params['description'] = f"{description_prefix} for Dynamic Analysis" - try: - submission_obj = Submission({ - "files": files, - "params": submission_params, - "metadata": metadata, - }) - except (ValueError, KeyError) as e: - return make_api_response("", err=str(e), status_code=400) + submission_params['submitter'] = user['uname'] + submission_params['quota_item'] = True - submit_result = submission_client.submit(submission_obj, expiry=expiry) - submission_received(submission_obj) - return make_api_response(submit_result.as_primitives()) + try: + return Submission({ "files": files, "params": submission_params, "metadata": metadata}), expiry + except (ValueError, KeyError) as e: + return make_api_response("", err=str(e), status_code=400) + +# noinspection PyUnusedLocal +@submit_api.route("///", methods=["GET"]) +@api_login(allow_readonly=False, require_role=[ROLES.submission_create], count_toward_quota=False) +def resubmit_with_profile(profile, sha256, *args, **kwargs): + """ + Resubmit a file using a submission profile + + Variables: + profile => Submission profile to be used in new submission + sha256 => Resource locator (SHA256) + Arguments (Optional): + copy_sid => Mimic the attributes of this SID. + name => Name of the file for the submission + + Data Block: + None + + Result example: + # Submission message object as a json dictionary + """ + user = kwargs['user'] + + submit_result = None + try: + ret_value = create_resubmission_task(sha256=sha256, profile=profile, user=user, **request.args) + if isinstance(ret_value, Response): + # Forward error response back to user + return ret_value + else: + # Otherwise we got submission object with an expiry + submission_obj, expiry = ret_value + submit_result = submission_client.submit(submission_obj, expiry=expiry) + submission_received(submission_obj) + return make_api_response(submit_result.as_primitives()) + except SubmissionException as e: + return make_api_response("", err=str(e), status_code=400) + finally: + if submit_result is None: + # We had an error during the submission, release the quotas for the user + decrement_submission_quota(user) + + +# noinspection PyUnusedLocal +@submit_api.route("/dynamic//", methods=["GET"]) +@api_login(allow_readonly=False, require_role=[ROLES.submission_create], count_toward_quota=False) +def resubmit_for_dynamic(sha256, *args, **kwargs): + """ + Resubmit a file for dynamic analysis + + Variables: + sha256 => Resource locator (SHA256) + + Arguments (Optional): + copy_sid => Mimic the attributes of this SID. + name => Name of the file for the submission + + Data Block: + None + + Result example: + # Submission message object as a json dictionary + """ + user = kwargs['user'] + submit_result = None + try: + ret_value = create_resubmission_task(sha256=sha256, user=user, **request.args) + if isinstance(ret_value, Response): + # Forward error response back to user + return ret_value + else: + # Otherwise we got submission object with an expiry + submission_obj, expiry = ret_value + submit_result = submission_client.submit(submission_obj, expiry=expiry) + submission_received(submission_obj) + return make_api_response(submit_result.as_primitives()) except SubmissionException as e: return make_api_response("", err=str(e), status_code=400) finally: @@ -255,15 +319,17 @@ def submit(**kwargs): "base64": "", // OPTIONAL VALUES - "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url + "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url - "metadata": { # Submission metadata - "key": val, # Key/Value pair for metadata parameters + "submission_profile": "Static Analysis", # Name of submission profile to use + + "metadata": { # Submission metadata + "key": val, # Key/Value pair for metadata parameters }, - "params": { # Submission parameters - "key": val, # Key/Value pair for params that differ from the user's defaults - }, # Default params can be fetch at /api/v3/user/submission_params// + "params": { # Submission parameters + "key": val, # Key/Value pair for params that differ from the user's defaults + }, # Default params can be fetch at /api/v4/user/submission_params// } Data Block (Binary): @@ -338,8 +404,18 @@ def submit(**kwargs): default_external_sources = user_settings.pop('default_external_sources', []) # Create task object - s_params = ui_to_submission_params(user_settings) - s_params.update(data.get("params", {})) + if (ROLES.submission_customize in user['roles']) or "ui_params" in data: + s_params = ui_to_submission_params(user_settings) + else: + s_params = {} + + # Update submission parameters as specified by the user + try: + s_params = update_submission_parameters(s_params, data, user) + except Exception as e: + return make_api_response({}, str(e), 400) + + default_external_sources = s_params.pop('default_external_sources', []) or default_external_sources if 'groups' not in s_params: s_params['groups'] = [g for g in user['groups'] if g in s_params['classification']] @@ -367,8 +443,8 @@ def submit(**kwargs): if not binary: if string_type: try: - found, _ = fetch_file(string_type, string_value, user, s_params, metadata, out_file, - default_external_sources) + found, _, name = fetch_file(string_type, string_value, user, s_params, metadata, out_file, + default_external_sources, name) if not found: raise FileNotFoundError( f"{string_type.upper()} does not exist in Assemblyline or any of the selected sources") diff --git a/assemblyline_ui/api/v4/ui.py b/assemblyline_ui/api/v4/ui.py index 363890a1..2806f67d 100644 --- a/assemblyline_ui/api/v4/ui.py +++ b/assemblyline_ui/api/v4/ui.py @@ -16,7 +16,7 @@ from assemblyline_ui.config import TEMP_DIR, STORAGE, FILESTORE, config, CLASSIFICATION as Classification, \ IDENTIFY, metadata_validator from assemblyline_ui.helper.service import ui_to_submission_params -from assemblyline_ui.helper.submission import submission_received +from assemblyline_ui.helper.submission import submission_received, update_submission_parameters from assemblyline_ui.helper.user import check_submission_quota, decrement_submission_quota from assemblyline_core.submission_client import SubmissionClient, SubmissionException @@ -252,14 +252,21 @@ def start_ui_submission(ui_sid, **kwargs): return make_api_response("", err=str(e), status_code=400) return make_api_response({"started": True, "sid": submission['sid']}) - allow_description_overwrite = False - if not ui_params['description']: - allow_description_overwrite = True - ui_params['description'] = f"Inspection of file: {fname}" - # Submit to dispatcher try: params = ui_to_submission_params(ui_params) + + # Update submission parameters as specified by the user + try: + params = update_submission_parameters(params, ui_params, user) + except Exception as e: + return make_api_response({}, str(e), 400) + + allow_description_overwrite = False + if not ui_params['description']: + allow_description_overwrite = True + ui_params['description'] = f"Inspection of file: {fname}" + metadata = params.pop("metadata", {}) # Enforce maximum DTL diff --git a/assemblyline_ui/api/v4/user.py b/assemblyline_ui/api/v4/user.py index a5d75411..27f23555 100644 --- a/assemblyline_ui/api/v4/user.py +++ b/assemblyline_ui/api/v4/user.py @@ -1,28 +1,55 @@ +from copy import deepcopy from typing import List -from assemblyline.odm.models.config import ExternalLinks -from flask import request, session as flsk_session from assemblyline.common.comms import send_activated_email, send_authorize_email from assemblyline.common.isotime import now_as_iso -from assemblyline.common.security import (check_password_requirements, get_password_hash, - get_password_requirement_message) +from assemblyline.common.security import ( + check_password_requirements, + get_password_hash, + get_password_requirement_message, +) from assemblyline.datastore.exceptions import SearchException -from assemblyline.odm.models.config import HASH_PATTERN_MAP -from assemblyline.odm.models.user import (ACL_MAP, ROLES, USER_ROLES, USER_TYPE_DEP, USER_TYPES, User, load_roles, - load_roles_form_acls) +from assemblyline.odm.models.config import HASH_PATTERN_MAP, ExternalLinks +from assemblyline.odm.models.user import ( + ACL_MAP, + ROLES, + USER_ROLES, + USER_TYPE_DEP, + USER_TYPES, + User, + load_roles, + load_roles_form_acls, +) from assemblyline.odm.models.user_favorites import Favorite from assemblyline_ui.api.base import api_login, make_api_response, make_subapi_blueprint -from assemblyline_ui.config import APPS_LIST, CLASSIFICATION, CLASSIFICATION_ALIASES, DAILY_QUOTA_TRACKER, LOGGER, \ - STORAGE, UI_MESSAGING, VERSION, config, AI_AGENT, UI_METADATA_VALIDATION +from assemblyline_ui.api.v4.federated_lookup import filtered_tag_names +from assemblyline_ui.config import ( + AI_AGENT, + APPS_LIST, + CLASSIFICATION, + CLASSIFICATION_ALIASES, + DAILY_QUOTA_TRACKER, + LOGGER, + STORAGE, + SUBMISSION_PROFILES, + UI_MESSAGING, + UI_METADATA_VALIDATION, + VERSION, + config, +) from assemblyline_ui.helper.search import list_all_fields from assemblyline_ui.helper.service import simplify_service_spec, ui_to_submission_params from assemblyline_ui.helper.user import ( - get_default_user_quotas, get_dynamic_classification, load_user_settings, save_user_account, save_user_settings, - API_PRIV_MAP) + API_PRIV_MAP, + get_default_user_quotas, + get_dynamic_classification, + load_user_settings, + save_user_account, + save_user_settings, +) from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException - -from .federated_lookup import filtered_tag_names - +from flask import request +from flask import session as flsk_session SUB_API = 'user' user_api = make_subapi_blueprint(SUB_API, api_version=4) @@ -91,10 +118,13 @@ def who_am_i(**kwargs): "max_dtl": 30, # Maximum number of days retrohunt job stay in the system }, "submission": { # Submission Configuration + "configurable_params": [], # Submission parameters that are configurable when using profiles "dtl": 10, # Default number of days submission stay in the system "max_dtl": 30, # Maximum number of days submission stay in the system + "max_file_size": 104857600, # Maximum size for files submitted in the system "file_sources": [], # List of file sources to perform remote submission into the system "metadata": {}, # Metadata compliance policy to submit to the system + "profiles": {}, # Submission profiles "verdicts": { # Verdict scoring configuration "info": 0, # Default minimum score for info "suspicious": 300, # Default minimum score for suspicious @@ -193,6 +223,29 @@ def who_am_i(**kwargs): [file_sources["sha256"]["sources"].append(x.name) for x in config.submission.sha256_sources if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x.classification)] + # Prepare submission profile configurations for UI + submission_profiles = {} + for name, profile in SUBMISSION_PROFILES.items(): + if CLASSIFICATION.is_accessible(kwargs['user']['classification'], profile.classification): + # We want to pass forward the configurations that have been explicitly set as a configuration + submission_profiles[name] = profile.as_primitives(strip_null=True) + + # Remove unwanted/redundant data from being shared + for field in ['name', 'classification']: + submission_profiles[name].pop(field, None) + + + # Expand service categories if used in submission profiles (assists with the UI locking down service selection) + service_categories = list(STORAGE.service.facet('category').keys()) + for profile in submission_profiles.values(): + for key, services in profile.get("services", {}).items(): + expanded_services = list() + for srv in services: + if srv in service_categories: + expanded_services.extend([i['name'] for i in STORAGE.service.search( + f"category:{srv}", as_obj=False, fl="name")['items']]) + profile['services'][key] = list(set(services).union(set(expanded_services))) + user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, @@ -218,8 +271,10 @@ def who_am_i(**kwargs): "submission": { "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl, + "max_file_size": config.submission.max_file_size, "file_sources": file_sources, "metadata": UI_METADATA_VALIDATION, + "profiles": submission_profiles, "verdicts": { "info": config.submission.verdicts.info, "suspicious": config.submission.verdicts.suspicious, @@ -892,19 +947,14 @@ def get_user_settings(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled - "classification": "", # Default classification for this user sumbissions - "description": "", # Default description for this user's submissions - "download_encoding": "blah", # Default encoding for downloaded files - "default_zip_password": "pass", # Default password for password protected ZIP - "expand_min_score": 100, # Default minimum score to auto-expand sections - "priority": 1000, # Default submission priority - "service_spec": [], # Default Service specific parameters - "ignore_cache": true, # Should file be reprocessed even if there are cached results - "groups": [ ... ], # Default groups selection for the user scans - "ttl": 30, # Default time to live in days of the users submissions - "services": [ ... ], # Default list of selected services - "ignore_filtering": false # Should filtering services by ignored? + "default_external_sources": [], # Default file sources for this user + "default_zip_password": "infected", # Default password for password protected ZIP + "download_encoding": "blah", # Default encoding for downloaded files, + "executive_summary": false, # Should the executive summary be shown by default + "expand_min_score": 100, # Default minimum score to auto-expand sections + "preferred_submission_profile": "default", # Default submission profile + "submission_profiles": [], # List of submission profiles + "submission_view": "report", # Default submission view } """ user = kwargs['user'] @@ -931,7 +981,6 @@ def set_user_settings(username, **kwargs): Data Block: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "default_zip_password": "zippy" # Default password used for protected file downloads "description": "", # Default description for this user's submissions @@ -952,8 +1001,6 @@ def set_user_settings(username, **kwargs): } """ user = kwargs['user'] - if username != user['uname'] and ROLES.administration not in user['roles']: - raise AccessDeniedException("You are not allowed to set settings for another user then yourself.") try: data = request.json @@ -961,10 +1008,24 @@ def set_user_settings(username, **kwargs): if not data.get('default_zip_password', ''): return make_api_response({"success": False}, "Encryption password can't be empty.", 403) - if save_user_settings(username, data): + # Changing your own settings + if username == user['uname']: + if ROLES.administration not in user['roles'] and ROLES.self_manage not in user['roles']: + raise AccessDeniedException("You are not allowed to modify your own settings.") + edit_user = user + + # Changing someone else's settings + else: + if ROLES.administration not in user['roles']: + raise AccessDeniedException("You are not allowed to set settings for another user then yourself.") + edit_user = STORAGE.user.get(username, as_obj=False) + + if save_user_settings(edit_user, data): return make_api_response({"success": True}) else: return make_api_response({"success": False}, "Failed to save user's settings", 500) + except PermissionError as e: + return make_api_response({"success": False}, str(e), 401) except ValueError as e: return make_api_response({"success": False}, str(e), 400) @@ -992,7 +1053,6 @@ def get_user_submission_params(username, **kwargs): Result example: { - "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "description": "", # Default description for this user's submissions "priority": 1000, # Default submission priority diff --git a/assemblyline_ui/config.py b/assemblyline_ui/config.py index 423ef964..9b596d1a 100644 --- a/assemblyline_ui/config.py +++ b/assemblyline_ui/config.py @@ -182,5 +182,7 @@ def get_signup_queue(key): ARCHIVE_MANAGER: ArchiveManager = ArchiveManager( config=config, datastore=STORAGE, filestore=FILESTORE, identify=IDENTIFY) SERVICE_LIST = forge.CachedObject(STORAGE.list_all_services, kwargs=dict(as_obj=False, full=True)) +SUBMISSION_PROFILES = {profile.name: profile for profile in config.submission.profiles} + # End global ################################################################# diff --git a/assemblyline_ui/helper/service.py b/assemblyline_ui/helper/service.py index 249bf859..ab269a9a 100644 --- a/assemblyline_ui/helper/service.py +++ b/assemblyline_ui/helper/service.py @@ -1,6 +1,7 @@ from copy import copy from typing import Any, Optional -from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST +from assemblyline.common.dict_utils import recursive_update +from assemblyline_ui.config import CLASSIFICATION, config, SERVICE_LIST, SUBMISSION_PROFILES, STORAGE from assemblyline.odm.models.submission import DEFAULT_SRV_SEL, SubmissionParams from assemblyline.odm.models.user_settings import UserSettings @@ -8,6 +9,20 @@ USER_SETTINGS_FIELDS = list(UserSettings.fields().keys()) SUBMISSION_PARAM_FIELDS = list(SubmissionParams.fields().keys()) + +def get_default_submission_profiles(user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED, + include_default=False): + out = {} + if include_default: + out['default'] = user_default_values.get('default', {}) + + for profile in SUBMISSION_PROFILES.values(): + if CLASSIFICATION.is_accessible(classification, profile.classification): + profile_values = copy(profile.params.as_primitives(strip_null=True)) + out[profile.name] = recursive_update(profile_values, user_default_values.get(profile.name, {})) + return out + + def get_default_service_spec(srv_list=None, user_default_values={}, classification=CLASSIFICATION.UNRESTRICTED): if not srv_list: srv_list = SERVICE_LIST diff --git a/assemblyline_ui/helper/submission.py b/assemblyline_ui/helper/submission.py index 4d7c57c7..43c34cdc 100644 --- a/assemblyline_ui/helper/submission.py +++ b/assemblyline_ui/helper/submission.py @@ -9,14 +9,17 @@ from typing import List from urllib.parse import urlparse +from assemblyline.common.dict_utils import recursive_update, get_recursive_delta from assemblyline.common.file import make_uri_file from assemblyline.common.isotime import now_as_iso from assemblyline.common.str_utils import safe_str from assemblyline.common.iprange import is_ip_reserved -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, SubmissionProfile, SubmissionProfileParams +from assemblyline.odm.models.user_settings import DEFAULT_USER_PROFILE_SETTINGS from assemblyline.odm.messages.submission import SubmissionMessage + from assemblyline.odm.models.user import ROLES -from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE +from assemblyline_ui.config import STORAGE, CLASSIFICATION, SUBMISSION_TRAFFIC, config, FILESTORE, ARCHIVESTORE, SUBMISSION_PROFILES, IDENTIFY # Baseline fetch methods FETCH_METHODS = set(list(HASH_PATTERN_MAP.keys()) + ['url']) @@ -37,6 +40,7 @@ MYIP = '127.0.0.1' + ############################# # download functions class FileTooBigException(Exception): @@ -50,9 +54,40 @@ class InvalidUrlException(Exception): class ForbiddenLocation(Exception): pass +def apply_changes_to_profile(profile: SubmissionProfile, updates: dict, submission_customize=False) -> dict: + validated_profile = profile.params.as_primitives(strip_null=True) + + # Check to see if user is trying to modify the service params of a service not explicitly defined in the profile + unrecognized_services = set(updates.get('service_spec', {}).keys()) - set(list(profile.editable_params.keys())) + if unrecognized_services and not submission_customize: + # User isn't allowed to change/set parameters of services not explicitly defined in the profile + raise PermissionError(f"User isn't allowed to modify the following services: {list(unrecognized_services)}") + + for param_type, list_of_params in profile.editable_params.items(): + if param_type == "submission": + # Submission-level parameters + for p in list(updates.keys()): + if not hasattr(SubmissionProfileParams, p): + # Don't need to be concerned about a parameter that has no limitation ie. description + continue + elif p in ['services', 'service_spec']: + # These will be checked later + continue + elif p not in list_of_params and not submission_customize: + # Submission parameter isn't allowed to be modified based on profile configuration + raise PermissionError(f"User isn't allowed to modify the \"{p}\" parameter of {profile.display_name} profile") + else: + # Service-level parameters + service_spec = updates.get('service_spec', {}).get(param_type, {}) + for key in list(service_spec.keys()): + if key not in list_of_params and not submission_customize: + # Service parameter isn't allowed to be changed + raise PermissionError(f"User isn't allowed to modify the \"{p}\" parameter of \"{param_type}\" service in \"{profile.display_name}\" profile") + + return recursive_update(validated_profile, updates) def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: dict, out_file: str, - default_external_sources: List[str]): + default_external_sources: List[str], name: str): sha256 = None fileinfo = None # If the method is by SHA256 hash, check to see if we already have that file @@ -175,15 +210,47 @@ def fetch_file(method: str, input: str, user: dict, s_params: dict, metadata: di if service not in s_params['services']['selected']: s_params['services']['selected'].append(service) + # Check if the downloaded content has the same hash as the fetch method + if method in HASH_PATTERN_MAP and name == input: + hash = IDENTIFY.fileinfo(out_file)[method] + if hash != input: + # Rename the file to the hash of the downloaded content to avoid confusion + name = hash + # A source suited for the task was found, skip the rest break - return found, fileinfo - - - - + return found, fileinfo, name + +def update_submission_parameters(s_params: dict, data: dict, user: dict) -> dict: + s_profile = SUBMISSION_PROFILES.get(data.get('submission_profile')) + submission_customize = ROLES.submission_customize in user['roles'] + + # Ensure classification is set based on the user before applying updates + classification = s_params.get("classification", user['classification']) + + # Apply provided params (if the user is allowed to) + if submission_customize: + s_params.update(data.get("params", {})) + elif not s_profile: + # No profile specified, raise an exception back to the user + raise Exception(f"You must specify a submission profile. One of: {list(SUBMISSION_PROFILES.keys())}") + + if s_profile: + if not CLASSIFICATION.is_accessible(user['classification'], s_profile.classification): + # User isn't allowed to use the submission profile specified + raise PermissionError(f"You aren't allowed to use '{s_profile.name}' submission profile") + # Apply the profile (but allow the user to change some properties) + s_params = recursive_update(s_params, data.get("params", {})) + s_params = get_recursive_delta(DEFAULT_USER_PROFILE_SETTINGS, s_params) + s_params = apply_changes_to_profile(s_profile, s_params, submission_customize) + s_params = recursive_update(DEFAULT_USER_PROFILE_SETTINGS, s_params) + + # Ensure the description key exists in the resulting submission params + s_params.setdefault("description", "") + s_params.setdefault("classification", classification) + return s_params def refang_url(url): diff --git a/assemblyline_ui/helper/user.py b/assemblyline_ui/helper/user.py index 53c3aeb5..3bd1f6e7 100644 --- a/assemblyline_ui/helper/user.py +++ b/assemblyline_ui/helper/user.py @@ -1,14 +1,32 @@ from typing import Optional -from flask import session as flsk_session - from assemblyline.common.str_utils import safe_str -from assemblyline.odm.models.user import User, load_roles, ROLES -from assemblyline.odm.models.user_settings import UserSettings -from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER, DAILY_QUOTA_TRACKER, LOGGER, STORAGE, SUBMISSION_TRACKER, \ - config, CLASSIFICATION as Classification, SERVICE_LIST, DOWNLOAD_ENCODING, DEFAULT_ZIP_PASSWORD -from assemblyline_ui.helper.service import get_default_service_spec, get_default_service_list, simplify_services -from assemblyline_ui.http_exceptions import AccessDeniedException, InvalidDataException, AuthenticationException +from assemblyline.common.dict_utils import get_recursive_delta +from assemblyline.odm.models.config import SubmissionProfileParams, SubmissionProfile +from assemblyline.odm.models.user import ROLES, User, load_roles +from assemblyline.odm.models.user_settings import UserSettings, DEFAULT_USER_PROFILE_SETTINGS +from assemblyline_ui.config import ASYNC_SUBMISSION_TRACKER +from assemblyline_ui.config import CLASSIFICATION as Classification +from assemblyline_ui.config import ( + DAILY_QUOTA_TRACKER, + DEFAULT_ZIP_PASSWORD, + DOWNLOAD_ENCODING, + LOGGER, + SERVICE_LIST, + STORAGE, + SUBMISSION_PROFILES, + SUBMISSION_TRACKER, + config, +) +from assemblyline_ui.helper.service import ( + get_default_service_list, + get_default_service_spec, + get_default_submission_profiles, + simplify_services, +) +from assemblyline_ui.helper.submission import apply_changes_to_profile +from assemblyline_ui.http_exceptions import AccessDeniedException, AuthenticationException, InvalidDataException +from flask import session as flsk_session ACCOUNT_USER_MODIFIABLE = ["name", "avatar", "password"] @@ -258,23 +276,25 @@ def get_dynamic_classification(current_c12n, user_info): return new_c12n -def get_default_user_settings(user): - return UserSettings({"classification": Classification.default_user_classification(user), - "ttl": config.submission.dtl, - "default_zip_password": DEFAULT_ZIP_PASSWORD, - "download_encoding": DOWNLOAD_ENCODING}).as_primitives() +def get_default_user_settings(user: dict) -> dict: + settings = DEFAULT_USER_PROFILE_SETTINGS + settings.update({"classification": Classification.default_user_classification(user), + "ttl": config.submission.dtl, + "default_zip_password": DEFAULT_ZIP_PASSWORD, + "download_encoding": DOWNLOAD_ENCODING}) + return UserSettings(settings).as_primitives() def load_user_settings(user): default_settings = get_default_user_settings(user) user_classfication = user.get('classification', Classification.UNRESTRICTED) - - settings = STORAGE.user_settings.get_if_exists(user['uname'], as_obj=False) - srv_list = [x for x in SERVICE_LIST if x['enabled']] + submission_customize = ROLES.submission_customize in user['roles'] + settings = STORAGE.user_settings.get_if_exists(user['uname']) if not settings: def_srv_list = None settings = default_settings else: + settings = settings.as_primitives(strip_null=True) # Make sure all defaults are there for key, item in default_settings.items(): if key not in settings: @@ -287,18 +307,90 @@ def load_user_settings(user): def_srv_list = settings.get('services', {}).get('selected', None) + srv_list = [x for x in SERVICE_LIST if x['enabled']] + settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) + # Normalize the user's classification + settings['classification'] = Classification.normalize_classification(settings['classification']) + + # Check if the user has instantiated their default submission profile + if submission_customize and not settings['submission_profiles'].get('default'): + settings['submission_profiles']['default'] = SubmissionProfileParams({key: value for key, value in settings.items() if key in SubmissionProfileParams.fields()}).as_primitives() + # Only display services that a user is allowed to see settings['service_spec'] = get_default_service_spec(srv_list, settings.get('service_spec', {}), user_classfication) settings['services'] = get_default_service_list(srv_list, def_srv_list, user_classfication) - settings['default_zip_password'] = settings.get('default_zip_password', DEFAULT_ZIP_PASSWORD) + settings['submission_profiles'] = get_default_submission_profiles(settings['submission_profiles'], + user_classfication, include_default=submission_customize) - # Normalize the user's classification - settings['classification'] = Classification.normalize_classification(settings['classification']) + + # Check if the user has a preferred submission profile + if not settings.get('preferred_submission_profile'): + # No preferred submission profile, set one based on the user's roles + if submission_customize: + # User can customize their submission, set the preferred profile to the legacy default + settings['preferred_submission_profile'] = 'default' + else: + # User cannot customize their submission, set the preferred profile to first one on the list + settings['preferred_submission_profile'] = list(settings['submission_profiles'].keys())[0] return settings -def save_user_settings(username, data): - data["services"] = {'selected': simplify_services(data["services"])} +def save_user_settings(user, data): + username = user.get('uname', None) + if username == None: + raise Exception("Invalid username") + + user_settings = STORAGE.user_settings.get(username) + if user_settings: + user_settings = user_settings.as_primitives() + else: + user_settings = {} + + for key in user_settings.keys(): + if key in data and key not in ["services", "service_spec", "submission_profiles"]: + user_settings[key] = data.get(key, None) + + user_settings["services"] = {'selected': simplify_services(data.get("services", []))} + + classification = user.get("classification", None) + submission_customize = ROLES.submission_customize in user['roles'] + srv_list = [x['name'] for x in SERVICE_LIST if x['enabled']] + srv_list += [x['category'] for x in SERVICE_LIST if x['enabled']] + srv_list = list(set(srv_list)) + + accessible_profiles = [name for name, profile in SUBMISSION_PROFILES.items() \ + if Classification.is_accessible(classification, profile.classification)] + + # Check submission profile preference selection + preferred_submission_profile = data.get('preferred_submission_profile', None) + if submission_customize: + # User is allowed to customize their own default profile + accessible_profiles += ['default'] + + if preferred_submission_profile in accessible_profiles: + user_settings['preferred_submission_profile'] = preferred_submission_profile + + submission_profiles = {} + for name in accessible_profiles: + user_params = data.get('submission_profiles', {}).get(name, {}) + profile_config: Optional[SubmissionProfile] = SUBMISSION_PROFILES.get(name) + + if profile_config == None: + if name == "default": + # There is no restriction on what you can set for your default submission profile + # Set profile based on preferences set at the root-level + data["services"] = user_settings['services'] + submission_profiles[name] = SubmissionProfileParams({key: value for key, value in data.items() + if key in SubmissionProfileParams.fields()}).as_primitives() + + else: + # Calculate what the profiles updates are based on default profile settings and the user-submitted changes + profile_updates = get_recursive_delta(DEFAULT_USER_PROFILE_SETTINGS, user_params) + + # Apply changes to the profile relative to what's allowed to be changed based on configuration + submission_profiles[name] = apply_changes_to_profile(profile_config, profile_updates, submission_customize) + + user_settings["submission_profiles"] = submission_profiles - return STORAGE.user_settings.save(username, data) + return STORAGE.user_settings.save(username, user_settings) diff --git a/pipelines/config.yml b/pipelines/config.yml index cfa82dfc..7a57fde4 100644 --- a/pipelines/config.yml +++ b/pipelines/config.yml @@ -41,6 +41,12 @@ submission: validator_type: keyword strict_schemes: - strict_ingest + profiles: + # Profile that will enforce metadata validation + - name: "static" + display_name: "Static Analysis" + params: + type: strict_ingest logging: log_level: DISABLED diff --git a/test/config/config.yml b/test/config/config.yml index 66e52072..e1d24619 100644 --- a/test/config/config.yml +++ b/test/config/config.yml @@ -45,6 +45,12 @@ submission: validator_type: keyword strict_schemes: - strict_ingest + profiles: + # Profile that will enforce metadata validation + - name: "static" + display_name: "Static Analysis" + params: + type: strict_ingest ui: enforce_quota: false diff --git a/test/test_badlist.py b/test/test_badlist.py index 618c889e..2338ff27 100644 --- a/test/test_badlist.py +++ b/test/test_badlist.py @@ -354,14 +354,22 @@ def test_badlist_delete_hash(datastore, login_session): def test_badlist_attribution(datastore, login_session): _, session, host = login_session + badlist_items = datastore.badlist.search("attribution.actor:*", + fl="id,attribution.actor", rows=100, as_obj=False)['items'] + while True: + item = random.choice(badlist_items) + + actor = random.choice(item['attribution']['actor']) + hash = item['id'] + + try: + # Test removing attribution from the hash in the Badlist + resp = get_api_data(session, f"{host}/api/v4/badlist/attribution/{hash}/actor/{actor}/", method="DELETE") + break + except APIError: + # TODO: Investigate why in some cases the API thinks the hash doesn't exist + pass - item = random.choice(datastore.badlist.search("attribution.actor:*", - fl="id,attribution.actor", rows=100, as_obj=False)['items']) - - actor = random.choice(item['attribution']['actor']) - hash = item['id'] - - resp = get_api_data(session, f"{host}/api/v4/badlist/attribution/{hash}/actor/{actor}/", method="DELETE") assert resp['success'] assert actor not in datastore.badlist.get_if_exists(hash).attribution.actor diff --git a/test/test_ingest.py b/test/test_ingest.py index cf7e8822..f2b9b249 100644 --- a/test/test_ingest.py +++ b/test/test_ingest.py @@ -10,7 +10,7 @@ from assemblyline.common import forge from assemblyline.odm.messages.submission import Submission -from assemblyline.odm.models.config import HASH_PATTERN_MAP +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES from assemblyline.odm.models.file import File from assemblyline.odm.randomizer import random_model_obj, get_random_phrase from assemblyline.odm.random_data import create_users, wipe_users, create_services, wipe_services @@ -293,7 +293,6 @@ def test_ingest_metadata_validation(datastore, login_session): 'base64': base64.b64encode(byte_str).decode('ascii'), 'metadata': {'test': 'ingest_base64_nameless'}, "params": {'type': 'strict_ingest'} - } resp = get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) assert isinstance(resp['ingest_id'], str) @@ -309,6 +308,19 @@ def test_ingest_metadata_validation(datastore, login_session): resp = get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) assert isinstance(resp['ingest_id'], str) + # Test submitting with a submission profile that has the ingest type preset + # With currently set metadata, this should raise an API error + with pytest.raises(APIError, match="Extra metadata found from submission"): + data.pop('params') + data['submission_profile'] = "static" + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Fix metadata and resubmit (still using a submission profile) + data['metadata'].pop('blah') + resp = get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + assert isinstance(resp['ingest_id'], str) + + # noinspection PyUnusedLocal def test_get_message(datastore, login_session): @@ -356,3 +368,36 @@ def test_get_message_list_with_paging(datastore, login_session): message_list += resp for x in range(NUM_FILES): assert message_list[x] == messages[x] + +def test_ingest_submission_profile(datastore, login_session, scheduler): + _, session, host = login_session + iq.delete() + + # Make the user a simple user and try to submit + datastore.user.update('admin', [ + (datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'roles', 'submission_create')]) + byte_str = get_random_phrase(wmin=30, wmax=75).encode() + sha256 = hashlib.sha256(byte_str).hexdigest() + data = { + 'base64': base64.b64encode(byte_str).decode('ascii'), + 'metadata': {'test': 'test_submit_base64_nameless'} + } + with pytest.raises(APIError, match="You must specify a submission profile"): + # A basic user must specify a submission profile name + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with no parameters + profile = DEFAULT_SUBMISSION_PROFILES[0] + data['submission_profile'] = profile["name"] + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with a parameter you aren't allowed to set + # The system should silently ignore your parameter and still create a submission + data['params'] = {'services': {'selected': ['blah']}} + # But also try setting a parameter that you are allowed to set + data['params'] = {'deep_scan': True} + get_api_data(session, f"{host}/api/v4/ingest/", method="POST", data=json.dumps(data)) + + # Restore original roles for later tests + datastore.user.update('admin', [(datastore.user.UPDATE_APPEND, 'type', 'admin'),]) diff --git a/test/test_submit.py b/test/test_submit.py index 60df0227..912f9da4 100644 --- a/test/test_submit.py +++ b/test/test_submit.py @@ -6,11 +6,11 @@ import random import tempfile -from conftest import get_api_data +from conftest import get_api_data, APIError from assemblyline.common import forge -from assemblyline.odm.models.config import HASH_PATTERN_MAP -from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions +from assemblyline.odm.models.config import HASH_PATTERN_MAP, DEFAULT_SUBMISSION_PROFILES, DEFAULT_SRV_SEL +from assemblyline.odm.random_data import create_users, wipe_users, create_submission, wipe_submissions, create_services, wipe_services from assemblyline.odm.randomizer import get_random_phrase from assemblyline.remote.datatypes.queues.named import NamedQueue from assemblyline_core.dispatching.dispatcher import SubmissionTask @@ -27,10 +27,12 @@ def datastore(datastore_connection, filestore): try: create_users(datastore_connection) submission = create_submission(datastore_connection, filestore) + create_services(datastore_connection) yield datastore_connection finally: wipe_users(datastore_connection) wipe_submissions(datastore_connection, filestore) + wipe_services(datastore_connection) sq.delete() @@ -49,6 +51,26 @@ def test_resubmit(datastore, login_session, scheduler): msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) assert msg.submission.sid == resp['sid'] +# noinspection PyUnusedLocal +def test_resubmit_profile(datastore, login_session, scheduler): + _, session, host = login_session + + sq.delete() + sha256 = random.choice(submission.results)[:64] + + # Submit file for resubmission with a profile selected + resp = get_api_data(session, f"{host}/api/v4/submit/static/{sha256}/") + assert resp['params']['description'].startswith('Resubmit') + assert resp['params']['description'].endswith('Static Analysis') + assert resp['sid'] != submission.sid + for f in resp['files']: + assert f['sha256'] == sha256 + assert set(resp['params']['services']['selected']) == set(DEFAULT_SRV_SEL) + + msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) + assert msg.submission.sid == resp['sid'] + + # noinspection PyUnusedLocal def test_resubmit_dynamic(datastore, login_session, scheduler): @@ -282,3 +304,38 @@ def test_submit_base64_nameless(datastore, login_session, scheduler): msg = SubmissionTask(scheduler=scheduler, datastore=datastore, **sq.pop(blocking=False)) assert msg.submission.sid == resp['sid'] + +def test_submit_submission_profile(datastore, login_session, scheduler): + _, session, host = login_session + sq.delete() + + # Make the user a simple user and try to submit + datastore.user.update('admin', [ + (datastore.user.UPDATE_REMOVE, 'type', 'admin'), + (datastore.user.UPDATE_APPEND, 'roles', 'submission_create')]) + byte_str = get_random_phrase(wmin=30, wmax=75).encode() + sha256 = hashlib.sha256(byte_str).hexdigest() + data = { + 'base64': base64.b64encode(byte_str).decode('ascii'), + 'metadata': {'test': 'test_submit_base64_nameless'} + } + with pytest.raises(APIError, match="You must specify a submission profile"): + # A basic user must specify a submission profile name + get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with no parameters + profile = DEFAULT_SUBMISSION_PROFILES[0] + data['submission_profile'] = profile['name'] + get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + + # Try using a submission profile with a parameter you aren't allowed to set + # The system should silently ignore your parameter and still create a submission + data['params'] = {'services': {'selected': ['blah']}} + # But also try setting a parameter that you are allowed to set + data['params'] = {'deep_scan': True} + resp = get_api_data(session, f"{host}/api/v4/submit/", method="POST", data=json.dumps(data)) + assert set(resp['params']['services']['selected']) == set(profile['params']['services']['selected']) + assert resp['params']['deep_scan'] == True + + # Restore original roles for later tests + datastore.user.update('admin', [(datastore.user.UPDATE_APPEND, 'type', 'admin'),]) diff --git a/test/test_user.py b/test/test_user.py index 25e33c91..075dacad 100644 --- a/test/test_user.py +++ b/test/test_user.py @@ -8,6 +8,7 @@ from assemblyline_ui.helper.user import load_user_settings from assemblyline.odm.models.user import User from assemblyline.odm.models.user_favorites import Favorite, UserFavorites +from assemblyline.odm.models.user_settings import UserSettings from assemblyline.odm.randomizer import random_model_obj from assemblyline.odm.random_data import create_users, wipe_users @@ -48,6 +49,7 @@ def datastore(datastore_connection): ds.user.save(u.uname, u) ds.user_favorites.save(u.uname, data) ds.user_avatar.save(u.uname, AVATAR) + ds.user_settings.save(u.uname, random_model_obj(UserSettings)) user_list.append(u.uname) yield ds @@ -278,16 +280,44 @@ def test_set_user_favorites(datastore, login_session): # noinspection PyUnusedLocal -def test_set_user_settings(datastore, login_session): +@pytest.mark.parametrize("allow_submission_customize", [True, False], ids=["submission_customize=true", "submission_customize=false"]) +def test_set_user_settings(datastore, login_session, allow_submission_customize): _, session, host = login_session username = random.choice(user_list) - - uset = load_user_settings({'uname': username}) + user = datastore.user.get(username, as_obj=False) + + if allow_submission_customize: + # User is allowed to customize their submission profiles + datastore.user.update(username, [(datastore.user.UPDATE_APPEND, 'roles', 'submission_customize')]) + if 'submission_customize' not in user['roles']: + user['roles'].append('submission_customize') + else: + # Users that aren't allow to customize submissions shouldn't be able to customize submission profiles parameters if the configuration doesn't allow it + datastore.user.update(username, [(datastore.user.UPDATE_REMOVE, 'roles', 'submission_customize')]) + if 'submission_customize' in user['roles']: + user['roles'].remove('submission_customize') + + uset = load_user_settings(user) uset['expand_min_score'] = 111 uset['priority'] = 111 - - resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset)) - assert resp['success'] - - datastore.user_settings.commit() - assert uset == load_user_settings({'uname': username}) + uset['submission_profiles']['static']['service_spec'] = { + "test": { + "p": True + } + } + + if allow_submission_customize: + # User is allowed to customize their submission profiles as they see fit + resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset)) + assert resp['success'] + + datastore.user_settings.commit() + new_user_settings = load_user_settings(user) + + # Ensure the changes are applied in the right places + assert new_user_settings['submission_profiles']['default']['priority'] == uset['priority'] + assert new_user_settings['submission_profiles']['static']['service_spec'] == uset['submission_profiles']['static']['service_spec'] + else: + with pytest.raises(APIError): + # User isn't allowed to customize their submission profiles, API should return an exception + resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset))