Merging master into the SubmissionProfile

assemblyline_ui/api/v4/file.py

@@ -31,7 +31,7 @@
 file_api = make_subapi_blueprint(SUB_API, api_version=4)
 file_api._doc = "Perform operations on files"
 
-API_MAX_SIZE = 10 * 1024 * 1024
+API_MAX_SIZE = 1 * 1024 * 1024
 
 
 @file_api.route("/ascii/<sha256>/", methods=["GET"])
@@ -50,7 +50,10 @@ def get_file_ascii(sha256, **kwargs):
     None
 
     Result example:
-    <THE ASCII FILE>
+    {
+      "content": <THE ASCII FILE>,
+      "truncated": false
+    }
     """
 
     user = kwargs['user']
@@ -59,15 +62,12 @@ def get_file_ascii(sha256, **kwargs):
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
-    if file_obj['size'] > API_MAX_SIZE:
-        return make_api_response({}, "This file is too big to be seen through this API.", 403)
-
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
         try:
-            data = FILESTORE.get(sha256)
+            data = FILESTORE.get(sha256)[:API_MAX_SIZE]
         except FileStoreException:
             data = None
 
@@ -84,7 +84,10 @@ def get_file_ascii(sha256, **kwargs):
         if not data:
             return make_api_response({}, "This file was not found in the system.", 404)
 
-        return make_api_response(data.translate(FILTER_ASCII).decode())
+        return make_api_response({
+            "content": data.translate(FILTER_ASCII).decode(),
+            "truncated": file_obj['size'] > API_MAX_SIZE
+        })
     else:
         return make_api_response({}, "You are not allowed to view this file.", 403)
 
@@ -428,7 +431,10 @@ def get_file_hex(sha256, **kwargs):
     /api/v4/file/hex/123456...654321/
 
     Result example:
-    <THE FILE HEX REPRESENTATION>
+    {
+      "content": <THE FILE HEX REPRESENTATION>,
+      "truncated": false
+    }
     """
     user = kwargs['user']
     file_obj = STORAGE.file.get(sha256, as_obj=False)
@@ -439,12 +445,9 @@ def get_file_hex(sha256, **kwargs):
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
-    if file_obj['size'] > API_MAX_SIZE:
-        return make_api_response({}, "This file is too big to be seen through this API.", 403)
-
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
         try:
-            data = FILESTORE.get(sha256)
+            data = FILESTORE.get(sha256)[:API_MAX_SIZE]
         except FileStoreException:
             data = None
 
@@ -462,9 +465,15 @@ def get_file_hex(sha256, **kwargs):
             return make_api_response({}, "This file was not found in the system.", 404)
 
         if bytes_only:
-            return make_api_response(dump(data).decode())
+            return make_api_response({
+                "content": dump(data).decode(),
+                "truncated": file_obj['size'] > API_MAX_SIZE
+            })
         else:
-            return make_api_response(hexdump(data, length=length))
+            return make_api_response({
+                "content": hexdump(data, length=length),
+                "truncated": file_obj['size'] > API_MAX_SIZE
+            })
     else:
         return make_api_response({}, "You are not allowed to view this file.", 403)
 
@@ -539,7 +548,10 @@ def get_file_strings(sha256, **kwargs):
     None
 
     Result example:
-    <THE LIST OF STRINGS>
+    {
+      "content": <THE LIST OF STRINGS>,
+      "truncated": false
+    }
     """
     user = kwargs['user']
     hlen = request.args.get('len', "6")
@@ -548,15 +560,12 @@ def get_file_strings(sha256, **kwargs):
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
-    if file_obj['size'] > API_MAX_SIZE:
-        return make_api_response({}, "This file is too big to be seen through this API.", 403)
-
     if not file_obj:
         return make_api_response({}, "The file was not found in the system.", 404)
 
     if user and Classification.is_accessible(user['classification'], file_obj['classification']):
         try:
-            data = FILESTORE.get(sha256)
+            data = FILESTORE.get(sha256)[:API_MAX_SIZE]
         except FileStoreException:
             data = None
 
@@ -580,7 +589,10 @@ def get_file_strings(sha256, **kwargs):
         # UTF-16 strings
         string_list += re.findall(pattern, data.decode("utf-16", errors="replace"))
 
-        return make_api_response("\n".join(string_list))
+        return make_api_response({
+            "content": "\n".join(string_list),
+            "truncated": file_obj['size'] > API_MAX_SIZE
+        })
     else:
         return make_api_response({}, "You are not allowed to view this file.", 403)
 

assemblyline_ui/api/v4/service.py

@@ -256,7 +256,7 @@ def add_service(**_):
             # Create a Service object
             tmp_service = Service(tmp_data)
 
-            _, tag_name, _ = get_latest_tag_for_service(tmp_service, config, LOGGER)
+            _, tag_name, _, _ = get_latest_tag_for_service(tmp_service, config, LOGGER)
             enable_allowed = bool(tag_name)
             tag_name = tag_name.encode() if tag_name else b'latest'
 
@@ -621,6 +621,7 @@ def get_service(servicename, **_):
             if 'update_interval' not in s:
                 s['update_interval'] = service['update_config']['update_interval_seconds']
     if service:
+        service['auto_update'] = service.get('auto_update', config.services.default_auto_update)
         # Ensure service classification is set in response
         service['classification'] = service.get('classification', Classification.UNRESTRICTED)
         return make_api_response(service)
@@ -670,6 +671,7 @@ def get_service_defaults(servicename, version, **_):
     service = STORAGE.service.get(f"{servicename}_{version}", as_obj=False)
     append_source_status(service)
     if service:
+        service['auto_update'] = service.get('auto_update', config.services.default_auto_update)
         # Ensure service classification is set in response
         service['classification'] = service.get('classification', Classification.UNRESTRICTED)
         return make_api_response(service)

assemblyline_ui/api/v4/user.py

@@ -326,7 +326,7 @@ def add_user_account(username, **_):
     }
     """
 
-    data = request.json
+    data: dict = request.json
 
     if "{" in username or "}" in username:
         return make_api_response({"success": False}, "You can't use '{}' in the username", 412)
@@ -353,6 +353,10 @@ def add_user_account(username, **_):
         # Clear non user account data
         avatar = data.pop('avatar', None)
 
+        # Check identity_id value
+        if not data.get('identity_id'):
+            data.pop('identity_id')
+
         if avatar is not None:
             STORAGE.user_avatar.save(username, avatar)
 
@@ -492,7 +496,7 @@ def set_user_account(username, **kwargs):
     }
     """
     try:
-        data = request.json
+        data: dict = request.json
         new_pass = data.pop('new_pass', None)
 
         old_user = STORAGE.user.get(username, as_obj=False)
@@ -519,6 +523,10 @@ def set_user_account(username, **kwargs):
         # Apply dynamic classification
         data['classification'] = get_dynamic_classification(data['classification'], data)
 
+        # Check identity_id value
+        if not data.get('identity_id'):
+            data.pop('identity_id')
+
         ret_val = save_user_account(username, data, kwargs['user'])
 
         if ret_val and \

assemblyline_ui/api/v4/workflow.py

@@ -300,7 +300,7 @@ def remove_workflow(workflow_id, **_):
                                  status_code=404)
 
 
-@workflow_api.route("/<workflow_id>/run", methods=["GET"])
+@workflow_api.route("/<workflow_id>/run/", methods=["GET"])
 @api_login(audit=False, allow_readonly=False, require_role=[ROLES.workflow_manage])
 def run_workflow(workflow_id, **_):
     """

test/test_file.py

@@ -83,7 +83,7 @@ def test_ascii(datastore, login_session):
 
     rand_hash = random.choice(file_res_list)[:64]
     resp = get_api_data(session, f"{host}/api/v4/file/ascii/{rand_hash}/")
-    assert resp == rand_hash
+    assert resp == {"content": rand_hash, "truncated": False}
 
 
 # noinspection PyUnusedLocal
@@ -103,7 +103,7 @@ def test_hex(datastore, login_session):
 
     rand_hash = random.choice(file_res_list)[:64]
     resp = get_api_data(session, f"{host}/api/v4/file/hex/{rand_hash}/")
-    assert resp.startswith("00000000:") and len(resp) == 311
+    assert resp["content"].startswith("00000000:") and len(resp["content"]) == 311
 
 
 # noinspection PyUnusedLocal
@@ -158,4 +158,4 @@ def test_strings(datastore, login_session):
 
     rand_hash = random.choice(file_res_list)[:64]
     resp = get_api_data(session, f"{host}/api/v4/file/strings/{rand_hash}/")
-    assert resp == rand_hash
+    assert resp == {"content": rand_hash, "truncated": False}

test/test_service.py

@@ -118,7 +118,13 @@ def test_get_service(datastore, login_session):
     resp = get_api_data(session, f"{host}/api/v4/service/{service}/")
     service_data = datastore.get_service_with_delta(service, as_obj=False)
     # Drop status information from signature sources
-    [s.pop('status', None)for s in resp.get('update_config', {}).get('sources', [])]
+    [s.pop('status', None) for s in resp.get('update_config', {}).get('sources', [])]
+
+    # Make sure we do not compare the auto_update field has this field is modified by the API
+    # to use the default system configuration if it is not set.
+    service_data.pop('auto_update', None)
+    resp.pop('auto_update', None)
+
     assert resp == service_data