Skip to content

Commit 119bdb3

Browse files
malvidinmalvidin
committed
Merge remote-tracking branch 'cccs/master' into quarantine_magic
# Conflicts: # assemblyline/common/custom.magic
2 parents 89fc7d5 + 25deb64 commit 119bdb3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+7267
-4247
lines changed

‎.github/dependabot.yml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: pip
4+
directory: "/"
5+
schedule:
6+
interval: daily
7+
time: "10:00"
8+
open-pull-requests-limit: 10

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ venv/
8080
ENV/
8181
env.bak/
8282
venv.bak/
83+
.vscode/
8384

8485
# Cython debug symbols
85-
cython_debug/
86+
cython_debug/

‎assemblyline/cachestore/__init__.py

+5
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ def get(self, cache_key):
4545

4646
return self.filestore.get(new_key)
4747

48+
def exists(self, cache_key):
49+
new_key = f"{self.component}_{cache_key}" if self.component else cache_key
50+
51+
return self.filestore.exists(new_key)
52+
4853
def delete(self, cache_key, db_delete=True):
4954
new_key = f"{self.component}_{cache_key}" if self.component else cache_key
5055

‎assemblyline/common/attack_map.py

+4,614-3,378
Large diffs are not rendered by default.

‎assemblyline/common/banner.py

+29-18
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,30 @@
1-
BANNER = r"""
2-
/^\/^\
3-
_|__| O|
4-
\/ /~ \_/ \
5-
\____|__________/ \
6-
\_______ \
7-
`\ \ \
8-
| | \
9-
/ / \
10-
/ / \\
11-
/ / \ \
12-
/ / \ \
13-
/ / _----_ \ \
14-
/ / _-~ ~-_ | |
15-
( ( _-~ _--_ ~-_ _/ |
16-
\ ~-____-~ _-~ ~-_ ~-_-~ /
17-
~-_ _-~ ~-_ _-~
18-
~--______-~ ~-___-~
1+
BANNER = r"""
2+
#########
3+
###### ##############################
4+
##### ###################### ###
5+
##### ###################### ###
6+
###### ############## ########
7+
############ ########
8+
########### --------
9+
############ /..........\
10+
############# /..............\
11+
############# \..../ \..../
12+
###### ##### |..| |..|
13+
####### #### |..| |..|
14+
########### ### |..| |..|
15+
################# #### \.| |./
16+
####################### #####
17+
##############################
18+
#############################
19+
##########################
20+
########################
21+
######################
22+
###################
23+
.......................
24+
........................
25+
..........................
26+
..........................
27+
..........................
28+
..........................
29+
................................
1930
"""

‎assemblyline/common/bundling.py

+39-11
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,25 @@
22
import json
33
import logging
44
import os
5+
import shutil
56
import subprocess
7+
import tempfile
68
import time
79

8-
import shutil
910
from copy import copy
10-
11-
from assemblyline.datastore.exceptions import MultiKeyError
11+
from cart import pack_stream, unpack_stream, is_cart
1212

1313
from assemblyline.common import forge
1414
from assemblyline.common.uid import get_random_id
15+
from assemblyline.datastore.exceptions import MultiKeyError
1516
from assemblyline.filestore import FileStoreException
1617

1718
config = forge.get_config()
1819
Classification = forge.get_classification()
1920
MAX_RETRY = 10
2021
WORK_DIR = "/tmp/bundling"
2122
BUNDLE_MAGIC = b'\x1f\x8b\x08'
23+
BUNDLE_TYPE = "archive/bundle/al"
2224

2325
log = logging.getLogger('assemblyline.bundling')
2426

@@ -149,13 +151,14 @@ def create_bundle(sid, working_dir=WORK_DIR):
149151
if submission is None:
150152
raise SubmissionNotFound("Can't find submission %s, skipping." % sid)
151153
else:
152-
target_file = os.path.join(working_dir, f"{temp_bundle_file}.tgz")
154+
target_file = os.path.join(working_dir, f"{temp_bundle_file}.cart")
155+
tgz_file = os.path.join(working_dir, f"{temp_bundle_file}.tgz")
153156

154157
try:
155158
os.makedirs(current_working_dir)
156-
except Exception as e:
157-
if isinstance(PermissionError, e):
158-
raise
159+
except PermissionError:
160+
raise
161+
except Exception:
159162
pass
160163

161164
# Create file information data
@@ -191,7 +194,11 @@ def create_bundle(sid, working_dir=WORK_DIR):
191194
pass
192195

193196
# Create the bundle
194-
subprocess.check_call("tar czf %s *" % target_file, shell=True, cwd=current_working_dir)
197+
subprocess.check_call("tar czf %s *" % tgz_file, shell=True, cwd=current_working_dir)
198+
199+
with open(target_file, 'wb') as oh:
200+
with open(tgz_file, 'rb') as ih:
201+
pack_stream(ih, oh, {'al': {"type": BUNDLE_TYPE}, 'name': f"{sid}.tgz"})
195202

196203
return target_file
197204

@@ -204,17 +211,33 @@ def create_bundle(sid, working_dir=WORK_DIR):
204211

205212
# noinspection PyBroadException,PyProtectedMember
206213
def import_bundle(path, working_dir=WORK_DIR, min_classification=Classification.UNRESTRICTED, allow_incomplete=False):
207-
with forge.get_datastore() as datastore:
214+
with forge.get_datastore(archive_access=True) as datastore:
208215
current_working_dir = os.path.join(working_dir, get_random_id())
209216
res_file = os.path.join(current_working_dir, "results.json")
210217
try:
211218
os.makedirs(current_working_dir)
212219
except Exception:
213220
pass
214221

222+
with open(path, 'rb') as original_file:
223+
if is_cart(original_file.read(256)):
224+
original_file.seek(0)
225+
226+
extracted_fd, extracted_path = tempfile.mkstemp()
227+
extracted_file = os.fdopen(extracted_fd, 'wb')
228+
229+
try:
230+
hdr, _ = unpack_stream(original_file, extracted_file)
231+
if hdr.get('al', {}).get('type', 'unknown') != BUNDLE_TYPE:
232+
raise BundlingException(f"Not a valid CaRTed bundle, should be of type: {BUNDLE_TYPE}")
233+
finally:
234+
extracted_file.close()
235+
else:
236+
extracted_path = path
237+
215238
# Extract the bundle
216239
try:
217-
subprocess.check_call(["tar", "-zxf", path, "-C", current_working_dir])
240+
subprocess.check_call(["tar", "-zxf", extracted_path, "-C", current_working_dir])
218241
except subprocess.CalledProcessError:
219242
raise BundlingException("Bundle decompression failed. Not a valid bundle...")
220243

@@ -275,8 +298,13 @@ def import_bundle(path, working_dir=WORK_DIR, min_classification=Classification.
275298
for ekey, err in errors['errors'].items():
276299
datastore.error.save(ekey, err)
277300

301+
return submission
278302
finally:
279-
# Perform working dir cleanup
303+
try:
304+
os.remove(extracted_path)
305+
except Exception:
306+
pass
307+
280308
try:
281309
os.remove(path)
282310
except Exception:

‎assemblyline/common/classification.py

+23-9
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ def __init__(self, classification_definition: Dict):
5757
self._classification_cache_short = set()
5858

5959
self.enforce = False
60+
self.dynamic_groups = False
6061

6162
# Add Invalid classification
6263
self.levels_map["INV"] = self.INVALID_LVL
@@ -72,6 +73,7 @@ def __init__(self, classification_definition: Dict):
7273

7374
try:
7475
self.enforce = classification_definition['enforce']
76+
self.dynamic_groups = classification_definition['dynamic_groups']
7577
if self.enforce:
7678
self._classification_cache = self.list_all_classification_combinations()
7779
self._classification_cache_short = self.list_all_classification_combinations(long_format=False)
@@ -256,6 +258,7 @@ def _get_c12n_groups(self, c12n: str, long_format: bool = True) -> Tuple[List, L
256258

257259
g1_set = set()
258260
g2_set = set()
261+
others = set()
259262

260263
grp_part = c12n.split("//")
261264
groups = []
@@ -280,9 +283,23 @@ def _get_c12n_groups(self, c12n: str, long_format: bool = True) -> Tuple[List, L
280283
elif g in self.subgroups_aliases:
281284
for a in self.subgroups_aliases[g]:
282285
g2_set.add(a)
286+
else:
287+
others.add(g)
288+
289+
if self.dynamic_groups:
290+
for o in others:
291+
if o not in self.access_req_map_lts \
292+
and o not in self.access_req_map_stl \
293+
and o not in self.access_req_aliases \
294+
and o not in self.levels_map \
295+
and o not in self.levels_map_lts \
296+
and o not in self.levels_aliases:
297+
g1_set.add(o)
283298

284299
if long_format:
285-
return sorted([self.groups_map_stl[r] for r in g1_set]), sorted([self.subgroups_map_stl[r] for r in g2_set])
300+
return sorted(
301+
[self.groups_map_stl.get(r, r) for r in g1_set]), sorted(
302+
[self.subgroups_map_stl[r] for r in g2_set])
286303
return sorted(list(g1_set)), sorted(list(g2_set))
287304

288305
@staticmethod
@@ -731,7 +748,7 @@ def is_valid(self, c12n: str, skip_auto_select: bool = False) -> bool:
731748
i not in self.access_req_map_lts.keys():
732749
check_groups = True
733750

734-
if check_groups:
751+
if check_groups and not self.dynamic_groups:
735752
# If not groups. That stuff does not exists...
736753
if i not in self.groups_aliases.keys() and \
737754
i not in self.groups_map_stl.keys() and \
@@ -879,9 +896,9 @@ def build_user_classification(self, c12n_1: str, c12n_2: str, long_format: bool
879896

880897
# Normalize classifications before comparing them
881898
if c12n_1 is not None:
882-
c12n_1 = self.normalize_classification(c12n_1)
899+
c12n_1 = self.normalize_classification(c12n_1, skip_auto_select=True)
883900
if c12n_2 is not None:
884-
c12n_2 = self.normalize_classification(c12n_2)
901+
c12n_2 = self.normalize_classification(c12n_2, skip_auto_select=True)
885902

886903
if c12n_1 is None:
887904
return c12n_2
@@ -895,8 +912,5 @@ def build_user_classification(self, c12n_1: str, c12n_2: str, long_format: bool
895912
groups = list(set(groups_1) | set(groups_2))
896913
subgroups = list(set(subgroups_1) | set(subgroups_2))
897914

898-
return self._get_normalized_classification_text(max(lvl_idx_1, lvl_idx_2),
899-
req,
900-
groups,
901-
subgroups,
902-
long_format=long_format)
915+
return self._get_normalized_classification_text(max(lvl_idx_1, lvl_idx_2), req, groups, subgroups,
916+
long_format=long_format, skip_auto_select=True)

0 commit comments

Comments
 (0)