-
Notifications
You must be signed in to change notification settings - Fork 34
/
Copy pathconfig.py
2119 lines (1795 loc) · 98 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
from typing import Any, Dict, List
from assemblyline import odm
from assemblyline.common.constants import PRIORITIES
from assemblyline.odm.models.service import EnvironmentVariable
from assemblyline.odm.models.service_delta import DockerConfigDelta
AUTO_PROPERTY_TYPE = ['access', 'classification', 'type', 'role', 'remove_role', 'group',
'multi_group', 'api_quota', 'api_daily_quota', 'submission_quota',
'submission_async_quota', 'submission_daily_quota']
DEFAULT_EMAIL_FIELDS = ['email', 'emails', 'extension_selectedEmailAddress', 'otherMails', 'preferred_username', 'upn']
DEFAULT_DAILY_API_QUOTA = 0
DEFAULT_API_QUOTA = 10
DEFAULT_DAILY_SUBMISSION_QUOTA = 0
DEFAULT_SUBMISSION_QUOTA = 5
DEFAULT_ASYNC_SUBMISSION_QUOTA = 0
@odm.model(index=False, store=False, description="Password Requirement")
class PasswordRequirement(odm.Model):
lower: bool = odm.Boolean(description="Password must contain lowercase letters")
number: bool = odm.Boolean(description="Password must contain numbers")
special: bool = odm.Boolean(description="Password must contain special characters")
upper: bool = odm.Boolean(description="Password must contain uppercase letters")
min_length: int = odm.Integer(description="Minimum password length")
DEFAULT_PASSWORD_REQUIREMENTS = {
"lower": False,
"number": False,
"special": False,
"upper": False,
"min_length": 12
}
@odm.model(index=False, store=False,
description="Configuration block for [GC Notify](https://notification.canada.ca/) signup and password reset")
class Notify(odm.Model):
base_url: str = odm.Optional(odm.Keyword(), description="Base URL")
api_key: str = odm.Optional(odm.Keyword(), description="API key")
registration_template: str = odm.Optional(odm.Keyword(), description="Registration template")
password_reset_template: str = odm.Optional(odm.Keyword(), description="Password reset template")
authorization_template: str = odm.Optional(odm.Keyword(), description="Authorization template")
activated_template: str = odm.Optional(odm.Keyword(), description="Activated Template")
DEFAULT_NOTIFY = {
"base_url": None,
"api_key": None,
"registration_template": None,
"password_reset_template": None,
"authorization_template": None,
"activated_template": None,
}
@odm.model(index=False, store=False, description="Configuration block for SMTP signup and password reset")
class SMTP(odm.Model):
from_adr: str = odm.Optional(odm.Keyword(), description="Email address used for sender")
host: str = odm.Optional(odm.Keyword(), description="SMTP host")
password: str = odm.Optional(odm.Keyword(), description="Password for SMTP server")
port: int = odm.Integer(description="Port of SMTP server")
tls: bool = odm.Boolean(description="Should we communicate with SMTP server via TLS?")
user: str = odm.Optional(odm.Keyword(), description="User to authenticate to the SMTP server")
DEFAULT_SMTP = {
"from_adr": None,
"host": None,
"password": None,
"port": 587,
"tls": True,
"user": None
}
@odm.model(index=False, store=False, description="Signup Configuration")
class Signup(odm.Model):
enabled: bool = odm.Boolean(description="Can a user automatically signup for the system")
smtp: SMTP = odm.Compound(SMTP, default=DEFAULT_SMTP, description="Signup via SMTP")
notify: Notify = odm.Compound(Notify, default=DEFAULT_NOTIFY, description="Signup via GC Notify")
valid_email_patterns: List[str] = odm.List(
odm.Keyword(),
description="Email patterns that will be allowed to automatically signup for an account")
DEFAULT_SIGNUP = {
"enabled": False,
"notify": DEFAULT_NOTIFY,
"smtp": DEFAULT_SMTP,
"valid_email_patterns": [".*", ".*@localhost"]
}
@odm.model(index=False, store=False)
class AutoProperty(odm.Model):
field: str = odm.Keyword(description="Field to apply `pattern` to")
pattern: str = odm.Keyword(description="Regex pattern for auto-prop assignment")
type: str = odm.Enum(AUTO_PROPERTY_TYPE, description="Type of property assignment on pattern match")
value: List[str] = odm.List(odm.Keyword(), auto=True, default=[], description="Assigned property value")
@odm.model(index=False, store=False, description="LDAP Configuration")
class LDAP(odm.Model):
enabled: bool = odm.Boolean(description="Should LDAP be enabled or not?")
admin_dn: str = odm.Optional(odm.Keyword(), description="DN of the group or the user who will get admin privileges")
bind_user: str = odm.Optional(odm.Keyword(), description="User use to query the LDAP server")
bind_pass: str = odm.Optional(odm.Keyword(), description="Password used to query the LDAP server")
auto_create: bool = odm.Boolean(description="Auto-create users if they are missing")
auto_sync: bool = odm.Boolean(description="Should we automatically sync with LDAP server on each login?")
auto_properties: List[AutoProperty] = odm.List(odm.Compound(AutoProperty), default=[],
description="Automatic role and classification assignments")
base: str = odm.Keyword(description="Base DN for the users")
classification_mappings: Dict[str, str] = odm.Any(description="Classification mapping")
email_field: str = odm.Keyword(description="Name of the field containing the email address")
group_lookup_query: str = odm.Keyword(description="How the group lookup is queried")
group_lookup_with_uid: bool = odm.Boolean(description="Use username/uid instead of dn for group lookup")
image_field: str = odm.Keyword(description="Name of the field containing the user's avatar")
image_format: str = odm.Keyword(description="Type of image used to store the avatar")
name_field: str = odm.Keyword(description="Name of the field containing the user's name")
signature_importer_dn: str = odm.Optional(
odm.Keyword(),
description="DN of the group or the user who will get signature_importer role")
signature_manager_dn: str = odm.Optional(
odm.Keyword(),
description="DN of the group or the user who will get signature_manager role")
uid_field: str = odm.Keyword(description="Field name for the UID")
uri: str = odm.Keyword(description="URI to the LDAP server")
DEFAULT_LDAP = {
"enabled": False,
"bind_user": None,
"bind_pass": None,
"auto_create": True,
"auto_sync": True,
"auto_properties": [],
"base": "ou=people,dc=assemblyline,dc=local",
"email_field": "mail",
"group_lookup_query": "(&(objectClass=Group)(member=%s))",
"group_lookup_with_uid": False,
"image_field": "jpegPhoto",
"image_format": "jpeg",
"name_field": "cn",
"uid_field": "uid",
"uri": "ldap://localhost:389",
# Deprecated
"admin_dn": None,
"classification_mappings": {},
"signature_importer_dn": None,
"signature_manager_dn": None,
}
@odm.model(index=False, store=False, description="Internal Authentication Configuration")
class Internal(odm.Model):
enabled: bool = odm.Boolean(description="Internal authentication allowed?")
failure_ttl: int = odm.Integer(description="How long to wait after `max_failures` before re-attempting login?")
max_failures: int = odm.Integer(description="Maximum number of fails allowed before timeout")
password_requirements: PasswordRequirement = odm.Compound(PasswordRequirement,
default=DEFAULT_PASSWORD_REQUIREMENTS,
description="Password requirements")
signup: Signup = odm.Compound(Signup, default=DEFAULT_SIGNUP, description="Signup method")
DEFAULT_INTERNAL = {
"enabled": True,
"failure_ttl": 60,
"max_failures": 5,
"password_requirements": DEFAULT_PASSWORD_REQUIREMENTS,
"signup": DEFAULT_SIGNUP
}
@odm.model(index=False, store=False, description="App provider")
class AppProvider(odm.Model):
access_token_url: str = odm.Keyword(description="URL used to get the access token")
user_get: str = odm.Optional(odm.Keyword(), description="Path from the base_url to fetch the user info")
group_get: str = odm.Optional(odm.Keyword(), description="Path from the base_url to fetch the group info")
scope: str = odm.Keyword()
client_id: str = odm.Optional(odm.Keyword(), description="ID of your application to authenticate to the OAuth")
client_secret: str = odm.Optional(odm.Keyword(),
description="Password to your application to authenticate to the OAuth provider")
@odm.model(index=False, store=False, description="OAuth Provider Configuration")
class OAuthProvider(odm.Model):
auto_create: bool = odm.Boolean(default=True, description="Auto-create users if they are missing")
auto_sync: bool = odm.Boolean(default=False, description="Should we automatically sync with OAuth provider?")
auto_properties: List[AutoProperty] = odm.List(odm.Compound(AutoProperty), default=[],
description="Automatic role and classification assignments")
app_provider: AppProvider = odm.Optional(odm.Compound(AppProvider))
uid_randomize: bool = odm.Boolean(default=False,
description="Should we generate a random username for the authenticated user?")
uid_randomize_digits: int = odm.Integer(default=0,
description="How many digits should we add at the end of the username?")
uid_randomize_delimiter: str = odm.Keyword(default="-",
description="What is the delimiter used by the random name generator?")
uid_regex: str = odm.Optional(
odm.Keyword(),
description="Regex used to parse an email address and capture parts to create a user ID out of it")
uid_format: str = odm.Optional(odm.Keyword(),
description="Format of the user ID based on the captured parts from the regex")
client_id: str = odm.Optional(odm.Keyword(),
description="ID of your application to authenticate to the OAuth provider")
client_secret: str = odm.Optional(odm.Keyword(),
description="Password to your application to authenticate to the OAuth provider")
redirect_uri: str = odm.Optional(odm.Keyword(),
description="URI to redirect to after authentication with OAuth provider")
request_token_url: str = odm.Optional(odm.Keyword(), description="URL to request token")
request_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters to request token")
access_token_url: str = odm.Optional(odm.Keyword(), description="URL to get access token")
access_token_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()), description="Parameters to get access token")
authorize_url: str = odm.Optional(odm.Keyword(), description="URL used to authorize access to a resource")
authorize_params: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),description="Parameters used to authorize access to a resource")
api_base_url: str = odm.Optional(odm.Keyword(), description="Base URL for downloading the user's and groups info")
client_kwargs: Dict[str, str] = odm.Optional(odm.Mapping(odm.Keyword()),
description="Keyword arguments passed to the different URLs")
jwks_uri: str = odm.Optional(odm.Keyword(), description="URL used to verify if a returned JWKS token is valid")
jwt_token_alg: str = odm.Keyword(default="RS256", description="Algorythm use the validate JWT OBO tokens")
uid_field: str = odm.Optional(odm.Keyword(), description="Name of the field that will contain the user ID")
user_get: str = odm.Optional(odm.Keyword(), description="Path from the base_url to fetch the user info")
user_groups: str = odm.Optional(odm.Keyword(), description="Path from the base_url to fetch the group info")
user_groups_data_field: str = odm.Optional(
odm.Keyword(),
description="Field return by the group info API call that contains the list of groups")
user_groups_name_field: str = odm.Optional(
odm.Keyword(),
description="Name of the field in the list of groups that contains the name of the group")
use_new_callback_format: bool = odm.Boolean(default=False, description="Should we use the new callback method?")
allow_external_tokens: bool = odm.Boolean(
default=False, description="Should token provided to the login API directly be use for authentication?")
external_token_alternate_audiences: List[str] = odm.List(
odm.Keyword(), default=[], description="List of valid alternate audiences for the external token.")
email_fields: List[str] = odm.List(odm.Keyword(), default=DEFAULT_EMAIL_FIELDS,
description="List of fields in the claim to get the email from")
username_field: str = odm.Keyword(default='uname', description="Name of the field that will contain the username")
validate_token_with_secret: bool = odm.Boolean(
default=False, description="Should we send the client secret while validating the access token?")
identity_id_field: str = odm.Keyword(default='oid', description="Field to fetch the managed identity ID from.")
DEFAULT_OAUTH_PROVIDER_AZURE = {
"access_token_url": 'https://login.microsoftonline.com/common/oauth2/token',
"api_base_url": 'https://login.microsoft.com/common/',
"authorize_url": 'https://login.microsoftonline.com/common/oauth2/authorize',
"client_id": None,
"client_secret": None,
"client_kwargs": {"scope": "openid email profile"},
"jwks_uri": "https://login.microsoftonline.com/common/discovery/v2.0/keys",
"user_get": "openid/userinfo"
}
DEFAULT_OAUTH_PROVIDER_GOOGLE = {
"access_token_url": 'https://oauth2.googleapis.com/token',
"api_base_url": 'https://openidconnect.googleapis.com/',
"authorize_url": 'https://accounts.google.com/o/oauth2/v2/auth',
"client_id": None,
"client_secret": None,
"client_kwargs": {"scope": "openid email profile"},
"jwks_uri": "https://www.googleapis.com/oauth2/v3/certs",
"user_get": "v1/userinfo"
}
DEFAULT_OAUTH_PROVIDER_AUTH_ZERO = {
"access_token_url": 'https://{TENANT}.auth0.com/oauth/token',
"api_base_url": 'https://{TENANT}.auth0.com/',
"authorize_url": 'https://{TENANT}.auth0.com/authorize',
"client_id": None,
"client_secret": None,
"client_kwargs": {"scope": "openid email profile"},
"jwks_uri": "https://{TENANT}.auth0.com/.well-known/jwks.json",
"user_get": "userinfo"
}
DEFAULT_OAUTH_PROVIDERS = {
'auth0': DEFAULT_OAUTH_PROVIDER_AUTH_ZERO,
'azure_ad': DEFAULT_OAUTH_PROVIDER_AZURE,
'google': DEFAULT_OAUTH_PROVIDER_GOOGLE,
}
@odm.model(index=False, store=False, description="OAuth Configuration")
class OAuth(odm.Model):
enabled: bool = odm.Boolean(description="Enable use of OAuth?")
gravatar_enabled: bool = odm.Boolean(description="Enable gravatar?")
providers: Dict[str, OAuthProvider] = odm.Mapping(odm.Compound(OAuthProvider),
default=DEFAULT_OAUTH_PROVIDERS,
description="OAuth provider configuration")
DEFAULT_OAUTH = {
"enabled": False,
"gravatar_enabled": True,
"providers": DEFAULT_OAUTH_PROVIDERS
}
DEFAULT_SAML_SETTINGS = {
"strict": False,
"debug": False,
"sp": {
"entity_id": "https://assemblyline/sp",
"assertion_consumer_service": {
"url": "https://localhost/api/v4/auth/saml/acs/",
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
},
"name_id_format": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified"
},
"idp": {
"entity_id": "https://mocksaml.com/api/saml/metadata",
"single_sign_on_service": {
"url": "https://mocksaml.com/api/saml/sso",
"binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
},
},
}
DEFAULT_SAML_ATTRIBUTES = {
"email_attribute": "email",
"fullname_attribute": "name",
"groups_attribute": "groups",
"roles_attribute": "roles",
"group_type_mapping": {},
}
@odm.model(index=False, store=False, description="SAML Assertion Consumer Service")
class SAMLAssertionConsumerService(odm.Model):
url: str = odm.Keyword(description="URL")
binding: str = odm.Keyword(description="Binding", default="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST")
@odm.model(index=False, store=False, description="SAML Single Sign On Service")
class SAMLSingleSignOnService(odm.Model):
url: str = odm.Keyword(description="URL")
binding: str = odm.Keyword(description="Binding", default="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect")
@odm.model(index=False, store=False, description="SAML Attribute")
class SAMLRequestedAttribute(odm.Model):
name: str = odm.Keyword(description="Name")
is_required: bool = odm.Boolean(description="Is required?", default=False)
name_format: str = odm.Keyword(description="Name Format",
default="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified")
friendly_name: str = odm.Keyword(description="Friendly Name", default="")
attribute_value: List[str] = odm.List(odm.Keyword(), description="Attribute Value", default=[])
@odm.model(index=False, store=False, description="SAML Attribute Consuming Service")
class SAMLAttributeConsumingService(odm.Model):
service_name: str = odm.Keyword(description="Service Name")
service_description: str = odm.Keyword(description="Service Description")
requested_attributes: List[SAMLRequestedAttribute] = odm.List(
odm.Compound(SAMLRequestedAttribute),
description="Requested Attributes", default=[])
@odm.model(index=False, store=False, description="SAML Service Provider")
class SAMLServiceProvider(odm.Model):
entity_id: str = odm.Keyword(description="Entity ID")
assertion_consumer_service: SAMLAssertionConsumerService = odm.Compound(
SAMLAssertionConsumerService, description="Assertion Consumer Service")
attribute_consuming_service: SAMLAttributeConsumingService = odm.Optional(
odm.Compound(SAMLAttributeConsumingService), description="Attribute Consuming Service")
name_id_format: str = odm.Keyword(description="Name ID Format",
default="urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified")
x509cert: str = odm.Optional(odm.Keyword(), description="X509 Certificate")
private_key: str = odm.Optional(odm.Keyword(), description="Private Key")
@odm.model(index=False, store=False, description="SAML Identity Provider")
class SAMLIdentityProvider(odm.Model):
entity_id: str = odm.Keyword(description="Entity ID")
single_sign_on_service: SAMLSingleSignOnService = odm.Compound(
SAMLSingleSignOnService, description="Single Sign On Service")
x509cert: str = odm.Optional(odm.Keyword(), description="X509 Certificate")
@odm.model(index=False, store=False, description="SAML Contact Entry")
class SAMLContactPerson(odm.Model):
given_name: str = odm.Keyword(description="Given Name")
email_address: str = odm.Keyword(description="Email Address")
@odm.model(index=False, store=False, description="SAML Contacts")
class SAMLContacts(odm.Model):
technical: SAMLContactPerson = odm.Compound(SAMLContactPerson, description="Technical Contact")
support: SAMLContactPerson = odm.Compound(SAMLContactPerson, description="Support Contact")
@odm.model(index=False, store=False, description="SAML Organization")
class SAMLOrganization(odm.Model):
name: str = odm.Keyword(description="Name")
display_name: str = odm.Keyword(description="Display Name")
url: str = odm.Keyword(description="URL")
@odm.model(index=False, store=False, description="SAML Security")
class SAMLSecurity(odm.Model):
name_id_encrypted: bool = odm.Optional(odm.Boolean(), description="Name ID Encrypted")
authn_requests_signed: bool = odm.Optional(odm.Boolean(), description="Authn Requests Signed")
logout_request_signed: bool = odm.Optional(odm.Boolean(), description="Logout Request Signed")
logout_response_signed: bool = odm.Optional(odm.Boolean(), description="Logout Response Signed")
sign_metadata: bool = odm.Optional(odm.Boolean(), description="Sign Metadata")
want_messages_signed: bool = odm.Optional(odm.Boolean(), description="Want Messages Signed")
want_assertions_signed: bool = odm.Optional(odm.Boolean(), description="Want Assertions Signed")
want_assertions_encrypted: bool = odm.Optional(odm.Boolean(), description="Want Assertions Encrypted")
want_name_id: bool = odm.Optional(odm.Boolean(), description="Want Name ID")
want_name_id_encrypted: bool = odm.Optional(odm.Boolean(), description="Want Name ID Encrypted")
want_attribute_statement: bool = odm.Optional(odm.Boolean(), description="Want Attribute Statement")
requested_authn_context: bool = odm.Optional(odm.Boolean(), description="Requested Authn Context")
requested_authn_context_comparison: str = odm.Optional(
odm.Keyword(), description="Requested Authn Context Comparison")
fail_on_authn_context_mismatch: bool = odm.Optional(odm.Boolean(), description="Fail On Authn Context Mismatch")
metadata_valid_until: str = odm.Optional(odm.Keyword(), description="Metadata Valid Until")
metadata_cache_duration: str = odm.Optional(odm.Keyword(), description="Metadata Cache Duration")
allow_single_label_domains: bool = odm.Optional(odm.Boolean(), description="Allow Single Label Domains")
signature_algorithm: str = odm.Optional(odm.Keyword(), description="Signature Algorithm")
digest_algorithm: str = odm.Optional(odm.Keyword(), description="Digest Algorithm")
allow_repeat_attribute_name: bool = odm.Optional(odm.Boolean(), description="Allow Repeat Attribute Name")
reject_deprecated_algorithm: bool = odm.Optional(odm.Boolean(), description="Reject Deprecated Algorithm")
@odm.model(index=False, store=False, description="SAML Settings")
class SAMLSettings(odm.Model):
strict: bool = odm.Boolean(description="Should we be strict in our SAML checks?", default=True)
debug: bool = odm.Boolean(description="Should we be in debug mode?", default=False)
sp: SAMLServiceProvider = odm.Compound(SAMLServiceProvider, description="SP settings")
idp: SAMLIdentityProvider = odm.Compound(SAMLIdentityProvider, description="IDP settings")
security: SAMLSecurity = odm.Optional(odm.Compound(SAMLSecurity), description="Security settings")
contact_person: SAMLContacts = odm.Optional(odm.Compound(SAMLContacts), description="Contact settings")
organization: Dict[str, SAMLOrganization] = odm.Optional(odm.Mapping(
odm.Compound(SAMLOrganization)), description="Organization settings")
@odm.model(index=False, store=False, description="SAML Attributes")
class SAMLAttributes(odm.Model):
username_attribute: str = odm.Optional(
odm.Keyword(default="uid"),
description="SAML attribute name for AL username")
email_attribute: str = odm.Keyword(description="SAML attribute name for a user's email address ", default="email")
fullname_attribute: str = odm.Keyword(description="SAML attribute name for a user's first name", default="name")
groups_attribute: str = odm.Keyword(description="SAML attribute name for the groups", default="groups")
roles_attribute: str = odm.Keyword(description="SAML attribute name for the roles", default="roles")
group_type_mapping: Dict[str, str] = odm.Mapping(
odm.Keyword(), description="SAML group to role mapping", default={})
@odm.model(index=False, store=False, description="SAML Configuration")
class SAML(odm.Model):
enabled: bool = odm.Boolean(description="Enable use of SAML?")
auto_create: bool = odm.Boolean(description="Auto-create users if they are missing", default=True)
auto_sync: bool = odm.Boolean(
description="Should we automatically sync with SAML server on each login?", default=True)
lowercase_urlencoding: bool = odm.Boolean(
description="Enable lowercase encoding if using ADFS as IdP", default=False)
attributes: SAMLAttributes = odm.Compound(
SAMLAttributes, default=DEFAULT_SAML_ATTRIBUTES, description="SAML attributes")
settings: SAMLSettings = odm.Compound(SAMLSettings, default=DEFAULT_SAML_SETTINGS,
description="SAML settings method")
DEFAULT_SAML = {
"enabled": False,
"auto_create": True,
"auto_sync": True,
"lowercase_urlencoding": False,
"attributes": DEFAULT_SAML_ATTRIBUTES,
"settings": DEFAULT_SAML_SETTINGS
}
@odm.model(index=False, store=False, description="Authentication Methods")
class Auth(odm.Model):
allow_2fa: bool = odm.Boolean(description="Allow 2FA?")
allow_apikeys: bool = odm.Boolean(description="Allow API keys?")
allow_extended_apikeys: bool = odm.Boolean(description="Allow extended API keys?")
allow_security_tokens: bool = odm.Boolean(description="Allow security tokens?")
internal: Internal = odm.Compound(Internal, default=DEFAULT_INTERNAL,
description="Internal authentication settings")
ldap: LDAP = odm.Compound(LDAP, default=DEFAULT_LDAP, description="LDAP settings")
oauth: OAuth = odm.Compound(OAuth, default=DEFAULT_OAUTH, description="OAuth settings")
saml: SAML = odm.Compound(SAML, default=DEFAULT_SAML, description="SAML settings")
DEFAULT_AUTH = {
"allow_2fa": True,
"allow_apikeys": True,
"allow_extended_apikeys": True,
"allow_security_tokens": True,
"internal": DEFAULT_INTERNAL,
"ldap": DEFAULT_LDAP,
"oauth": DEFAULT_OAUTH,
"saml": DEFAULT_SAML
}
@odm.model(index=False, store=False, description="Alerter Configuration")
class Alerter(odm.Model):
alert_ttl: int = odm.integer(description="Time to live (days) for an alert in the system")
constant_alert_fields: List[str] = odm.sequence(
odm.keyword(), default=[],
description="List of fields that should not change during an alert update",
deprecation="This behavior is no longer configurable")
constant_ignore_keys: List[str] = odm.sequence(
odm.keyword(), default=[],
description="List of keys to ignore in the constant alert fields.",
deprecation="This behavior is no longer configurable")
default_group_field: str = odm.keyword(description="Default field used for alert grouping view")
delay: int = odm.integer(
description="Time in seconds that we give extended scans and workflow to complete their work "
"before we start showing alerts in the alert viewer.")
filtering_group_fields: List[str] = odm.sequence(
odm.keyword(),
description="List of group fields that when selected will ignore certain alerts where this field is missing.")
non_filtering_group_fields: List[str] = odm.sequence(
odm.keyword(), description="List of group fields that are sure to be present in all alerts.")
process_alert_message: str = odm.keyword(
description="Python path to the function that will process an alert message.")
threshold: int = odm.integer(description="Minimum score to reach for a submission to be considered an alert.")
DEFAULT_ALERTER = {
"alert_ttl": 90,
"constant_alert_fields": [],
"constant_ignore_keys": [],
"default_group_field": "file.sha256",
"delay": 300,
"filtering_group_fields": [
"file.name",
"status",
"priority"
],
"non_filtering_group_fields": [
"file.md5",
"file.sha1",
"file.sha256"
],
"process_alert_message": "assemblyline_core.alerter.processing.process_alert_message",
"threshold": 500
}
@odm.model(index=False, store=False, description="Dispatcher Configuration")
class Dispatcher(odm.Model):
timeout: float = odm.Integer(
description="Time between re-dispatching attempts, as long as some action (submission or any task completion) "
"happens before this timeout ends, the timeout resets.")
max_inflight: int = odm.Integer(description="Maximum submissions allowed to be in-flight")
DEFAULT_DISPATCHER = {
"timeout": 15*60,
"max_inflight": 1000
}
# Configuration options regarding data expiry
@odm.model(index=False, store=False)
class Expiry(odm.Model):
batch_delete = odm.Boolean(
description="Perform expiry in batches?<br>"
"Delete queries are rounded by day therefore all delete operation happen at the same time at midnight")
delay = odm.Integer(description="Delay, in hours, that will be applied to the expiry query so we can keep"
"data longer then previously set or we can offset deletion during non busy hours")
delete_storage = odm.Boolean(description="Should we also cleanup the file storage?")
sleep_time = odm.Integer(description="Time, in seconds, to sleep in between each expiry run")
workers = odm.Integer(description="Number of concurrent workers")
delete_workers = odm.Integer(description="Worker processes for file storage deletes.")
iteration_max_tasks = odm.Integer(description="How many query chunks get run per iteration.")
delete_batch_size = odm.Integer(max=10000, description="How large a batch get deleted per iteration.")
safelisted_tag_dtl = odm.Integer(min=0, description="The default period, in days, before tags expire from Safelist")
badlisted_tag_dtl = odm.Integer(min=0, description="The default period, in days, before tags expire from Badlist")
DEFAULT_EXPIRY = {
'batch_delete': False,
'delay': 0,
'delete_storage': True,
'sleep_time': 15,
'workers': 20,
'delete_workers': 2,
'iteration_max_tasks': 50,
'delete_batch_size': 2000,
'safelisted_tag_dtl': 0,
'badlisted_tag_dtl': 0
}
@odm.model(index=False, store=False, description="Ingester Configuration")
class Ingester(odm.Model):
always_create_submission: bool = odm.Boolean(default=False,
description="Always create submissions even on cache hit?")
default_user: str = odm.Keyword(description="Default user for bulk ingestion and unattended submissions")
default_services: List[str] = odm.List(odm.Keyword(), description="Default service selection")
default_resubmit_services: List[str] = odm.List(odm.Keyword(),
description="Default service selection for resubmits")
description_prefix: str = odm.Keyword(
description="A prefix for descriptions. When a description is automatically generated, it will be "
"the hash prefixed by this string")
is_low_priority: str = odm.Keyword(
description="Path to a callback function filtering ingestion tasks that should have their priority "
"forcefully reset to low")
get_whitelist_verdict: str = odm.Keyword()
whitelist: str = odm.Keyword()
default_max_extracted: int = odm.Integer(
description="How many extracted files may be added to a Submission. Overrideable via submission parameters.")
default_max_supplementary: int = odm.Integer(
description="How many supplementary files may be added to a Submission. Overrideable via submission parameters")
expire_after: int = odm.Integer(description="Period, in seconds, in which a task should be expired")
stale_after_seconds: int = odm.Integer(description="Drop a task altogether after this many seconds")
incomplete_expire_after_seconds: int = odm.Integer(description="How long should scores be kept before expiry")
incomplete_stale_after_seconds: int = odm.Integer(description="How long should scores be cached in the ingester")
sampling_at: Dict[str, int] = odm.Mapping(odm.Integer(),
description="Thresholds at certain buckets before sampling")
max_inflight = odm.Integer(description="How long can a queue get before we start dropping files")
cache_dtl: int = odm.Integer(min=0, description="How long are files results cached")
DEFAULT_INGESTER = {
'cache_dtl': 2,
'default_user': 'internal',
'default_services': [],
'default_resubmit_services': [],
'description_prefix': 'Bulk',
'is_low_priority': 'assemblyline.common.null.always_false',
'get_whitelist_verdict': 'assemblyline.common.signaturing.drop',
'whitelist': 'assemblyline.common.null.whitelist',
'default_max_extracted': 100,
'default_max_supplementary': 100,
'expire_after': 15 * 24 * 60 * 60,
'stale_after_seconds': 1 * 24 * 60 * 60,
'incomplete_expire_after_seconds': 3600,
'incomplete_stale_after_seconds': 1800,
'sampling_at': {
'low': 10000000,
'medium': 2000000,
'high': 1000000,
'critical': 500000,
},
'max_inflight': 500
}
@odm.model(index=False, store=False, description="Redis Service configuration")
class RedisServer(odm.Model):
host: str = odm.Keyword(description="Hostname of Redis instance")
port: int = odm.Integer(description="Port of Redis instance")
DEFAULT_REDIS_NP = {
"host": "127.0.0.1",
"port": 6379
}
DEFAULT_REDIS_P = {
"host": "127.0.0.1",
"port": 6380
}
@odm.model(index=False, store=False)
class ESMetrics(odm.Model):
hosts: List[str] = odm.Optional(odm.List(odm.Keyword()), description="Elasticsearch hosts")
host_certificates: str = odm.Optional(odm.Keyword(), description="Host certificates")
warm = odm.Integer(description="How long, per unit of time, should a document remain in the 'warm' tier?")
cold = odm.Integer(description="How long, per unit of time, should a document remain in the 'cold' tier?")
delete = odm.Integer(description="How long, per unit of time, should a document remain before being deleted?")
unit = odm.Enum(['d', 'h', 'm'], description="Unit of time used by `warm`, `cold`, `delete` phases")
DEFAULT_ES_METRICS = {
'hosts': None,
'host_certificates': None,
'warm': 2,
'cold': 30,
'delete': 90,
'unit': 'd'
}
@odm.model(index=False, store=False)
class APMServer(odm.Model):
server_url: str = odm.Optional(odm.Keyword(), description="URL to API server")
token: str = odm.Optional(odm.Keyword(), description="Authentication token for server")
DEFAULT_APM_SERVER = {
'server_url': None,
'token': None
}
@odm.model(index=False, store=False, description="Metrics Configuration")
class Metrics(odm.Model):
apm_server: APMServer = odm.Compound(APMServer, default=DEFAULT_APM_SERVER, description="APM server configuration")
elasticsearch: ESMetrics = odm.Compound(ESMetrics, default=DEFAULT_ES_METRICS,
description="Where to export metrics?")
export_interval: int = odm.Integer(description="How often should we be exporting metrics?")
redis: RedisServer = odm.Compound(RedisServer, default=DEFAULT_REDIS_NP, description="Redis for Dashboard metrics")
DEFAULT_METRICS = {
'apm_server': DEFAULT_APM_SERVER,
'elasticsearch': DEFAULT_ES_METRICS,
'export_interval': 5,
'redis': DEFAULT_REDIS_NP,
}
@odm.model(index=False, store=False, description="Malware Archive Configuration")
class ArchiverMetadata(odm.Model):
default = odm.Optional(odm.Keyword(description="Default value for the metadata"))
editable = odm.Boolean(default=False, description="Can the user provide a custom value")
values = odm.List(odm.Keyword(), default=[], description="List of possible values to pick from")
EXEMPLE_ARCHIVER_METADATA = {
'rationale': {
'default': "File is malicious",
'editable': True,
'values': ["File is malicious", "File is interesting", "I just feel like keeping this..."]
}
}
@odm.model(index=False, store=False, description="Named Value")
class NamedValue(odm.Model):
name = odm.Keyword(description="Name")
value = odm.Keyword(description="Value")
@odm.model(index=False, store=False, description="Webhook Configuration")
class Webhook(odm.Model):
password = odm.Optional(odm.Keyword(default=""), description="Password used to authenticate with source")
ca_cert = odm.Optional(odm.Keyword(default=""), description="CA cert for source")
ssl_ignore_errors = odm.Boolean(default=False, description="Ignore SSL errors when reaching out to source?")
proxy = odm.Optional(odm.Keyword(default=""), description="Proxy server for source")
method = odm.Keyword(default='POST', description="HTTP method used to access webhook")
uri = odm.Keyword(description="URI to source")
username = odm.Optional(odm.Keyword(default=""), description="Username used to authenticate with source")
headers = odm.List(odm.Compound(NamedValue), default=[], description="Headers")
retries = odm.Integer(default=3)
DEFAULT_ARCHIVER_WEBHOOK = {
'password': None,
'ca_cert': None,
'ssl_ignore_errors': False,
'proxy': None,
'method': "POST",
'uri': "https://archiving-hook",
'username': None,
'headers': [],
'retries': 3
}
@odm.model(index=False, store=False, description="Malware Archive Configuration")
class Archiver(odm.Model):
alternate_dtl: int = odm.Integer(description="Alternate number of days to keep the data in the "
"malware archive. (0: Disabled, will keep data forever)")
metadata: Dict = odm.Mapping(
odm.Compound(ArchiverMetadata),
description="Proxy configuration that is passed to Python Requests",
deprecation="The configuration for the archive metadata validation and requirements has moved to"
"`submission.metadata.archive`.")
minimum_required_services: List[str] = odm.List(
odm.keyword(),
default=[],
description="List of minimum required service before archiving takes place")
webhook = odm.Optional(odm.Compound(Webhook), description="Webhook to call before triggering the archiving process")
use_metadata: bool = odm.Boolean(
default=False, description="Should the UI ask form metadata to be filed out when archiving",
deprecation="This field is no longer required...")
use_webhook: bool = odm.Optional(odm.Boolean(
default=False,
description="Should the archiving go through the webhook prior to actually trigger the archiving function"))
DEFAULT_ARCHIVER = {
'alternate_dtl': 0,
'metadata': {},
'minimum_required_services': [],
'use_webhook': False,
'use_metadata': False,
'webhook': DEFAULT_ARCHIVER_WEBHOOK
}
@odm.model(index=False, store=False, description="Plumber Configuration")
class Plumber(odm.Model):
notification_queue_interval: int = odm.Integer(
description="Interval at which the notification queue cleanup should run")
notification_queue_max_age: int = odm.Integer(
description="Max age in seconds notification queue messages can be")
DEFAULT_PLUMBER = {
'notification_queue_interval': 30 * 60,
'notification_queue_max_age': 24 * 60 * 60
}
@odm.model(index=False, store=False, description="Redis Configuration")
class Redis(odm.Model):
nonpersistent: RedisServer = odm.Compound(RedisServer, default=DEFAULT_REDIS_NP,
description="A volatile Redis instance")
persistent: RedisServer = odm.Compound(RedisServer, default=DEFAULT_REDIS_P,
description="A persistent Redis instance")
DEFAULT_REDIS = {
"nonpersistent": DEFAULT_REDIS_NP,
"persistent": DEFAULT_REDIS_P
}
@odm.model(index=False, store=False, description="A configuration for mounting existing volumes to a container")
class Mount(odm.Model):
name: str = odm.Keyword(description="Name of volume mount")
path: str = odm.Text(description="Target mount path")
read_only: bool = odm.Boolean(default=True, description="Should this be mounted as read-only?")
privileged_only: bool = odm.Boolean(default=False,
description="Should this mount only be available for privileged services?")
# Kubernetes-specific
resource_type: str = odm.Enum(default='volume', values=['secret', 'configmap', 'volume'],
description="Type of mountable Kubernetes resource")
resource_name: str = odm.Optional(odm.Keyword(), description="Name of resource (Kubernetes only)")
resource_key: str = odm.Optional(odm.Keyword(), description="Key of ConfigMap/Secret (Kubernetes only)")
# TODO: Deprecate in next major change in favour of general configuration above for mounting Kubernetes resources
config_map: str = odm.Optional(
odm.Keyword(),
description="Name of ConfigMap (Kubernetes only)",
deprecation="Use `resource_type: configmap` and fill in the `resource_name` "
"& `resource_key` fields to mount ConfigMaps")
key: str = odm.Optional(
odm.Keyword(),
description="Key of ConfigMap (Kubernetes only)",
deprecation="Use `resource_type: configmap` and fill in the `resource_name` "
"& `resource_key` fields to mount ConfigMaps")
KUBERNETES_TOLERATION_OPS = ['Exists', 'Equal']
KUBERNETES_TOLERATION_EFFECTS = ['NoSchedule', 'PreferNoSchedule', 'NoExecute']
@odm.model(index=False, store=False, description="Limit a set of kubernetes objects based on a label query.")
class Toleration(odm.Model):
key = odm.Optional(odm.Keyword(), description="The taint key that the toleration applies to")
operator = odm.Enum(values=KUBERNETES_TOLERATION_OPS,
default="Equal", description="Relationship between taint key and value")
value = odm.Optional(odm.Keyword(), description="Taint value the toleration matches to")
effect = odm.Optional(odm.Enum(KUBERNETES_TOLERATION_EFFECTS), description="The taint effect to match.")
toleration_seconds = odm.Optional(odm.Integer(min=0),
description="The period of time the toleration tolerates the taint")
@odm.model(index=False, store=False,
description="A set of default values to be used running a service when no other value is set")
class ScalerServiceDefaults(odm.Model):
growth: int = odm.Integer(description="Period, in seconds, to wait before scaling up a service deployment")
shrink: int = odm.Integer(description="Period, in seconds, to wait before scaling down a service deployment")
backlog: int = odm.Integer(description="Backlog threshold that dictates scaling adjustments")
min_instances: int = odm.Integer(description="The minimum number of service instances to be running")
environment: List[EnvironmentVariable] = odm.List(odm.Compound(EnvironmentVariable), default=[],
description="Environment variables to pass onto services")
mounts: List[Mount] = odm.List(odm.Compound(Mount), default=[],
description="A list of volume mounts for every service")
tolerations: List[Toleration] = odm.List(
odm.Compound(Toleration),
default=[],
description="Toleration to apply to service pods.\n"
"Reference: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/")
# The operations we support for label and field selectors are based on the common subset of
# what kubernetes supports on the list_node API endpoint and the nodeAffinity field
# on pod specifications. The selector needs to work in both cases because we use these
# selectors both for probing what nodes are available (list_node) and making sure
# the pods only run on the pods that are returned there (using nodeAffinity)
@odm.model(index=False, store=False, description="Limit a set of kubernetes objects based on a field query.")
class FieldSelector(odm.Model):
key = odm.keyword(description="Name of a field to select on.")
equal = odm.boolean(default=True, description="When true key must equal value, when false it must not")
value = odm.keyword(description="Value to compare field to.")
# Excluded from this list is Gt and Lt for above reason
KUBERNETES_LABEL_OPS = ['In', 'NotIn', 'Exists', 'DoesNotExist']
@odm.model(index=False, store=False, description="Limit a set of kubernetes objects based on a label query.")
class LabelSelector(odm.Model):
key = odm.keyword(description="Name of label to select on.")
operator = odm.Enum(KUBERNETES_LABEL_OPS, description="Operation to select label with.")
values = odm.sequence(odm.keyword(), description="Value list to compare label to.")
@odm.model(index=False, store=False)
class Selector(odm.Model):
field = odm.sequence(odm.compound(FieldSelector), default=[],
description="Field selector for resource under kubernetes")
label = odm.sequence(odm.compound(LabelSelector), default=[],
description="Label selector for resource under kubernetes")
@odm.model(index=False, store=False)
class Scaler(odm.Model):
service_defaults: ScalerServiceDefaults = odm.Compound(ScalerServiceDefaults,
description="Defaults Scaler will assign to a service.")
cpu_overallocation: float = odm.Float(description="Percentage of CPU overallocation")
memory_overallocation: float = odm.Float(description="Percentage of RAM overallocation")
overallocation_node_limit = odm.Optional(odm.Integer(description="If the system has this many nodes or "
"more overallocation is ignored"))
additional_labels: List[str] = odm.Optional(
odm.List(odm.Text()), description="Additional labels to be applied to services('=' delimited)")
privileged_services_additional_labels: List[str] = odm.Optional(
odm.List(odm.Text()), description="Additional labels to be applied to privileged services only('=' delimited)")
linux_node_selector = odm.compound(Selector, description="Selector for linux nodes under kubernetes")
# windows_node_selector = odm.compound(Selector, description="Selector for windows nodes under kubernetes")
cluster_pod_list = odm.boolean(default=True, description="Sets if scaler list pods for all namespaces. "
"Disabling this lets you use stricter cluster roles but will make cluster resource "
"usage less accurate, setting a namespace resource quota might be needed.")
DEFAULT_SCALER = {
'additional_labels': None,
'cpu_overallocation': 1,
'memory_overallocation': 1,
'overallocation_node_limit': None,
'privileged_services_additional_labels': None,
'service_defaults': {
'growth': 60,
'shrink': 30,
'backlog': 100,
'min_instances': 0,
'environment': [
{'name': 'SERVICE_API_HOST', 'value': 'http://service-server:5003'},
{'name': 'AL_SERVICE_TASK_LIMIT', 'value': 'inf'},
],
},
'linux_node_selector': {
'field': [],
'label': [],
},
# 'windows_node_selector': {
# 'field': [],
# 'label': [],
# },
}
@odm.model(index=False, store=False)
class RegistryConfiguration(odm.Model):
name: str = odm.Text(description="Name of container registry")
proxies: Dict = odm.Optional(odm.Mapping(odm.Text()),
description="Proxy configuration that is passed to Python Requests")
token_server: str = odm.Optional(odm.Text(),
description="Token server name to facilitate anonymous pull access")
@odm.model(index=False, store=False)
class Updater(odm.Model):
job_dockerconfig: DockerConfigDelta = odm.Compound(
DockerConfigDelta, description="Container configuration used for service registration/updates")
registry_configs: List = odm.List(odm.Compound(RegistryConfiguration),
description="Configurations to be used with container registries")
DEFAULT_UPDATER = {
'job_dockerconfig': {
'cpu_cores': 1,
'ram_mb': 1024,
'ram_mb_min': 256,
},
'registry_configs': [{
'name': 'registry.hub.docker.com',
'proxies': {}
}]
}
@odm.model(index=False, store=False)
class VacuumSafelistItem(odm.Model):
name = odm.Keyword()
conditions = odm.Mapping(odm.Keyword())
@odm.model(index=False, store=False)
class Vacuum(odm.Model):
list_cache_directory: str = odm.Keyword()
worker_cache_directory: str = odm.Keyword()