From d45071577b88b99c909d9c01060c809b765669b5 Mon Sep 17 00:00:00 2001 From: Adam Douglass Date: Thu, 16 Sep 2021 20:01:21 +0000 Subject: [PATCH 1/3] Remove depricated model fields --- assemblyline/odm/models/service.py | 3 --- assemblyline/odm/models/service_delta.py | 3 --- assemblyline/odm/random_data/__init__.py | 1 - 3 files changed, 7 deletions(-) diff --git a/assemblyline/odm/models/service.py b/assemblyline/odm/models/service.py index 3c2251447..d2af1edd7 100644 --- a/assemblyline/odm/models/service.py +++ b/assemblyline/odm/models/service.py @@ -60,10 +60,7 @@ class UpdateSource(odm.Model): @odm.model(index=False, store=False) class UpdateConfig(odm.Model): - # build_options = odm.Optional(odm.Compound(DockerfileConfig)) # If we are going to build a container, how? generates_signatures = odm.Boolean(index=True, default=False) - method = odm.Enum(values=['run', 'build']) # Are we going to run or build a container? - run_options = odm.Optional(odm.Compound(DockerConfig)) # If we are going to run a container, which one? sources: list[UpdateSource] = odm.List(odm.Compound(UpdateSource), default=[]) # Generic external resources we need update_interval_seconds: int = odm.Integer() # Update check interval in seconds wait_for_update: bool = odm.Boolean(default=False) diff --git a/assemblyline/odm/models/service_delta.py b/assemblyline/odm/models/service_delta.py index 501c5e2e7..2e8185a3b 100644 --- a/assemblyline/odm/models/service_delta.py +++ b/assemblyline/odm/models/service_delta.py @@ -53,10 +53,7 @@ class DependencyConfigDelta(odm.Model): @odm.model(index=False, store=False) class UpdateConfigDelta(odm.Model): - # build_options = odm.Optional(odm.Compound(DockerfileConfigDelta)) generates_signatures = odm.Optional(odm.Boolean(), index=True) - method = odm.Optional(odm.Enum(values=['run', 'build'])) - run_options = odm.Optional(odm.Compound(DockerConfigDelta)) sources = odm.Optional(odm.List(odm.Compound(UpdateSourceDelta))) update_interval_seconds = odm.Optional(odm.Integer()) wait_for_update = odm.Optional(odm.Boolean()) diff --git a/assemblyline/odm/random_data/__init__.py b/assemblyline/odm/random_data/__init__.py index debd572a2..d5ee5f1f0 100644 --- a/assemblyline/odm/random_data/__init__.py +++ b/assemblyline/odm/random_data/__init__.py @@ -92,7 +92,6 @@ def create_services(ds: AssemblylineDatastore, log=None, limit=None): if random.choice([True, False]): service_data['update_config'] = { - "method": "run", "sources": [random_model_obj(UpdateSource)], "update_interval_seconds": 600, "generates_signatures": True From 9e5538ac086e58472eafdd1ebd124fcc0f117c03 Mon Sep 17 00:00:00 2001 From: Adam Douglass Date: Tue, 21 Sep 2021 16:56:11 +0000 Subject: [PATCH 2/3] Style and linter fixes --- assemblyline/odm/base.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/assemblyline/odm/base.py b/assemblyline/odm/base.py index fb933a2ff..e1cc340f4 100644 --- a/assemblyline/odm/base.py +++ b/assemblyline/odm/base.py @@ -64,8 +64,8 @@ logger = logging.getLogger('assemblyline.odm') -def flat_to_nested(data: dict): - sub_data = {} +def flat_to_nested(data: dict[str, typing.Any]) -> dict: + sub_data: dict[str, typing.Any] = {} nested_keys = [] for key, value in data.items(): if '.' in key: @@ -88,10 +88,6 @@ class KeyMaskException(KeyError): pass -class UndefinedFunction(Exception): - pass - - class _Field: def __init__(self, name=None, index=None, store=None, copyto=None, default=None): self.index = index @@ -170,8 +166,8 @@ def fields(self): return {'': self} def check(self, value, **kwargs): - raise UndefinedFunction("This function is not defined in the default field. " - "Each fields has to have their own definition") + raise NotImplementedError("This function is not defined in the default field. " + "Each fields has to have their own definition") class _DeletedField: @@ -918,7 +914,7 @@ def as_primitives(self, hidden_fields=False, strip_null=False): elif isinstance(value, TypedMapping): out[key] = {k: v.as_primitives(strip_null=strip_null) if isinstance(v, Model) else v for k, v in value.items()} - elif isinstance(value, (List, TypedList)): + elif isinstance(value, TypedList): out[key] = [v.as_primitives(strip_null=strip_null) if isinstance(v, Model) else v for v in value] elif isinstance(value, ClassificationObject): From c4eac09ba0e2f7fa57c0854bb50c7bfb096f0e49 Mon Sep 17 00:00:00 2001 From: Adam Douglass Date: Tue, 21 Sep 2021 18:57:30 +0000 Subject: [PATCH 3/3] change deployment settings for new files --- dev/core/docker-compose-sca-upd.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dev/core/docker-compose-sca-upd.yml b/dev/core/docker-compose-sca-upd.yml index 5ce4e087f..14b9240f7 100644 --- a/dev/core/docker-compose-sca-upd.yml +++ b/dev/core/docker-compose-sca-upd.yml @@ -4,10 +4,13 @@ services: al_scaler: image: cccs/assemblyline_dev:4.0.19 environment: - FILE_UPDATE_VOLUME: /tmp/update_data - FILE_UPDATE_DIRECTORY: /mount/update_root/ + DOCKER_CONFIGURATION_PATH: /mount/service_config/ + DOCKER_CONFIGURATION_VOLUME: service_config volumes: - - /tmp/update_data:/mount/update_root/ + - type: volume + source: service_config + target: /mount/service_config/ + read_only: false - ${PATH_REWRITE:-.}/config/:/etc/assemblyline/ - ${ROOT_REWRITE:-../../..}/:/opt/alv4/ - /var/run/docker.sock:/var/run/docker.sock # NOTE, this container has access to docker socket (this is like root) @@ -23,11 +26,8 @@ services: al_updater: image: cccs/assemblyline_dev:4.0.19 - environment: - FILE_UPDATE_VOLUME: /tmp/update_data - FILE_UPDATE_DIRECTORY: /mount/update_root/ + # environment: volumes: - - /tmp/update_data:/mount/update_root/ - ${PATH_REWRITE:-.}/config/:/etc/assemblyline/ - ${ROOT_REWRITE:-../../..}/:/opt/alv4/ - /var/run/docker.sock:/var/run/docker.sock # NOTE, this container has access to docker socket (this is like root) @@ -45,3 +45,7 @@ networks: default: external: true name: external + +volumes: + service_config: + name: service_config \ No newline at end of file