Merge pull request #367 from CybercentreCanada/persistent-service-update

Persistent service update
CybercentreCanada · Sep 22, 2021 · c5d5d64 · c5d5d64
2 parents 64ae19d + c4eac09
commit c5d5d64
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 23 deletions.
diff --git a/assemblyline/odm/base.py b/assemblyline/odm/base.py
@@ -64,8 +64,8 @@
 logger = logging.getLogger('assemblyline.odm')
 
 
-def flat_to_nested(data: dict):
-    sub_data = {}
+def flat_to_nested(data: dict[str, typing.Any]) -> dict:
+    sub_data: dict[str, typing.Any] = {}
     nested_keys = []
     for key, value in data.items():
         if '.' in key:
@@ -88,10 +88,6 @@ class KeyMaskException(KeyError):
     pass
 
 
-class UndefinedFunction(Exception):
-    pass
-
-
 class _Field:
     def __init__(self, name=None, index=None, store=None, copyto=None, default=None):
         self.index = index
@@ -170,8 +166,8 @@ def fields(self):
         return {'': self}
 
     def check(self, value, **kwargs):
-        raise UndefinedFunction("This function is not defined in the default field. "
-                                "Each fields has to have their own definition")
+        raise NotImplementedError("This function is not defined in the default field. "
+                                  "Each fields has to have their own definition")
 
 
 class _DeletedField:
@@ -918,7 +914,7 @@ def as_primitives(self, hidden_fields=False, strip_null=False):
                 elif isinstance(value, TypedMapping):
                     out[key] = {k: v.as_primitives(strip_null=strip_null)
                                 if isinstance(v, Model) else v for k, v in value.items()}
-                elif isinstance(value, (List, TypedList)):
+                elif isinstance(value, TypedList):
                     out[key] = [v.as_primitives(strip_null=strip_null)
                                 if isinstance(v, Model) else v for v in value]
                 elif isinstance(value, ClassificationObject):

diff --git a/assemblyline/odm/models/service.py b/assemblyline/odm/models/service.py
@@ -60,10 +60,7 @@ class UpdateSource(odm.Model):
 
 @odm.model(index=False, store=False)
 class UpdateConfig(odm.Model):
-    # build_options = odm.Optional(odm.Compound(DockerfileConfig))  # If we are going to build a container, how?
     generates_signatures = odm.Boolean(index=True, default=False)
-    method = odm.Enum(values=['run', 'build'])                    # Are we going to run or build a container?
-    run_options = odm.Optional(odm.Compound(DockerConfig))        # If we are going to run a container, which one?
     sources: list[UpdateSource] = odm.List(odm.Compound(UpdateSource), default=[])    # Generic external resources we need
     update_interval_seconds: int = odm.Integer()                       # Update check interval in seconds
     wait_for_update: bool = odm.Boolean(default=False)

diff --git a/assemblyline/odm/models/service_delta.py b/assemblyline/odm/models/service_delta.py
@@ -53,10 +53,7 @@ class DependencyConfigDelta(odm.Model):
 
 @odm.model(index=False, store=False)
 class UpdateConfigDelta(odm.Model):
-    # build_options = odm.Optional(odm.Compound(DockerfileConfigDelta))
     generates_signatures = odm.Optional(odm.Boolean(), index=True)
-    method = odm.Optional(odm.Enum(values=['run', 'build']))
-    run_options = odm.Optional(odm.Compound(DockerConfigDelta))
     sources = odm.Optional(odm.List(odm.Compound(UpdateSourceDelta)))
     update_interval_seconds = odm.Optional(odm.Integer())
     wait_for_update = odm.Optional(odm.Boolean())

diff --git a/assemblyline/odm/random_data/__init__.py b/assemblyline/odm/random_data/__init__.py
@@ -92,7 +92,6 @@ def create_services(ds: AssemblylineDatastore, log=None, limit=None):
 
         if random.choice([True, False]):
             service_data['update_config'] = {
-                "method": "run",
                 "sources": [random_model_obj(UpdateSource)],
                 "update_interval_seconds": 600,
                 "generates_signatures": True

diff --git a/dev/core/docker-compose-sca-upd.yml b/dev/core/docker-compose-sca-upd.yml
@@ -4,10 +4,13 @@ services:
   al_scaler:
     image: cccs/assemblyline_dev:4.0.19
     environment:
-      FILE_UPDATE_VOLUME: /tmp/update_data
-      FILE_UPDATE_DIRECTORY: /mount/update_root/
+      DOCKER_CONFIGURATION_PATH: /mount/service_config/
+      DOCKER_CONFIGURATION_VOLUME: service_config
     volumes:
-      - /tmp/update_data:/mount/update_root/
+      - type: volume
+        source: service_config
+        target: /mount/service_config/
+        read_only: false
       - ${PATH_REWRITE:-.}/config/:/etc/assemblyline/
       - ${ROOT_REWRITE:-../../..}/:/opt/alv4/
       - /var/run/docker.sock:/var/run/docker.sock # NOTE, this container has access to docker socket (this is like root)
@@ -23,11 +26,8 @@ services:
 
   al_updater:
     image: cccs/assemblyline_dev:4.0.19
-    environment:
-      FILE_UPDATE_VOLUME: /tmp/update_data
-      FILE_UPDATE_DIRECTORY: /mount/update_root/
+    # environment:
     volumes:
-      - /tmp/update_data:/mount/update_root/
       - ${PATH_REWRITE:-.}/config/:/etc/assemblyline/
       - ${ROOT_REWRITE:-../../..}/:/opt/alv4/
       - /var/run/docker.sock:/var/run/docker.sock # NOTE, this container has access to docker socket (this is like root)
@@ -45,3 +45,7 @@ networks:
   default:
     external: true
     name: external
+
+volumes:
+  service_config:
+    name: service_config