From 51578b6e1464353ba48775b1aca5b96c1d8f1048 Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Tue, 3 Aug 2021 08:00:04 -0700
Subject: [PATCH 01/22] init k8s dev setup

---
 dev/k8s/README                    |  5 ++++
 dev/k8s/local_dev.Dockerfile      | 40 +++++++++++++++++++++++++++++++
 dev/k8s/local_dev_containers.sh   | 22 +++++++++++++++++
 dev/k8s/service-server.Dockerfile |  4 ++++
 dev/k8s/socketio.Dockerfile       |  4 ++++
 dev/k8s/ui.Dockerfile             |  4 ++++
 6 files changed, 79 insertions(+)
 create mode 100644 dev/k8s/README
 create mode 100644 dev/k8s/local_dev.Dockerfile
 create mode 100755 dev/k8s/local_dev_containers.sh
 create mode 100644 dev/k8s/service-server.Dockerfile
 create mode 100644 dev/k8s/socketio.Dockerfile
 create mode 100644 dev/k8s/ui.Dockerfile

diff --git a/dev/k8s/README b/dev/k8s/README
new file mode 100644
index 000000000..7fa2c42ae
--- /dev/null
+++ b/dev/k8s/README
@@ -0,0 +1,5 @@
+- use K8S appliance for setup
+- ensure registry addon for microk8s is enabled (check: GET localhost:32000/v2/_catalog)
+- run local_dev_containers.sh script with tag as param
+- run helm install|upgrade using new tag in values.yaml
+- use Lens or commandline to monitor status of deployment
diff --git a/dev/k8s/local_dev.Dockerfile b/dev/k8s/local_dev.Dockerfile
new file mode 100644
index 000000000..db750787e
--- /dev/null
+++ b/dev/k8s/local_dev.Dockerfile
@@ -0,0 +1,40 @@
+FROM python:3.9-slim-buster
+
+ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui
+
+# SSDEEP pkg requirments
+RUN apt-get update -yy \
+    && apt-get install -yy build-essential libffi-dev libfuzzy-dev libldap2-dev libsasl2-dev libmagic1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create Assemblyline source directory
+RUN mkdir -p /etc/assemblyline
+RUN mkdir -p /var/cache/assemblyline
+RUN mkdir -p /var/lib/assemblyline
+RUN mkdir -p /var/lib/assemblyline/flowjs
+RUN mkdir -p /var/lib/assemblyline/bundling
+RUN mkdir -p /var/log/assemblyline
+RUN mkdir -p /opt/alv4
+WORKDIR /opt/alv4
+
+#
+COPY assemblyline-base assemblyline-base
+RUN pip install -e ./assemblyline-base[test]
+
+COPY assemblyline-core assemblyline-core
+RUN pip install -e ./assemblyline-core[test]
+
+COPY assemblyline-ui assemblyline-ui
+RUN pip install -e ./assemblyline-ui[test]
+
+COPY assemblyline_client assemblyline_client
+RUN pip install -e ./assemblyline_client[test]
+
+COPY assemblyline-service-server assemblyline-service-server
+RUN pip install -e ./assemblyline-service-server[test]
+
+RUN pip uninstall -y assemblyline
+RUN pip uninstall -y assemblyline_core
+RUN pip uninstall -y assemblyline_ui
+RUN pip uninstall -y assemblyline_service_server
+RUN pip uninstall -y assemblyline_client
diff --git a/dev/k8s/local_dev_containers.sh b/dev/k8s/local_dev_containers.sh
new file mode 100755
index 000000000..7abb115b4
--- /dev/null
+++ b/dev/k8s/local_dev_containers.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -ex
+
+# Script assumes running from context of alv4 to pull in base, core, service-server, client, ui dirs for main container build
+
+echo "Building $1"
+
+# Build & push main container
+(docker build . -t localhost:32000/cccs/assemblyline:$1 -f assemblyline-base/dev/k8s/local_dev.Dockerfile)
+(docker push localhost:32000/cccs/assemblyline:$1)
+
+
+# Build core containers
+(docker tag localhost:32000/cccs/assemblyline:$1 localhost:32000/cccs/assemblyline-core:$1)
+(docker build . -t localhost:32000/cccs/assemblyline-ui:$1 -f assemblyline-base/dev/k8s/ui.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-socketio:$1 -f assemblyline-base/dev/k8s/socketio.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-service-server:$1 -f assemblyline-base/dev/k8s/service-server.Dockerfile --build-arg build_no=$1)
+
+# Push core local registry
+(docker push localhost:32000/cccs/assemblyline-core:$1)
+(docker push localhost:32000/cccs/assemblyline-ui:$1)
+(docker push localhost:32000/cccs/assemblyline-socketio:$1)
+(docker push localhost:32000/cccs/assemblyline-service-server:$1)
diff --git a/dev/k8s/service-server.Dockerfile b/dev/k8s/service-server.Dockerfile
new file mode 100644
index 000000000..bfae6c802
--- /dev/null
+++ b/dev/k8s/service-server.Dockerfile
@@ -0,0 +1,4 @@
+ARG build_no=dev0
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+CMD ["gunicorn", "assemblyline_service_server.patched:app", "--config=python:assemblyline_service_server.gunicorn_config", "--worker-class", "gevent"]
diff --git a/dev/k8s/socketio.Dockerfile b/dev/k8s/socketio.Dockerfile
new file mode 100644
index 000000000..8ca99d468
--- /dev/null
+++ b/dev/k8s/socketio.Dockerfile
@@ -0,0 +1,4 @@
+ARG build_no=dev0
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+CMD ["gunicorn", "-b", ":5002", "-w", "1", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "assemblyline_ui.socketsrv:app"]
diff --git a/dev/k8s/ui.Dockerfile b/dev/k8s/ui.Dockerfile
new file mode 100644
index 000000000..8d3961383
--- /dev/null
+++ b/dev/k8s/ui.Dockerfile
@@ -0,0 +1,4 @@
+ARG build_no=dev0
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+CMD ["gunicorn", "assemblyline_ui.patched:app", "--config=python:assemblyline_ui.gunicorn_config", "--worker-class", "gevent"]

From 912df90c0473e890332d18a579c9cbdf0e35504f Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Tue, 3 Aug 2021 08:21:38 -0700
Subject: [PATCH 02/22] install socketio deps

---
 dev/k8s/local_dev.Dockerfile    | 2 +-
 dev/k8s/local_dev_containers.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/k8s/local_dev.Dockerfile b/dev/k8s/local_dev.Dockerfile
index db750787e..eba7b8ac0 100644
--- a/dev/k8s/local_dev.Dockerfile
+++ b/dev/k8s/local_dev.Dockerfile
@@ -25,7 +25,7 @@ COPY assemblyline-core assemblyline-core
 RUN pip install -e ./assemblyline-core[test]
 
 COPY assemblyline-ui assemblyline-ui
-RUN pip install -e ./assemblyline-ui[test]
+RUN pip install -e ./assemblyline-ui[socketio,test]
 
 COPY assemblyline_client assemblyline_client
 RUN pip install -e ./assemblyline_client[test]
diff --git a/dev/k8s/local_dev_containers.sh b/dev/k8s/local_dev_containers.sh
index 7abb115b4..b78adda4c 100755
--- a/dev/k8s/local_dev_containers.sh
+++ b/dev/k8s/local_dev_containers.sh
@@ -15,7 +15,7 @@ echo "Building $1"
 (docker build . -t localhost:32000/cccs/assemblyline-socketio:$1 -f assemblyline-base/dev/k8s/socketio.Dockerfile --build-arg build_no=$1)
 (docker build . -t localhost:32000/cccs/assemblyline-service-server:$1 -f assemblyline-base/dev/k8s/service-server.Dockerfile --build-arg build_no=$1)
 
-# Push core local registry
+# Push core to local registry
 (docker push localhost:32000/cccs/assemblyline-core:$1)
 (docker push localhost:32000/cccs/assemblyline-ui:$1)
 (docker push localhost:32000/cccs/assemblyline-socketio:$1)

From 725d4516f24988208c7e14726a8be26ac36cca57 Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Tue, 3 Aug 2021 08:26:22 -0700
Subject: [PATCH 03/22] root context isn't necessary for dev core build

---
 dev/k8s/local_dev_containers.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/dev/k8s/local_dev_containers.sh b/dev/k8s/local_dev_containers.sh
index b78adda4c..205c3da04 100755
--- a/dev/k8s/local_dev_containers.sh
+++ b/dev/k8s/local_dev_containers.sh
@@ -10,10 +10,11 @@ echo "Building $1"
 
 
 # Build core containers
+cd assemblyline-base/dev/k8s/
 (docker tag localhost:32000/cccs/assemblyline:$1 localhost:32000/cccs/assemblyline-core:$1)
-(docker build . -t localhost:32000/cccs/assemblyline-ui:$1 -f assemblyline-base/dev/k8s/ui.Dockerfile --build-arg build_no=$1)
-(docker build . -t localhost:32000/cccs/assemblyline-socketio:$1 -f assemblyline-base/dev/k8s/socketio.Dockerfile --build-arg build_no=$1)
-(docker build . -t localhost:32000/cccs/assemblyline-service-server:$1 -f assemblyline-base/dev/k8s/service-server.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-ui:$1 -f ui.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-socketio:$1 -f socketio.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-service-server:$1 -f service-server.Dockerfile --build-arg build_no=$1)
 
 # Push core to local registry
 (docker push localhost:32000/cccs/assemblyline-core:$1)

From 2327e25941acd4edb68de1ef823e19f101c49e8e Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Tue, 3 Aug 2021 08:47:51 -0700
Subject: [PATCH 04/22] cleanup readme + link to appliance

---
 dev/k8s/README    | 5 -----
 dev/k8s/README.md | 7 +++++++
 2 files changed, 7 insertions(+), 5 deletions(-)
 delete mode 100644 dev/k8s/README
 create mode 100644 dev/k8s/README.md

diff --git a/dev/k8s/README b/dev/k8s/README
deleted file mode 100644
index 7fa2c42ae..000000000
--- a/dev/k8s/README
+++ /dev/null
@@ -1,5 +0,0 @@
-- use K8S appliance for setup
-- ensure registry addon for microk8s is enabled (check: GET localhost:32000/v2/_catalog)
-- run local_dev_containers.sh script with tag as param
-- run helm install|upgrade using new tag in values.yaml
-- use Lens or commandline to monitor status of deployment
diff --git a/dev/k8s/README.md b/dev/k8s/README.md
new file mode 100644
index 000000000..fa74a921b
--- /dev/null
+++ b/dev/k8s/README.md
@@ -0,0 +1,7 @@
+# Assemblyline Dev Setup (Kubernetes)
+- Follow steps in [K8S appliance](https://github.com/CybercentreCanada/assemblyline-helm-chart/tree/master/appliance) for local Kubernetes setup
+- Enable registry add-on for microk8s
+  - check: GET localhost:32000/v2/_catalog)
+- When ready to build, run local_dev_containers.sh script with tag as parameter
+- run helm install|upgrade using new tag in values.yaml
+- use Lens or command-line to monitor status of deployment

From 8c5b40a4a902ee8821fd8d58fe1774a5e1c066ee Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Tue, 3 Aug 2021 08:53:15 -0700
Subject: [PATCH 05/22]  add registry note

---
 dev/k8s/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/k8s/README.md b/dev/k8s/README.md
index fa74a921b..5684feb8e 100644
--- a/dev/k8s/README.md
+++ b/dev/k8s/README.md
@@ -1,7 +1,7 @@
 # Assemblyline Dev Setup (Kubernetes)
 - Follow steps in [K8S appliance](https://github.com/CybercentreCanada/assemblyline-helm-chart/tree/master/appliance) for local Kubernetes setup
-- Enable registry add-on for microk8s
-  - check: GET localhost:32000/v2/_catalog)
+- Enable registry add-on for microk8s (other registries can be used like Harbor but involves more setup which isn't covered here)
+  - Test: curl localhost:32000/v2/_catalog
 - When ready to build, run local_dev_containers.sh script with tag as parameter
-- run helm install|upgrade using new tag in values.yaml
-- use Lens or command-line to monitor status of deployment
+- Run helm install|upgrade using new tag in values.yaml
+- Use Lens or command-line to monitor status of deployment

From 2972183929c04d3ea3406a1b4d9ac7576cc68e7b Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Tue, 3 Aug 2021 11:41:52 -0700
Subject: [PATCH 06/22] add changes for dev service-base

---
 dev/k8s/local_dev.Dockerfile    | 11 ++++++++++-
 dev/k8s/service-base.Dockerfile | 14 ++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 dev/k8s/service-base.Dockerfile

diff --git a/dev/k8s/local_dev.Dockerfile b/dev/k8s/local_dev.Dockerfile
index eba7b8ac0..cff13e104 100644
--- a/dev/k8s/local_dev.Dockerfile
+++ b/dev/k8s/local_dev.Dockerfile
@@ -1,6 +1,6 @@
 FROM python:3.9-slim-buster
 
-ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui
+ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui:/opt/alv4/assemblyline-v4-service:/opt/alv4/assemblyline-service-client
 
 # SSDEEP pkg requirments
 RUN apt-get update -yy \
@@ -33,8 +33,17 @@ RUN pip install -e ./assemblyline_client[test]
 COPY assemblyline-service-server assemblyline-service-server
 RUN pip install -e ./assemblyline-service-server[test]
 
+COPY assemblyline-service-client assemblyline-service-client
+RUN pip install -e ./assemblyline-service-client[test]
+
+COPY assemblyline-v4-service assemblyline-v4-service
+RUN pip install -e ./assemblyline-v4-service[test]
+
+
 RUN pip uninstall -y assemblyline
 RUN pip uninstall -y assemblyline_core
 RUN pip uninstall -y assemblyline_ui
 RUN pip uninstall -y assemblyline_service_server
 RUN pip uninstall -y assemblyline_client
+RUN pip uninstall -y assemblyline_service_client
+RUN pip uninstall -y assemblyline_v4_service
diff --git a/dev/k8s/service-base.Dockerfile b/dev/k8s/service-base.Dockerfile
new file mode 100644
index 000000000..a486e80b5
--- /dev/null
+++ b/dev/k8s/service-base.Dockerfile
@@ -0,0 +1,14 @@
+ARG build_no=dev0
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+# Setup environment varibles
+ENV PYTHONPATH $PYTHONPATH:/opt/al_service
+ENV SERVICE_API_HOST http://al_service_server:5003
+ENV SERVICE_API_AUTH_KEY ThisIsARandomAuthKey...ChangeMe!
+ENV CONTAINER_MODE true
+
+RUN mkdir -p /opt/al_service
+RUN touch /opt/al_service/__init__.py
+
+USER assemblyline
+CMD ["python", "/opt/alv4/assemblyline-v4-service/docker/process_handler.py"]

From cf193839851d321da49c52b08b50601a5be5aad5 Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Wed, 4 Aug 2021 05:56:03 -0700
Subject: [PATCH 07/22] reference VS Code usage

---
 dev/k8s/README.md               | 6 ++++--
 dev/k8s/service-base.Dockerfile | 1 -
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/dev/k8s/README.md b/dev/k8s/README.md
index 5684feb8e..f0a0f7c45 100644
--- a/dev/k8s/README.md
+++ b/dev/k8s/README.md
@@ -1,7 +1,9 @@
 # Assemblyline Dev Setup (Kubernetes)
 - Follow steps in [K8S appliance](https://github.com/CybercentreCanada/assemblyline-helm-chart/tree/master/appliance) for local Kubernetes setup
-- Enable registry add-on for microk8s (other registries can be used like Harbor but involves more setup which isn't covered here)
+- Enable registry add-on for microK8S (other registries can be used like Harbor but involves more setup which isn't covered here)
   - Test: curl localhost:32000/v2/_catalog
 - When ready to build, run local_dev_containers.sh script with tag as parameter
 - Run helm install|upgrade using new tag in values.yaml
-- Use Lens or command-line to monitor status of deployment
+- Use Lens, command-line, or VS Code's [Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.vscode-kubernetes-tools) extension to monitor status of and/or debug deployment
+- You can create local service-base images by passing a build-arg on a docker build command
+  - ie. docker build . -f service-base.Dockerfile --build-arg dev0
diff --git a/dev/k8s/service-base.Dockerfile b/dev/k8s/service-base.Dockerfile
index a486e80b5..77a44f1ed 100644
--- a/dev/k8s/service-base.Dockerfile
+++ b/dev/k8s/service-base.Dockerfile
@@ -10,5 +10,4 @@ ENV CONTAINER_MODE true
 RUN mkdir -p /opt/al_service
 RUN touch /opt/al_service/__init__.py
 
-USER assemblyline
 CMD ["python", "/opt/alv4/assemblyline-v4-service/docker/process_handler.py"]

From 653da6bb73fbe5503ce36ea7d1be5a1307c7337b Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Wed, 4 Aug 2021 06:29:28 -0700
Subject: [PATCH 08/22] default to latest for local builds

---
 dev/k8s/README.md                 | 8 ++++----
 dev/k8s/local_dev_containers.sh   | 3 +--
 dev/k8s/service-base.Dockerfile   | 2 +-
 dev/k8s/service-server.Dockerfile | 2 +-
 dev/k8s/socketio.Dockerfile       | 2 +-
 dev/k8s/ui.Dockerfile             | 2 +-
 6 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/dev/k8s/README.md b/dev/k8s/README.md
index f0a0f7c45..5ffbce40c 100644
--- a/dev/k8s/README.md
+++ b/dev/k8s/README.md
@@ -2,8 +2,8 @@
 - Follow steps in [K8S appliance](https://github.com/CybercentreCanada/assemblyline-helm-chart/tree/master/appliance) for local Kubernetes setup
 - Enable registry add-on for microK8S (other registries can be used like Harbor but involves more setup which isn't covered here)
   - Test: curl localhost:32000/v2/_catalog
-- When ready to build, run local_dev_containers.sh script with tag as parameter
-- Run helm install|upgrade using new tag in values.yaml
+- When ready to build, run local_dev_containers.sh script with tag as parameter.
+- Run helm install|upgrade using new tags in values.yaml.
 - Use Lens, command-line, or VS Code's [Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.vscode-kubernetes-tools) extension to monitor status of and/or debug deployment
-- You can create local service-base images by passing a build-arg on a docker build command
-  - ie. docker build . -f service-base.Dockerfile --build-arg dev0
+- You can create local service-base images by passing an optional build-arg on a docker build command otherwise will pull latest.
+  - ie. docker build . -f service-base.Dockerfile --build-arg build_no=dev0
diff --git a/dev/k8s/local_dev_containers.sh b/dev/k8s/local_dev_containers.sh
index 205c3da04..3c2864303 100755
--- a/dev/k8s/local_dev_containers.sh
+++ b/dev/k8s/local_dev_containers.sh
@@ -6,8 +6,7 @@ echo "Building $1"
 
 # Build & push main container
 (docker build . -t localhost:32000/cccs/assemblyline:$1 -f assemblyline-base/dev/k8s/local_dev.Dockerfile)
-(docker push localhost:32000/cccs/assemblyline:$1)
-
+(docker tag localhost:32000/cccs/assemblyline:$1 localhost:32000/cccs/assemblyline:latest)
 
 # Build core containers
 cd assemblyline-base/dev/k8s/
diff --git a/dev/k8s/service-base.Dockerfile b/dev/k8s/service-base.Dockerfile
index 77a44f1ed..662f5e42a 100644
--- a/dev/k8s/service-base.Dockerfile
+++ b/dev/k8s/service-base.Dockerfile
@@ -1,4 +1,4 @@
-ARG build_no=dev0
+ARG build_no=latest
 FROM localhost:32000/cccs/assemblyline:$build_no
 
 # Setup environment varibles
diff --git a/dev/k8s/service-server.Dockerfile b/dev/k8s/service-server.Dockerfile
index bfae6c802..31613d223 100644
--- a/dev/k8s/service-server.Dockerfile
+++ b/dev/k8s/service-server.Dockerfile
@@ -1,4 +1,4 @@
-ARG build_no=dev0
+ARG build_no=latest
 FROM localhost:32000/cccs/assemblyline:$build_no
 
 CMD ["gunicorn", "assemblyline_service_server.patched:app", "--config=python:assemblyline_service_server.gunicorn_config", "--worker-class", "gevent"]
diff --git a/dev/k8s/socketio.Dockerfile b/dev/k8s/socketio.Dockerfile
index 8ca99d468..a4d54e4b6 100644
--- a/dev/k8s/socketio.Dockerfile
+++ b/dev/k8s/socketio.Dockerfile
@@ -1,4 +1,4 @@
-ARG build_no=dev0
+ARG build_no=latest
 FROM localhost:32000/cccs/assemblyline:$build_no
 
 CMD ["gunicorn", "-b", ":5002", "-w", "1", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "assemblyline_ui.socketsrv:app"]
diff --git a/dev/k8s/ui.Dockerfile b/dev/k8s/ui.Dockerfile
index 8d3961383..b54d7244a 100644
--- a/dev/k8s/ui.Dockerfile
+++ b/dev/k8s/ui.Dockerfile
@@ -1,4 +1,4 @@
-ARG build_no=dev0
+ARG build_no=latest
 FROM localhost:32000/cccs/assemblyline:$build_no
 
 CMD ["gunicorn", "assemblyline_ui.patched:app", "--config=python:assemblyline_ui.gunicorn_config", "--worker-class", "gevent"]

From 5c1a6ab18c23147a727630ae1e693bdea29d3259 Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Thu, 5 Aug 2021 10:05:59 -0700
Subject: [PATCH 09/22] Bridge to Kubernetes for debugging + build service

---
 dev/k8s/README.md               | 25 ++++++++++++++++++++++++-
 dev/k8s/local_dev_containers.sh |  4 ++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/dev/k8s/README.md b/dev/k8s/README.md
index 5ffbce40c..f32bb8fb8 100644
--- a/dev/k8s/README.md
+++ b/dev/k8s/README.md
@@ -4,6 +4,29 @@
   - Test: curl localhost:32000/v2/_catalog
 - When ready to build, run local_dev_containers.sh script with tag as parameter.
 - Run helm install|upgrade using new tags in values.yaml.
-- Use Lens, command-line, or VS Code's [Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.vscode-kubernetes-tools) extension to monitor status of and/or debug deployment
+- Use Lens or kubectl to monitor status of deployment
 - You can create local service-base images by passing an optional build-arg on a docker build command otherwise will pull latest.
   - ie. docker build . -f service-base.Dockerfile --build-arg build_no=dev0
+- Debugging: Visual Code's [Bridge to Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.mindaro) &
+[Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.vscode-kubernetes-tools) extensions
+  - TODO: figure out how to use it with Scaler/Updater that make calls to Kubernetes API
+  - Add to settings.json (assuming using microk8s installed from snap):
+    '''
+        "vs-kubernetes": {
+          "vs-kubernetes.namespace": "al",
+          "vs-kubernetes.kubectl-path": "/snap/kubectl/current/kubectl",
+          "vs-kubernetes.helm-path": "/snap/helm/current/helm",
+          "vs-kubernetes.minikube-path": "/snap/bin/microk8s",
+          "vs-kubernetes.kubectlVersioning": "user-provided",
+          "vs-kubernetes.outputFormat": "yaml",
+          "vs-kubernetes.kubeconfig": "/var/snap/microk8s/current/credentials/client.config",
+          "vs-kubernetes.knownKubeconfigs": [],
+          "vs-kubernetes.autoCleanupOnDebugTerminate": false,
+          "vs-kubernetes.nodejs-autodetect-remote-root": true,
+          "vs-kubernetes.nodejs-remote-root": "",
+          "vs-kubernetes.nodejs-debug-port": 9229,
+          "vs-kubernetes.local-tunnel-debug-provider": "",
+          "checkForMinikubeUpgrade": false,
+          "imageBuildTool": "Docker"
+      }
+    '''
diff --git a/dev/k8s/local_dev_containers.sh b/dev/k8s/local_dev_containers.sh
index 3c2864303..cfdd5454b 100755
--- a/dev/k8s/local_dev_containers.sh
+++ b/dev/k8s/local_dev_containers.sh
@@ -20,3 +20,7 @@ cd assemblyline-base/dev/k8s/
 (docker push localhost:32000/cccs/assemblyline-ui:$1)
 (docker push localhost:32000/cccs/assemblyline-socketio:$1)
 (docker push localhost:32000/cccs/assemblyline-service-server:$1)
+
+# Build service-base
+(docker build . -t cccs/assemblyline-v4-service-base:$1 -f service-base.Dockerfile --build-arg build_no=$1)
+(docker tag cccs/assemblyline-v4-service-base:$1 cccs/assemblyline-v4-service-base:latest)

From 54a0e665a59d3eef6b5c1b69525a7faa24d61f27 Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Thu, 5 Aug 2021 10:11:34 -0700
Subject: [PATCH 10/22] fix code block

---
 dev/k8s/README.md | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/dev/k8s/README.md b/dev/k8s/README.md
index f32bb8fb8..f7a7e83cf 100644
--- a/dev/k8s/README.md
+++ b/dev/k8s/README.md
@@ -11,22 +11,22 @@
 [Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.vscode-kubernetes-tools) extensions
   - TODO: figure out how to use it with Scaler/Updater that make calls to Kubernetes API
   - Add to settings.json (assuming using microk8s installed from snap):
-    '''
-        "vs-kubernetes": {
-          "vs-kubernetes.namespace": "al",
-          "vs-kubernetes.kubectl-path": "/snap/kubectl/current/kubectl",
-          "vs-kubernetes.helm-path": "/snap/helm/current/helm",
-          "vs-kubernetes.minikube-path": "/snap/bin/microk8s",
-          "vs-kubernetes.kubectlVersioning": "user-provided",
-          "vs-kubernetes.outputFormat": "yaml",
-          "vs-kubernetes.kubeconfig": "/var/snap/microk8s/current/credentials/client.config",
-          "vs-kubernetes.knownKubeconfigs": [],
-          "vs-kubernetes.autoCleanupOnDebugTerminate": false,
-          "vs-kubernetes.nodejs-autodetect-remote-root": true,
-          "vs-kubernetes.nodejs-remote-root": "",
-          "vs-kubernetes.nodejs-debug-port": 9229,
-          "vs-kubernetes.local-tunnel-debug-provider": "",
-          "checkForMinikubeUpgrade": false,
-          "imageBuildTool": "Docker"
-      }
-    '''
+    ```
+    "vs-kubernetes": {
+      "vs-kubernetes.namespace": "al",
+      "vs-kubernetes.kubectl-path": "/snap/kubectl/current/kubectl",
+      "vs-kubernetes.helm-path": "/snap/helm/current/helm",
+      "vs-kubernetes.minikube-path": "/snap/bin/microk8s",
+      "vs-kubernetes.kubectlVersioning": "user-provided",
+      "vs-kubernetes.outputFormat": "yaml",
+      "vs-kubernetes.kubeconfig": "/var/snap/microk8s/current/credentials/client.config",
+      "vs-kubernetes.knownKubeconfigs": [],
+      "vs-kubernetes.autoCleanupOnDebugTerminate": false,
+      "vs-kubernetes.nodejs-autodetect-remote-root": true,
+      "vs-kubernetes.nodejs-remote-root": "",
+      "vs-kubernetes.nodejs-debug-port": 9229,
+      "vs-kubernetes.local-tunnel-debug-provider": "",
+      "checkForMinikubeUpgrade": false,
+      "imageBuildTool": "Docker"
+    }
+    ```

From a621cffaaaa0bc90f2d86d81cd26e518bdacaa73 Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Thu, 5 Aug 2021 10:50:02 -0700
Subject: [PATCH 11/22] How-to run scaler/updater using k8s bridge

---
 dev/k8s/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/k8s/README.md b/dev/k8s/README.md
index f7a7e83cf..f19ede269 100644
--- a/dev/k8s/README.md
+++ b/dev/k8s/README.md
@@ -9,8 +9,7 @@
   - ie. docker build . -f service-base.Dockerfile --build-arg build_no=dev0
 - Debugging: Visual Code's [Bridge to Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.mindaro) &
 [Kubernetes](https://marketplace.visualstudio.com/items?itemName=ms-kubernetes-tools.vscode-kubernetes-tools) extensions
-  - TODO: figure out how to use it with Scaler/Updater that make calls to Kubernetes API
-  - Add to settings.json (assuming using microk8s installed from snap):
+  - Add the following to settings.json (assuming using microk8s installed from snap):
     ```
     "vs-kubernetes": {
       "vs-kubernetes.namespace": "al",
@@ -30,3 +29,4 @@
       "imageBuildTool": "Docker"
     }
     ```
+  - Specific to Updater/Scaler: You need to provide an environment variable in your launch targets called 'KUBECONFIG' that points to where your kubeconfig file is.

From dd86627b5b1f27c593901042e16e16f49c3460f9 Mon Sep 17 00:00:00 2001
From: Adam Douglass <adam.douglass@cyber.gc.ca>
Date: Mon, 19 Jul 2021 13:20:18 -0400
Subject: [PATCH 12/22] Add cachestore methods for bulk files

---
 assemblyline/cachestore/__init__.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/assemblyline/cachestore/__init__.py b/assemblyline/cachestore/__init__.py
index eeb01c8a1..1b3ca8328 100644
--- a/assemblyline/cachestore/__init__.py
+++ b/assemblyline/cachestore/__init__.py
@@ -40,12 +40,36 @@ def save(self, cache_key, data, ttl=DEFAULT_CACHE_LEN, force=False):
         self.datastore.cached_file.save(new_key, {'expiry_ts': now_as_iso(ttl), 'component': self.component})
         self.filestore.put(new_key, data, force=force)
 
-    def get(self, cache_key):
+    def upload(self, cache_key: str, path: str, ttl=DEFAULT_CACHE_LEN):
+        if not COMPONENT_VALIDATOR.match(cache_key):
+            raise ValueError("Invalid cache_key for cache item. "
+                             "(Only letters, numbers, underscores and dots allowed)")
+
+        new_key = f"{self.component}_{cache_key}" if self.component else cache_key
+
+        self.datastore.cached_file.save(new_key, {'expiry_ts': now_as_iso(ttl), 'component': self.component})
+        self.filestore.upload(new_key, path, force=True)
+
+    def touch(self, cache_key: str, ttl=DEFAULT_CACHE_LEN):
+        if not COMPONENT_VALIDATOR.match(cache_key):
+            raise ValueError("Invalid cache_key for cache item. "
+                             "(Only letters, numbers, underscores and dots allowed)")
+        if not self.exists(cache_key):
+            raise KeyError(cache_key)
+
+        new_key = f"{self.component}_{cache_key}" if self.component else cache_key
+        self.datastore.cached_file.save(new_key, {'expiry_ts': now_as_iso(ttl), 'component': self.component})
+
+    def get(self, cache_key: str) -> bytes:
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
 
         return self.filestore.get(new_key)
 
-    def exists(self, cache_key):
+    def download(self, cache_key: str, path: str):
+        new_key = f"{self.component}_{cache_key}" if self.component else cache_key
+        return self.filestore.download(new_key, path)
+
+    def exists(self, cache_key: str):
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
 
         return self.filestore.exists(new_key)

From c4ab3fe5837158d4c6da9f878e19868eb251c1a5 Mon Sep 17 00:00:00 2001
From: Adam Douglass <adam.douglass@cyber.gc.ca>
Date: Mon, 19 Jul 2021 13:23:23 -0400
Subject: [PATCH 13/22] Filestore type annotations

---
 assemblyline/cachestore/__init__.py       | 11 +++++------
 assemblyline/filestore/__init__.py        | 17 ++++++++++-------
 assemblyline/filestore/transport/azure.py |  2 +-
 assemblyline/filestore/transport/ftp.py   |  2 +-
 assemblyline/filestore/transport/http.py  |  2 +-
 assemblyline/filestore/transport/local.py |  2 +-
 assemblyline/filestore/transport/s3.py    |  2 +-
 assemblyline/filestore/transport/sftp.py  |  2 +-
 8 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/assemblyline/cachestore/__init__.py b/assemblyline/cachestore/__init__.py
index 1b3ca8328..28450908e 100644
--- a/assemblyline/cachestore/__init__.py
+++ b/assemblyline/cachestore/__init__.py
@@ -1,5 +1,6 @@
 
 import re
+from typing import AnyStr
 
 from assemblyline.common import forge
 from assemblyline.common.isotime import now_as_iso
@@ -10,7 +11,7 @@
 
 
 class CacheStore(object):
-    def __init__(self, component, config=None, datastore=None):
+    def __init__(self, component: str, config=None, datastore=None):
         if not component:
             raise ValueError("Cannot instanciate a cachestore without providing a component name.")
 
@@ -24,13 +25,13 @@ def __init__(self, component, config=None, datastore=None):
         self.datastore = datastore or forge.get_datastore(config=config)
         self.filestore = FileStore(*config.filestore.cache)
 
-    def __enter__(self):
+    def __enter__(self) -> 'CacheStore':
         return self
 
     def __exit__(self, ex_type, exc_val, exc_tb):
         self.filestore.close()
 
-    def save(self, cache_key, data, ttl=DEFAULT_CACHE_LEN, force=False):
+    def save(self, cache_key: str, data: AnyStr, ttl=DEFAULT_CACHE_LEN, force=False):
         if not COMPONENT_VALIDATOR.match(cache_key):
             raise ValueError("Invalid cache_key for cache item. "
                              "(Only letters, numbers, underscores and dots allowed)")
@@ -62,7 +63,6 @@ def touch(self, cache_key: str, ttl=DEFAULT_CACHE_LEN):
 
     def get(self, cache_key: str) -> bytes:
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
-
         return self.filestore.get(new_key)
 
     def download(self, cache_key: str, path: str):
@@ -71,10 +71,9 @@ def download(self, cache_key: str, path: str):
 
     def exists(self, cache_key: str):
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
-
         return self.filestore.exists(new_key)
 
-    def delete(self, cache_key, db_delete=True):
+    def delete(self, cache_key: str, db_delete=True):
         new_key = f"{self.component}_{cache_key}" if self.component else cache_key
 
         self.filestore.delete(new_key)
diff --git a/assemblyline/filestore/__init__.py b/assemblyline/filestore/__init__.py
index 1423734ba..306e61624 100644
--- a/assemblyline/filestore/__init__.py
+++ b/assemblyline/filestore/__init__.py
@@ -1,6 +1,7 @@
 
 import json
 import logging
+from typing import AnyStr, TYPE_CHECKING, List, Tuple
 from urllib.parse import urlparse, parse_qs, unquote
 
 import elasticapm
@@ -12,6 +13,8 @@
 from assemblyline.filestore.transport.local import TransportLocal
 from assemblyline.filestore.transport.s3 import TransportS3
 from assemblyline.filestore.transport.sftp import TransportSFTP
+if TYPE_CHECKING:
+    from assemblyline.filestore.transport.base import Transport
 
 
 class FileStoreException(Exception):
@@ -163,7 +166,7 @@ def close(self):
                 self.log.warning('Transport problem: %s', trace)
 
     @elasticapm.capture_span(span_type='filestore')
-    def delete(self, path, location='all'):
+    def delete(self, path: str, location='all'):
         with elasticapm.capture_span(name='delete', span_type='filestore', labels={'path': path}):
             for t in self.slice(location):
                 try:
@@ -173,7 +176,7 @@ def delete(self, path, location='all'):
                     self.log.info('Transport problem: %s', trace)
 
     @elasticapm.capture_span(span_type='filestore')
-    def download(self, src_path, dest_path, location='any'):
+    def download(self, src_path: str, dest_path: str, location='any'):
         successful = False
         transports = []
         download_errors = []
@@ -191,7 +194,7 @@ def download(self, src_path, dest_path, location='any'):
         return transports
 
     @elasticapm.capture_span(span_type='filestore')
-    def exists(self, path, location='any'):
+    def exists(self, path, location='any') -> List[Transport]:
         transports = []
         for t in self.slice(location):
             try:
@@ -205,7 +208,7 @@ def exists(self, path, location='any'):
         return transports
 
     @elasticapm.capture_span(span_type='filestore')
-    def get(self, path, location='any'):
+    def get(self, path: str, location='any') -> bytes:
         for t in self.slice(location):
             try:
                 if t.exists(path):
@@ -215,7 +218,7 @@ def get(self, path, location='any'):
                 self.log.warning('Transport problem: %s', trace)
 
     @elasticapm.capture_span(span_type='filestore')
-    def put(self, dst_path, content, location='all', force=False):
+    def put(self, dst_path: str, content: AnyStr, location='all', force=False) -> List[Transport]:
         transports = []
         for t in self.slice(location):
             if force or not t.exists(dst_path):
@@ -239,7 +242,7 @@ def slice(self, location):
         return transports
 
     @elasticapm.capture_span(span_type='filestore')
-    def upload(self, src_path, dst_path, location='all', force=False):
+    def upload(self, src_path: str, dst_path: str, location='all', force=False) -> List[Transport]:
         transports = []
         for t in self.slice(location):
             if force or not t.exists(dst_path):
@@ -251,7 +254,7 @@ def upload(self, src_path, dst_path, location='all', force=False):
         return transports
 
     @elasticapm.capture_span(span_type='filestore')
-    def upload_batch(self, local_remote_tuples, location='all'):
+    def upload_batch(self, local_remote_tuples, location='all') -> List[Tuple[str, str, str]]:
         failed_tuples = []
         for (src_path, dst_path) in local_remote_tuples:
             try:
diff --git a/assemblyline/filestore/transport/azure.py b/assemblyline/filestore/transport/azure.py
index 46f720d58..9ac404e60 100644
--- a/assemblyline/filestore/transport/azure.py
+++ b/assemblyline/filestore/transport/azure.py
@@ -134,7 +134,7 @@ def upload(self, src_path, dst_path):
                     raise
 
     # Buffer based functions
-    def get(self, path):
+    def get(self, path: str) -> bytes:
         key = self.normalize(path)
         my_blob = BytesIO()
 
diff --git a/assemblyline/filestore/transport/ftp.py b/assemblyline/filestore/transport/ftp.py
index 9d906e0de..4d63e2af1 100644
--- a/assemblyline/filestore/transport/ftp.py
+++ b/assemblyline/filestore/transport/ftp.py
@@ -200,7 +200,7 @@ def upload_batch(self, local_remote_tuples):
 
     # Buffer based functions
     @reconnect_retry_on_fail
-    def get(self, path) -> bytes:
+    def get(self, path: str) -> bytes:
         path = self.normalize(path)
         bio = BytesIO()
         self.ftp.retrbinary('RETR ' + path, bio.write)
diff --git a/assemblyline/filestore/transport/http.py b/assemblyline/filestore/transport/http.py
index 4128515ea..7b783bb21 100644
--- a/assemblyline/filestore/transport/http.py
+++ b/assemblyline/filestore/transport/http.py
@@ -98,7 +98,7 @@ def upload_batch(self, local_remote_tuples):
         raise TransportException("READ ONLY TRANSPORT: Method not implemented")
 
     # Buffer based functions
-    def get(self, path):
+    def get(self, path: str) -> bytes:
         path = self.normalize(path)
         resp = self.session.get(path, auth=self.auth, cert=self.pki, verify=self.verify)
         if resp.ok:
diff --git a/assemblyline/filestore/transport/local.py b/assemblyline/filestore/transport/local.py
index 27b624696..59d3c0a93 100644
--- a/assemblyline/filestore/transport/local.py
+++ b/assemblyline/filestore/transport/local.py
@@ -89,7 +89,7 @@ def upload(self, src_path, dst_path):
         assert (self.exists(dst_path))
 
     # Buffer based functions
-    def get(self, path):
+    def get(self, path: str) -> bytes:
         path = self.normalize(path)
         fh = None
         try:
diff --git a/assemblyline/filestore/transport/s3.py b/assemblyline/filestore/transport/s3.py
index de3683479..7da3f1a0e 100644
--- a/assemblyline/filestore/transport/s3.py
+++ b/assemblyline/filestore/transport/s3.py
@@ -154,7 +154,7 @@ def upload(self, src_path, dst_path):
         self.with_retries(self.client.upload_file, src_path, self.bucket, dst_path)
 
     # Buffer based functions
-    def get(self, path):
+    def get(self, path: str) -> bytes:
         fd, dst_path = tempfile.mkstemp(prefix="s3_transport.", suffix=".download")
         os.close(fd)  # We don't need the file descriptor open
 
diff --git a/assemblyline/filestore/transport/sftp.py b/assemblyline/filestore/transport/sftp.py
index c9d700fba..f76954fa2 100644
--- a/assemblyline/filestore/transport/sftp.py
+++ b/assemblyline/filestore/transport/sftp.py
@@ -140,7 +140,7 @@ def upload_batch(self, local_remote_tuples):
 
     # Buffer based functions
     @reconnect_retry_on_fail
-    def get(self, path):
+    def get(self, path: str) -> bytes:
         path = self.normalize(path)
         bio = BytesIO()
         with self.sftp.open(path) as sftp_handle:

From a9803f06a34da321f252df5b423ec501d9a242e8 Mon Sep 17 00:00:00 2001
From: Adam Douglass <adam.douglass@cyber.gc.ca>
Date: Tue, 20 Jul 2021 10:23:50 -0400
Subject: [PATCH 14/22] Add option for core dependency containers

---
 assemblyline/odm/models/service.py       | 1 +
 assemblyline/odm/models/service_delta.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/assemblyline/odm/models/service.py b/assemblyline/odm/models/service.py
index f923dd603..5d5a8f308 100644
--- a/assemblyline/odm/models/service.py
+++ b/assemblyline/odm/models/service.py
@@ -39,6 +39,7 @@ class PersistentVolume(odm.Model):
 class DependencyConfig(odm.Model):
     container = odm.Compound(DockerConfig)
     volumes = odm.Mapping(odm.Compound(PersistentVolume), default={})
+    run_as_core: bool = odm.Boolean(default=False)
 
 
 @odm.model(index=False, store=False)
diff --git a/assemblyline/odm/models/service_delta.py b/assemblyline/odm/models/service_delta.py
index 2b75fc11c..b4ac7c227 100644
--- a/assemblyline/odm/models/service_delta.py
+++ b/assemblyline/odm/models/service_delta.py
@@ -48,6 +48,7 @@ class PersistentVolumeDelta(odm.Model):
 class DependencyConfigDelta(odm.Model):
     container = odm.Optional(odm.Compound(DockerConfigDelta))
     volumes = odm.Mapping(odm.Compound(PersistentVolumeDelta), default={})
+    run_as_core: bool = odm.Optional(odm.Boolean())
 
 
 @odm.model(index=False, store=False)

From b1d243ac6c01e2e1f0f83cc7e5296b26b3be8d7f Mon Sep 17 00:00:00 2001
From: Adam <adam.douglass@cyber.gc.ca>
Date: Wed, 21 Jul 2021 15:13:44 -0400
Subject: [PATCH 15/22] typo

---
 assemblyline/cachestore/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/assemblyline/cachestore/__init__.py b/assemblyline/cachestore/__init__.py
index 28450908e..6b02c80d0 100644
--- a/assemblyline/cachestore/__init__.py
+++ b/assemblyline/cachestore/__init__.py
@@ -13,7 +13,7 @@
 class CacheStore(object):
     def __init__(self, component: str, config=None, datastore=None):
         if not component:
-            raise ValueError("Cannot instanciate a cachestore without providing a component name.")
+            raise ValueError("Cannot instantiate a cachestore without providing a component name.")
 
         if not COMPONENT_VALIDATOR.match(component):
             raise ValueError("Invalid component name. (Only letters, numbers, underscores and dots allowed)")

From 0daf8f73864de70bc2f83a7414e4938eafcd43ba Mon Sep 17 00:00:00 2001
From: Adam Douglass <Adam.Douglass@cyber.gc.ca>
Date: Mon, 9 Aug 2021 18:23:36 +0000
Subject: [PATCH 16/22] let updater use a second set of variables

---
 .gitignore                         |  1 +
 assemblyline/filestore/__init__.py |  2 ++
 assemblyline/odm/models/config.py  |  3 +++
 dev/k8s/local_dev.Dockerfile       | 19 ++++++++++++-------
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index 5414691bc..bc55d3127 100644
--- a/.gitignore
+++ b/.gitignore
@@ -84,3 +84,4 @@ venv.bak/
 
 # Cython debug symbols
 cython_debug/
+assemblyline/common/frequency.c
diff --git a/assemblyline/filestore/__init__.py b/assemblyline/filestore/__init__.py
index 306e61624..5119cf2b9 100644
--- a/assemblyline/filestore/__init__.py
+++ b/assemblyline/filestore/__init__.py
@@ -1,3 +1,4 @@
+from __future__ import annotations
 
 import json
 import logging
@@ -13,6 +14,7 @@
 from assemblyline.filestore.transport.local import TransportLocal
 from assemblyline.filestore.transport.s3 import TransportS3
 from assemblyline.filestore.transport.sftp import TransportSFTP
+
 if TYPE_CHECKING:
     from assemblyline.filestore.transport.base import Transport
 
diff --git a/assemblyline/odm/models/config.py b/assemblyline/odm/models/config.py
index 27badff6e..95dd6d97b 100644
--- a/assemblyline/odm/models/config.py
+++ b/assemblyline/odm/models/config.py
@@ -703,6 +703,8 @@ class Services(odm.Model):
     stages: List[str] = odm.List(odm.Keyword())
     # Substitution variables for image paths (for custom registry support)
     image_variables: Dict[str, str] = odm.Mapping(odm.Keyword(default=''))
+    # Same as above, but only applied in the updater, used in dev setups and local registries
+    update_image_variables: Dict[str, str] = odm.Mapping(odm.Keyword(default=''))
     # Default update channel to be used for new services
     preferred_update_channel: str = odm.Keyword()
     # Allow container registries with self signed certs for service updates
@@ -720,6 +722,7 @@ class Services(odm.Model):
     "min_service_workers": 0,
     "stages": SERVICE_STAGES,
     "image_variables": {},
+    "update_image_variables": {},
     "preferred_update_channel": "stable",
     "allow_insecure_registry": False,
     "cpu_reservation": 0.25
diff --git a/dev/k8s/local_dev.Dockerfile b/dev/k8s/local_dev.Dockerfile
index cff13e104..34bc96ec0 100644
--- a/dev/k8s/local_dev.Dockerfile
+++ b/dev/k8s/local_dev.Dockerfile
@@ -17,6 +17,15 @@ RUN mkdir -p /var/log/assemblyline
 RUN mkdir -p /opt/alv4
 WORKDIR /opt/alv4
 
+# Install and uninstall the pypi version, so that docker can cache the
+# dependency installation making repeated rebuilds with changing local changes faster
+RUN pip install assemblyline[test] assemblyline_core[test] assemblyline_ui[test,scoketio] \
+                assemblyline_client[test] assemblyline_service_server[test] \
+                assemblyline_service_client[test] assemblyline_v4_service[test] \
+    && pip uninstall -y assemblyline assemblyline_core assemblyline_ui \
+                    assemblyline_service_server assemblyline_client \
+                    assemblyline_service_client assemblyline_v4_service
+
 #
 COPY assemblyline-base assemblyline-base
 RUN pip install -e ./assemblyline-base[test]
@@ -40,10 +49,6 @@ COPY assemblyline-v4-service assemblyline-v4-service
 RUN pip install -e ./assemblyline-v4-service[test]
 
 
-RUN pip uninstall -y assemblyline
-RUN pip uninstall -y assemblyline_core
-RUN pip uninstall -y assemblyline_ui
-RUN pip uninstall -y assemblyline_service_server
-RUN pip uninstall -y assemblyline_client
-RUN pip uninstall -y assemblyline_service_client
-RUN pip uninstall -y assemblyline_v4_service
+RUN pip uninstall -y assemblyline assemblyline_core assemblyline_ui \
+                    assemblyline_service_server assemblyline_client \
+                    assemblyline_service_client assemblyline_v4_service

From 2e7240115b0fefdbd3a98a5c6c94b1df8a2144e0 Mon Sep 17 00:00:00 2001
From: Adam Douglass <Adam.Douglass@cyber.gc.ca>
Date: Fri, 13 Aug 2021 19:44:23 +0000
Subject: [PATCH 17/22] refactoring module to base

---
 assemblyline/common/server_base.py | 267 +++++++++++++++++++++++++++++
 1 file changed, 267 insertions(+)
 create mode 100644 assemblyline/common/server_base.py

diff --git a/assemblyline/common/server_base.py b/assemblyline/common/server_base.py
new file mode 100644
index 000000000..197af2762
--- /dev/null
+++ b/assemblyline/common/server_base.py
@@ -0,0 +1,267 @@
+"""
+A base classes and utilities to provide a common set of behaviours for
+the assemblyline core server nodes.
+"""
+import enum
+import functools
+import time
+import threading
+import logging
+import signal
+import sys
+import io
+import os
+from typing import cast, Dict, Callable
+
+from assemblyline.remote.datatypes import get_client
+from assemblyline.remote.datatypes.hash import Hash
+from assemblyline.odm.models.service import Service
+from assemblyline.common import forge, log as al_log
+
+
+SHUTDOWN_SECONDS_LIMIT = 10
+
+# Don't write to the heartbeat file if it hasn't been at least this many seconds since the last write.
+HEARTBEAT_TIME_LIMIT = 3
+
+
+class ServerBase(threading.Thread):
+    """Utility class for Assemblyline server processes.
+
+    Inheriting from thread so that the main work is done off the main thread.
+    This lets the main thread handle interrupts properly, even when the workload
+    makes a blocking call that would normally stop this.
+    """
+    def __init__(self, component_name: str, logger: logging.Logger = None,
+                 shutdown_timeout: float = SHUTDOWN_SECONDS_LIMIT, config=None):
+        super().__init__(name=component_name)
+        al_log.init_logging(component_name)
+        self.config = config or forge.get_config()
+
+        self.running = None
+        self.stopping = threading.Event()
+
+        self.log = logger or logging.getLogger(component_name)
+        self._exception = None
+        self._traceback = None
+        self._shutdown_timeout = shutdown_timeout if shutdown_timeout is not None else SHUTDOWN_SECONDS_LIMIT
+        self._old_sigint = None
+        self._old_sigterm = None
+        self._stopped = False
+        self._last_heartbeat = 0
+
+    def __enter__(self):
+        self.log.info(f"Initialized")
+        return self
+
+    def __exit__(self, _exc_type, _exc_val, _exc_tb):
+        if _exc_type is not None:
+            self.log.exception(f'Terminated because of an {_exc_type} exception')
+        else:
+            self.log.info(f'Terminated')
+
+    def __stop(self):
+        """Hard stop, can still be blocked in some cases, but we should try to avoid them."""
+        time.sleep(self._shutdown_timeout)
+        self.log.error(f"Server {self.__class__.__name__} has shutdown hard after waiting {self._shutdown_timeout} seconds to stop")
+
+        if not self._stopped:
+            self._stopped = True
+            exit(1)  # So any static analysis tools get the behaviour of this function 'correct'
+        import ctypes
+        ctypes.string_at(0)  # SEGFAULT out of here
+
+    def interrupt_handler(self, signum, stack_frame):
+        self.log.info(f"Instance caught signal. Coming down...")
+        self.stop()
+        if signum == signal.SIGINT and self._old_sigint:
+            self._old_sigint(signum, stack_frame)
+        if signum == signal.SIGTERM and self._old_sigterm:
+            self._old_sigterm(signum, stack_frame)
+
+    def raising_join(self):
+        self.join()
+        if self._traceback and self._exception:
+            raise self._exception.with_traceback(self._traceback)
+
+    # noinspection PyBroadException
+    def run(self):
+        try:
+            self.try_run()
+        except Exception:
+            _, self._exception, self._traceback = sys.exc_info()
+            self.log.exception("Exiting:")
+
+    def sleep(self, timeout):
+        self.stopping.wait(timeout)
+        return self.running
+        
+    def serve_forever(self):
+        self.start()
+        # We may not want to let the main thread block on a single join call. 
+        # It can interfere with signal handling.
+        while self.sleep(1):
+            pass
+
+    def start(self):
+        """Start the server workload."""
+        self.running = True
+        super().start()
+        self.log.info(f"Started")
+        self._old_sigint = signal.signal(signal.SIGINT, self.interrupt_handler)
+        self._old_sigterm = signal.signal(signal.SIGTERM, self.interrupt_handler)
+
+    def stop(self):
+        """Ask nicely for the server to stop.
+
+        After a timeout, a hard stop will be triggered.
+        """
+        # The running loops should stop within a few seconds of this flag being set.
+        self.running = False
+        self.stopping.set()
+
+        # If it doesn't stop within a few seconds, this other thread should kill the entire process
+        stop_thread = threading.Thread(target=self.__stop)
+        stop_thread.daemon = True
+        stop_thread.start()
+
+    def try_run(self):
+        pass
+
+    def heartbeat(self, timestamp: int = None):
+        """Touch a special file on disk to indicate this service is responsive.
+
+        This should be called in the main processing loop of a component, calling it in
+        a background thread defeats the purpose. Ideally it should be called at least a couple
+        times a minute.
+        """
+        if timestamp is not None:
+            timestamp = (timestamp, timestamp)
+
+        if self.config.logging.heartbeat_file:
+            # Only do the heartbeat every few seconds at most. If a fast component is
+            # calling this for every message processed we don't want to slow it down
+            # by doing a "disk" system call every few milliseconds
+            now = time.time()
+            if now - self._last_heartbeat < HEARTBEAT_TIME_LIMIT:
+                return
+            self._last_heartbeat = now
+            with io.open(self.config.logging.heartbeat_file, 'ab'):
+                os.utime(self.config.logging.heartbeat_file, times=timestamp)
+
+    def sleep_with_heartbeat(self, duration):
+        """Sleep while calling heartbeat periodically."""
+        while duration > 0:
+            self.heartbeat()
+            sleep_time = min(duration, HEARTBEAT_TIME_LIMIT * 2)
+            self.sleep(sleep_time)
+            duration -= sleep_time
+
+
+# This table in redis tells us about the current stage of operation a service is in.
+# This is complementary to the 'enabled' flag in the service spec.
+# If the service is marked as enabled=true, each component should take steps needed to move it to the 'Running' stage.
+# If the service is marked as enabled=false, each component should take steps needed to stop it.
+class ServiceStage(enum.IntEnum):
+    # A service is not running
+    # - if enabled scaler will start dependent containers and move to next stage
+    Off = 0
+    # A service is not running, but dependencies have been started
+    # - if enabled updater will try to
+    Update = 1
+    # At this stage scaler will begin
+    Running = 2
+    Paused = 3
+
+    # If at any time a service is disabled, scaler will stop the dependent containers
+
+
+def get_service_stage_hash(redis):
+    """A hash from service name to ServiceStage enum values."""
+    return Hash('service-stage', redis)
+
+
+class CoreBase(ServerBase):
+    """Expands the basic server setup in server base with some initialization steps most core servers take."""
+
+    def __init__(self, component_name: str, logger: logging.Logger = None,
+                 shutdown_timeout: float = None, config=None, datastore=None,
+                 redis=None, redis_persist=None):
+        super().__init__(component_name=component_name, logger=logger, shutdown_timeout=shutdown_timeout, config=config)
+        self.datastore = datastore or forge.get_datastore(self.config)
+
+        # Connect to all of our persistent redis structures
+        self.redis = redis or get_client(
+            host=self.config.core.redis.nonpersistent.host,
+            port=self.config.core.redis.nonpersistent.port,
+            private=False,
+        )
+        self.redis_persist = redis_persist or get_client(
+            host=self.config.core.redis.persistent.host,
+            port=self.config.core.redis.persistent.port,
+            private=False,
+        )
+
+        # Create a cached service data object, and access to the service status
+        self.service_info = cast(Dict[str, Service], forge.CachedObject(self._get_services))
+        self._service_stage_hash = get_service_stage_hash(self.redis)
+
+    def _get_services(self):
+        # noinspection PyUnresolvedReferences
+        return {x.name: x for x in self.datastore.list_all_services(full=True)}
+
+    def get_service_stage(self, service_name: str) -> ServiceStage:
+        return ServiceStage(self._service_stage_hash.get(service_name) or ServiceStage.Off)
+
+
+class ThreadedCoreBase(CoreBase):
+    def __init__(self, component_name: str, logger: logging.Logger = None,
+                 shutdown_timeout: float = None, config=None, datastore=None,
+                 redis=None, redis_persist=None):
+        super().__init__(component_name=component_name, logger=logger, shutdown_timeout=shutdown_timeout,
+                         config=config, datastore=datastore, redis=redis, redis_persist=redis_persist)
+
+        # Thread events related to exiting
+        self.main_loop_exit = threading.Event()
+
+    def stop(self):
+        super().stop()
+        self.main_loop_exit.wait(30)
+
+    def log_crashes(self, fn):
+        @functools.wraps(fn)
+        def with_logs(*args, **kwargs):
+            # noinspection PyBroadException
+            try:
+                fn(*args, **kwargs)
+            except Exception:
+                self.log.exception(f'Crash in dispatcher: {fn.__name__}')
+        return with_logs
+
+    def maintain_threads(self, expected_threads: Dict[str, Callable]):
+        expected_threads = {name: self.log_crashes(start) for name, start in expected_threads.items()}
+        threads = {}
+
+        # Run as long as we need to
+        while self.running:
+            # Check for any crashed threads
+            for name, thread in list(threads.items()):
+                if not thread.is_alive():
+                    self.log.warning(f'Restarting thread: {name}')
+                    threads.pop(name)
+
+            # Start any missing threads
+            for name, function in expected_threads.items():
+                if name not in threads:
+                    self.log.info(f'Starting thread: {name}')
+                    threads[name] = thread = threading.Thread(target=function, name=name)
+                    thread.start()
+
+            # Take a break before doing it again
+            super().heartbeat()
+            self.sleep(2)
+
+        for _t in threads.values():
+            _t.join()
+
+        self.main_loop_exit.set()

From 8c363d96577c5de5bd9409c74b53243fb88fc87b Mon Sep 17 00:00:00 2001
From: Adam Douglass <Adam.Douglass@cyber.gc.ca>
Date: Fri, 13 Aug 2021 19:45:03 +0000
Subject: [PATCH 18/22] changes for building services on dev containers

---
 dev/k8s/local_dev.Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dev/k8s/local_dev.Dockerfile b/dev/k8s/local_dev.Dockerfile
index 34bc96ec0..7f958b49d 100644
--- a/dev/k8s/local_dev.Dockerfile
+++ b/dev/k8s/local_dev.Dockerfile
@@ -1,6 +1,6 @@
 FROM python:3.9-slim-buster
 
-ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui:/opt/alv4/assemblyline-v4-service:/opt/alv4/assemblyline-service-client
+ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui:/opt/alv4/assemblyline_client:/opt/alv4/assemblyline-v4-service:/opt/alv4/assemblyline-service-client
 
 # SSDEEP pkg requirments
 RUN apt-get update -yy \
@@ -8,6 +8,7 @@ RUN apt-get update -yy \
     && rm -rf /var/lib/apt/lists/*
 
 # Create Assemblyline source directory
+RUN useradd -b /var/lib -U -m assemblyline
 RUN mkdir -p /etc/assemblyline
 RUN mkdir -p /var/cache/assemblyline
 RUN mkdir -p /var/lib/assemblyline

From 8b34d95e3f5820605091ad4f55646686dc11c70c Mon Sep 17 00:00:00 2001
From: Adam Douglass <Adam.Douglass@cyber.gc.ca>
Date: Wed, 25 Aug 2021 21:09:01 +0000
Subject: [PATCH 19/22] Add type annotations

---
 assemblyline/common/server_base.py | 14 ++++++++-----
 assemblyline/datastore/helper.py   |  6 +++---
 assemblyline/odm/models/service.py | 32 +++++++++++++++---------------
 3 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/assemblyline/common/server_base.py b/assemblyline/common/server_base.py
index 197af2762..90d93318c 100644
--- a/assemblyline/common/server_base.py
+++ b/assemblyline/common/server_base.py
@@ -2,6 +2,7 @@
 A base classes and utilities to provide a common set of behaviours for
 the assemblyline core server nodes.
 """
+from __future__ import annotations
 import enum
 import functools
 import time
@@ -11,13 +12,16 @@
 import sys
 import io
 import os
-from typing import cast, Dict, Callable
+from typing import cast, Callable, TYPE_CHECKING
 
 from assemblyline.remote.datatypes import get_client
 from assemblyline.remote.datatypes.hash import Hash
 from assemblyline.odm.models.service import Service
 from assemblyline.common import forge, log as al_log
 
+if TYPE_CHECKING:
+    from assemblyline.datastore.helper import AssemblylineDatastore
+
 
 SHUTDOWN_SECONDS_LIMIT = 10
 
@@ -92,7 +96,7 @@ def run(self):
             _, self._exception, self._traceback = sys.exc_info()
             self.log.exception("Exiting:")
 
-    def sleep(self, timeout):
+    def sleep(self, timeout: float):
         self.stopping.wait(timeout)
         return self.running
         
@@ -188,7 +192,7 @@ def __init__(self, component_name: str, logger: logging.Logger = None,
                  shutdown_timeout: float = None, config=None, datastore=None,
                  redis=None, redis_persist=None):
         super().__init__(component_name=component_name, logger=logger, shutdown_timeout=shutdown_timeout, config=config)
-        self.datastore = datastore or forge.get_datastore(self.config)
+        self.datastore: AssemblylineDatastore = datastore or forge.get_datastore(self.config)
 
         # Connect to all of our persistent redis structures
         self.redis = redis or get_client(
@@ -203,7 +207,7 @@ def __init__(self, component_name: str, logger: logging.Logger = None,
         )
 
         # Create a cached service data object, and access to the service status
-        self.service_info = cast(Dict[str, Service], forge.CachedObject(self._get_services))
+        self.service_info = cast(dict[str, Service], forge.CachedObject(self._get_services))
         self._service_stage_hash = get_service_stage_hash(self.redis)
 
     def _get_services(self):
@@ -238,7 +242,7 @@ def with_logs(*args, **kwargs):
                 self.log.exception(f'Crash in dispatcher: {fn.__name__}')
         return with_logs
 
-    def maintain_threads(self, expected_threads: Dict[str, Callable]):
+    def maintain_threads(self, expected_threads: dict[str, Callable[..., None]]):
         expected_threads = {name: self.log_crashes(start) for name, start in expected_threads.items()}
         threads = {}
 
diff --git a/assemblyline/datastore/helper.py b/assemblyline/datastore/helper.py
index d4a550aad..d5d50bd6c 100644
--- a/assemblyline/datastore/helper.py
+++ b/assemblyline/datastore/helper.py
@@ -916,17 +916,17 @@ def get_attack_matrix_from_keys(self, keys):
         return out
 
     @elasticapm.capture_span(span_type='datastore')
-    def get_service_with_delta(self, service_name, version=None, as_obj=True):
+    def get_service_with_delta(self, service_name, version=None, as_obj=True) -> Union[Service, dict, None]:
         svc = self.ds.service_delta.get(service_name)
         if svc is None:
-            return svc
+            return None
 
         if version is not None:
             svc.version = version
 
         svc_version_data = self.ds.service.get(f"{service_name}_{svc.version}")
         if svc_version_data is None:
-            return svc_version_data
+            return None
 
         svc_version_data = recursive_update(svc_version_data.as_primitives(strip_null=True),
                                             svc.as_primitives(strip_null=True))
diff --git a/assemblyline/odm/models/service.py b/assemblyline/odm/models/service.py
index 5d5a8f308..a38b472d6 100644
--- a/assemblyline/odm/models/service.py
+++ b/assemblyline/odm/models/service.py
@@ -9,8 +9,8 @@
 
 @odm.model(index=False, store=False)
 class EnvironmentVariable(odm.Model):
-    name = odm.Keyword()
-    value = odm.Keyword()
+    name: str = odm.Keyword()
+    value: str = odm.Keyword()
 
 
 @odm.model(index=False, store=False)
@@ -44,16 +44,16 @@ class DependencyConfig(odm.Model):
 
 @odm.model(index=False, store=False)
 class UpdateSource(odm.Model):
-    name = odm.Keyword()
-    password = odm.Optional(odm.Keyword(default=""))
-    pattern = odm.Optional(odm.Keyword(default=""))
-    private_key = odm.Optional(odm.Keyword(default=""))
-    ca_cert = odm.Optional(odm.Keyword(default=""))
-    ssl_ignore_errors = odm.Boolean(default=False)
-    proxy = odm.Optional(odm.Keyword(default=""))
-    uri = odm.Keyword()
-    username = odm.Optional(odm.Keyword(default=""))
-    headers = odm.List(odm.Compound(EnvironmentVariable), default=[])
+    name: str = odm.Keyword()
+    password: Opt[str] = odm.Optional(odm.Keyword(default=""))
+    pattern: Opt[str] = odm.Optional(odm.Keyword(default=""))
+    private_key: Opt[str] = odm.Optional(odm.Keyword(default=""))
+    ca_cert: Opt[str] = odm.Optional(odm.Keyword(default=""))
+    ssl_ignore_errors: bool = odm.Boolean(default=False)
+    proxy: Opt[str] = odm.Optional(odm.Keyword(default=""))
+    uri: str = odm.Keyword()
+    username: Opt[str] = odm.Optional(odm.Keyword(default=""))
+    headers: list[EnvironmentVariable] = odm.List(odm.Compound(EnvironmentVariable), default=[])
     default_classification = odm.Classification(default=Classification.UNRESTRICTED)
 
 
@@ -63,9 +63,9 @@ class UpdateConfig(odm.Model):
     generates_signatures = odm.Boolean(index=True, default=False)
     method = odm.Enum(values=['run', 'build'])                    # Are we going to run or build a container?
     run_options = odm.Optional(odm.Compound(DockerConfig))        # If we are going to run a container, which one?
-    sources = odm.List(odm.Compound(UpdateSource), default=[])    # Generic external resources we need
-    update_interval_seconds = odm.Integer()                       # Update check interval in seconds
-    wait_for_update = odm.Boolean(default=False)
+    sources: list[UpdateSource] = odm.List(odm.Compound(UpdateSource), default=[])    # Generic external resources we need
+    update_interval_seconds: int = odm.Integer()                       # Update check interval in seconds
+    wait_for_update: bool = odm.Boolean(default=False)
 
 
 @odm.model(index=False, store=False)
@@ -91,7 +91,7 @@ class Service(odm.Model):
     is_external = odm.Boolean(default=False)
     licence_count = odm.Integer(default=0)
 
-    name = odm.Keyword(store=True, copyto="__text__")
+    name: str = odm.Keyword(store=True, copyto="__text__")
     version = odm.Keyword(store=True)
 
     # Should the result cache be disabled for this service

From 065e8367c08b12ffd737a87449d1409f481930b6 Mon Sep 17 00:00:00 2001
From: Adam Douglass <Adam.Douglass@cyber.gc.ca>
Date: Wed, 25 Aug 2021 21:09:20 +0000
Subject: [PATCH 20/22] Changes to dev setup

---
 dev/k8s/local_dev.Dockerfile    | 10 +++++-----
 dev/k8s/service-base.Dockerfile |  1 +
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/dev/k8s/local_dev.Dockerfile b/dev/k8s/local_dev.Dockerfile
index 7f958b49d..3a765cf81 100644
--- a/dev/k8s/local_dev.Dockerfile
+++ b/dev/k8s/local_dev.Dockerfile
@@ -1,6 +1,6 @@
 FROM python:3.9-slim-buster
 
-ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui:/opt/alv4/assemblyline_client:/opt/alv4/assemblyline-v4-service:/opt/alv4/assemblyline-service-client
+ENV PYTHONPATH=${PYTHONPATH}:/var/lib/assemblyline/.local/lib/python3.9/site-packages/:/opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui:/opt/alv4/assemblyline_client:/opt/alv4/assemblyline-v4-service:/opt/alv4/assemblyline-service-client
 
 # SSDEEP pkg requirments
 RUN apt-get update -yy \
@@ -15,8 +15,8 @@ RUN mkdir -p /var/lib/assemblyline
 RUN mkdir -p /var/lib/assemblyline/flowjs
 RUN mkdir -p /var/lib/assemblyline/bundling
 RUN mkdir -p /var/log/assemblyline
-RUN mkdir -p /opt/alv4
 WORKDIR /opt/alv4
+ENV PATH=/var/lib/assemblyline/.local/bin:$PATH
 
 # Install and uninstall the pypi version, so that docker can cache the
 # dependency installation making repeated rebuilds with changing local changes faster
@@ -50,6 +50,6 @@ COPY assemblyline-v4-service assemblyline-v4-service
 RUN pip install -e ./assemblyline-v4-service[test]
 
 
-RUN pip uninstall -y assemblyline assemblyline_core assemblyline_ui \
-                    assemblyline_service_server assemblyline_client \
-                    assemblyline_service_client assemblyline_v4_service
+# RUN pip uninstall -y assemblyline assemblyline_core assemblyline_ui \
+#                     assemblyline_service_server assemblyline_client \
+#                     assemblyline_service_client assemblyline_v4_service
\ No newline at end of file
diff --git a/dev/k8s/service-base.Dockerfile b/dev/k8s/service-base.Dockerfile
index 662f5e42a..8182d5ed9 100644
--- a/dev/k8s/service-base.Dockerfile
+++ b/dev/k8s/service-base.Dockerfile
@@ -9,5 +9,6 @@ ENV CONTAINER_MODE true
 
 RUN mkdir -p /opt/al_service
 RUN touch /opt/al_service/__init__.py
+USER assemblyline
 
 CMD ["python", "/opt/alv4/assemblyline-v4-service/docker/process_handler.py"]

From cf728e858c6cf936aa234664cc0dfc7b793fb64a Mon Sep 17 00:00:00 2001
From: Adam Douglass <Adam.Douglass@cyber.gc.ca>
Date: Thu, 26 Aug 2021 15:16:14 +0000
Subject: [PATCH 21/22] Undo refactor moving service base

---
 assemblyline/common/server_base.py | 271 -----------------------------
 1 file changed, 271 deletions(-)
 delete mode 100644 assemblyline/common/server_base.py

diff --git a/assemblyline/common/server_base.py b/assemblyline/common/server_base.py
deleted file mode 100644
index 90d93318c..000000000
--- a/assemblyline/common/server_base.py
+++ /dev/null
@@ -1,271 +0,0 @@
-"""
-A base classes and utilities to provide a common set of behaviours for
-the assemblyline core server nodes.
-"""
-from __future__ import annotations
-import enum
-import functools
-import time
-import threading
-import logging
-import signal
-import sys
-import io
-import os
-from typing import cast, Callable, TYPE_CHECKING
-
-from assemblyline.remote.datatypes import get_client
-from assemblyline.remote.datatypes.hash import Hash
-from assemblyline.odm.models.service import Service
-from assemblyline.common import forge, log as al_log
-
-if TYPE_CHECKING:
-    from assemblyline.datastore.helper import AssemblylineDatastore
-
-
-SHUTDOWN_SECONDS_LIMIT = 10
-
-# Don't write to the heartbeat file if it hasn't been at least this many seconds since the last write.
-HEARTBEAT_TIME_LIMIT = 3
-
-
-class ServerBase(threading.Thread):
-    """Utility class for Assemblyline server processes.
-
-    Inheriting from thread so that the main work is done off the main thread.
-    This lets the main thread handle interrupts properly, even when the workload
-    makes a blocking call that would normally stop this.
-    """
-    def __init__(self, component_name: str, logger: logging.Logger = None,
-                 shutdown_timeout: float = SHUTDOWN_SECONDS_LIMIT, config=None):
-        super().__init__(name=component_name)
-        al_log.init_logging(component_name)
-        self.config = config or forge.get_config()
-
-        self.running = None
-        self.stopping = threading.Event()
-
-        self.log = logger or logging.getLogger(component_name)
-        self._exception = None
-        self._traceback = None
-        self._shutdown_timeout = shutdown_timeout if shutdown_timeout is not None else SHUTDOWN_SECONDS_LIMIT
-        self._old_sigint = None
-        self._old_sigterm = None
-        self._stopped = False
-        self._last_heartbeat = 0
-
-    def __enter__(self):
-        self.log.info(f"Initialized")
-        return self
-
-    def __exit__(self, _exc_type, _exc_val, _exc_tb):
-        if _exc_type is not None:
-            self.log.exception(f'Terminated because of an {_exc_type} exception')
-        else:
-            self.log.info(f'Terminated')
-
-    def __stop(self):
-        """Hard stop, can still be blocked in some cases, but we should try to avoid them."""
-        time.sleep(self._shutdown_timeout)
-        self.log.error(f"Server {self.__class__.__name__} has shutdown hard after waiting {self._shutdown_timeout} seconds to stop")
-
-        if not self._stopped:
-            self._stopped = True
-            exit(1)  # So any static analysis tools get the behaviour of this function 'correct'
-        import ctypes
-        ctypes.string_at(0)  # SEGFAULT out of here
-
-    def interrupt_handler(self, signum, stack_frame):
-        self.log.info(f"Instance caught signal. Coming down...")
-        self.stop()
-        if signum == signal.SIGINT and self._old_sigint:
-            self._old_sigint(signum, stack_frame)
-        if signum == signal.SIGTERM and self._old_sigterm:
-            self._old_sigterm(signum, stack_frame)
-
-    def raising_join(self):
-        self.join()
-        if self._traceback and self._exception:
-            raise self._exception.with_traceback(self._traceback)
-
-    # noinspection PyBroadException
-    def run(self):
-        try:
-            self.try_run()
-        except Exception:
-            _, self._exception, self._traceback = sys.exc_info()
-            self.log.exception("Exiting:")
-
-    def sleep(self, timeout: float):
-        self.stopping.wait(timeout)
-        return self.running
-        
-    def serve_forever(self):
-        self.start()
-        # We may not want to let the main thread block on a single join call. 
-        # It can interfere with signal handling.
-        while self.sleep(1):
-            pass
-
-    def start(self):
-        """Start the server workload."""
-        self.running = True
-        super().start()
-        self.log.info(f"Started")
-        self._old_sigint = signal.signal(signal.SIGINT, self.interrupt_handler)
-        self._old_sigterm = signal.signal(signal.SIGTERM, self.interrupt_handler)
-
-    def stop(self):
-        """Ask nicely for the server to stop.
-
-        After a timeout, a hard stop will be triggered.
-        """
-        # The running loops should stop within a few seconds of this flag being set.
-        self.running = False
-        self.stopping.set()
-
-        # If it doesn't stop within a few seconds, this other thread should kill the entire process
-        stop_thread = threading.Thread(target=self.__stop)
-        stop_thread.daemon = True
-        stop_thread.start()
-
-    def try_run(self):
-        pass
-
-    def heartbeat(self, timestamp: int = None):
-        """Touch a special file on disk to indicate this service is responsive.
-
-        This should be called in the main processing loop of a component, calling it in
-        a background thread defeats the purpose. Ideally it should be called at least a couple
-        times a minute.
-        """
-        if timestamp is not None:
-            timestamp = (timestamp, timestamp)
-
-        if self.config.logging.heartbeat_file:
-            # Only do the heartbeat every few seconds at most. If a fast component is
-            # calling this for every message processed we don't want to slow it down
-            # by doing a "disk" system call every few milliseconds
-            now = time.time()
-            if now - self._last_heartbeat < HEARTBEAT_TIME_LIMIT:
-                return
-            self._last_heartbeat = now
-            with io.open(self.config.logging.heartbeat_file, 'ab'):
-                os.utime(self.config.logging.heartbeat_file, times=timestamp)
-
-    def sleep_with_heartbeat(self, duration):
-        """Sleep while calling heartbeat periodically."""
-        while duration > 0:
-            self.heartbeat()
-            sleep_time = min(duration, HEARTBEAT_TIME_LIMIT * 2)
-            self.sleep(sleep_time)
-            duration -= sleep_time
-
-
-# This table in redis tells us about the current stage of operation a service is in.
-# This is complementary to the 'enabled' flag in the service spec.
-# If the service is marked as enabled=true, each component should take steps needed to move it to the 'Running' stage.
-# If the service is marked as enabled=false, each component should take steps needed to stop it.
-class ServiceStage(enum.IntEnum):
-    # A service is not running
-    # - if enabled scaler will start dependent containers and move to next stage
-    Off = 0
-    # A service is not running, but dependencies have been started
-    # - if enabled updater will try to
-    Update = 1
-    # At this stage scaler will begin
-    Running = 2
-    Paused = 3
-
-    # If at any time a service is disabled, scaler will stop the dependent containers
-
-
-def get_service_stage_hash(redis):
-    """A hash from service name to ServiceStage enum values."""
-    return Hash('service-stage', redis)
-
-
-class CoreBase(ServerBase):
-    """Expands the basic server setup in server base with some initialization steps most core servers take."""
-
-    def __init__(self, component_name: str, logger: logging.Logger = None,
-                 shutdown_timeout: float = None, config=None, datastore=None,
-                 redis=None, redis_persist=None):
-        super().__init__(component_name=component_name, logger=logger, shutdown_timeout=shutdown_timeout, config=config)
-        self.datastore: AssemblylineDatastore = datastore or forge.get_datastore(self.config)
-
-        # Connect to all of our persistent redis structures
-        self.redis = redis or get_client(
-            host=self.config.core.redis.nonpersistent.host,
-            port=self.config.core.redis.nonpersistent.port,
-            private=False,
-        )
-        self.redis_persist = redis_persist or get_client(
-            host=self.config.core.redis.persistent.host,
-            port=self.config.core.redis.persistent.port,
-            private=False,
-        )
-
-        # Create a cached service data object, and access to the service status
-        self.service_info = cast(dict[str, Service], forge.CachedObject(self._get_services))
-        self._service_stage_hash = get_service_stage_hash(self.redis)
-
-    def _get_services(self):
-        # noinspection PyUnresolvedReferences
-        return {x.name: x for x in self.datastore.list_all_services(full=True)}
-
-    def get_service_stage(self, service_name: str) -> ServiceStage:
-        return ServiceStage(self._service_stage_hash.get(service_name) or ServiceStage.Off)
-
-
-class ThreadedCoreBase(CoreBase):
-    def __init__(self, component_name: str, logger: logging.Logger = None,
-                 shutdown_timeout: float = None, config=None, datastore=None,
-                 redis=None, redis_persist=None):
-        super().__init__(component_name=component_name, logger=logger, shutdown_timeout=shutdown_timeout,
-                         config=config, datastore=datastore, redis=redis, redis_persist=redis_persist)
-
-        # Thread events related to exiting
-        self.main_loop_exit = threading.Event()
-
-    def stop(self):
-        super().stop()
-        self.main_loop_exit.wait(30)
-
-    def log_crashes(self, fn):
-        @functools.wraps(fn)
-        def with_logs(*args, **kwargs):
-            # noinspection PyBroadException
-            try:
-                fn(*args, **kwargs)
-            except Exception:
-                self.log.exception(f'Crash in dispatcher: {fn.__name__}')
-        return with_logs
-
-    def maintain_threads(self, expected_threads: dict[str, Callable[..., None]]):
-        expected_threads = {name: self.log_crashes(start) for name, start in expected_threads.items()}
-        threads = {}
-
-        # Run as long as we need to
-        while self.running:
-            # Check for any crashed threads
-            for name, thread in list(threads.items()):
-                if not thread.is_alive():
-                    self.log.warning(f'Restarting thread: {name}')
-                    threads.pop(name)
-
-            # Start any missing threads
-            for name, function in expected_threads.items():
-                if name not in threads:
-                    self.log.info(f'Starting thread: {name}')
-                    threads[name] = thread = threading.Thread(target=function, name=name)
-                    thread.start()
-
-            # Take a break before doing it again
-            super().heartbeat()
-            self.sleep(2)
-
-        for _t in threads.values():
-            _t.join()
-
-        self.main_loop_exit.set()

From a70633e868b78e24fa5a3588849ef728473be46a Mon Sep 17 00:00:00 2001
From: cccs-rs <62077998+cccs-rs@users.noreply.github.com>
Date: Mon, 30 Aug 2021 14:26:33 -0700
Subject: [PATCH 22/22] include assemblyline_client for dev

---
 docker/al_dev/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/al_dev/Dockerfile b/docker/al_dev/Dockerfile
index d8ee3a0b9..903663874 100644
--- a/docker/al_dev/Dockerfile
+++ b/docker/al_dev/Dockerfile
@@ -3,7 +3,7 @@ FROM python:3.9-slim-buster
 # assemblyline-core, assemblyline-service-server and assemblyline-service-client code is checked out
 
 # Setup environment varibles
-ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui
+ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline_client:/opt/alv4/assemblyline-ui
 
 # SSDEEP pkg requirments
 RUN apt-get update && apt-get install -yy build-essential libffi-dev libfuzzy-dev libldap2-dev libsasl2-dev libmagic1 && rm -rf /var/lib/apt/lists/*