Kubernetes development folder

dev/k8s/README.md

@@ -0,0 +1,7 @@
+# Assemblyline Dev Setup (Kubernetes)
+- Run setup script for [alv4_setup](https://github.com/cccs-sgaron/alv4_setup)
+- When ready to build, run local_dev_containers.sh script with tag as parameter.
+- Run helm upgrade using new tags in values.yaml.
+- Use Lens or kubectl to monitor status of deployment
+- You can create local service-base images by passing an optional build-arg on a docker build command otherwise will pull latest.
+  - ie. docker build . -f service-base.Dockerfile --build-arg build_no=dev0

dev/k8s/local_dev.Dockerfile

@@ -0,0 +1,52 @@
+FROM python:3.9-slim-buster
+
+ENV PYTHONPATH /opt/alv4/assemblyline-base:/opt/alv4/assemblyline-core:/opt/alv4/assemblyline-service-server:/opt/alv4/assemblyline-service-client:/opt/alv4/assemblyline-ui:/opt/alv4/assemblyline-v4-service:/opt/alv4/assemblyline-service-client
+
+# SSDEEP pkg requirments
+RUN apt-get update -yy \
+    && apt-get install -yy build-essential libffi-dev libfuzzy-dev libldap2-dev libsasl2-dev libmagic1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create Assemblyline source directory
+RUN mkdir -p /etc/assemblyline
+RUN mkdir -p /var/cache/assemblyline
+RUN mkdir -p /var/lib/assemblyline
+RUN mkdir -p /var/lib/assemblyline/flowjs
+RUN mkdir -p /var/lib/assemblyline/bundling
+RUN mkdir -p /var/log/assemblyline
+RUN mkdir -p /opt/alv4
+WORKDIR /opt/alv4
+
+# Optional package for debugging with VS Code; needed for Debug (Attach)
+# https://github.com/Azure/vscode-kubernetes-tools/blob/master/debug-on-kubernetes.md
+RUN pip install debugpy
+
+COPY assemblyline-base assemblyline-base
+RUN pip install -e ./assemblyline-base[test]
+
+COPY assemblyline-core assemblyline-core
+RUN pip install -e ./assemblyline-core[test]
+
+COPY assemblyline-ui assemblyline-ui
+RUN pip install -e ./assemblyline-ui[socketio,test]
+
+COPY assemblyline_client assemblyline_client
+RUN pip install -e ./assemblyline_client[test]
+
+COPY assemblyline-service-server assemblyline-service-server
+RUN pip install -e ./assemblyline-service-server[test]
+
+COPY assemblyline-service-client assemblyline-service-client
+RUN pip install -e ./assemblyline-service-client[test]
+
+COPY assemblyline-v4-service assemblyline-v4-service
+RUN pip install -e ./assemblyline-v4-service[test]
+
+
+RUN pip uninstall -y assemblyline
+RUN pip uninstall -y assemblyline_core
+RUN pip uninstall -y assemblyline_ui
+RUN pip uninstall -y assemblyline_service_server
+RUN pip uninstall -y assemblyline_client
+RUN pip uninstall -y assemblyline_service_client
+RUN pip uninstall -y assemblyline_v4_service

dev/k8s/local_dev_containers.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -ex
+
+# Script assumes running from context of alv4 to pull in base, core, service-server, client, ui dirs for main container build
+
+echo "Building $1"
+
+# Build & push main container
+(docker build . -t localhost:32000/cccs/assemblyline:$1 -f assemblyline-base/dev/k8s/local_dev.Dockerfile)
+(docker tag localhost:32000/cccs/assemblyline:$1 localhost:32000/cccs/assemblyline:latest)
+
+# Build core containers
+cd assemblyline-base/dev/k8s/
+(docker tag localhost:32000/cccs/assemblyline:$1 localhost:32000/cccs/assemblyline-core:$1)
+(docker build . -t localhost:32000/cccs/assemblyline-ui:$1 -f ui.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-socketio:$1 -f socketio.Dockerfile --build-arg build_no=$1)
+(docker build . -t localhost:32000/cccs/assemblyline-service-server:$1 -f service-server.Dockerfile --build-arg build_no=$1)
+
+# Push core to local registry
+(docker push localhost:32000/cccs/assemblyline-core:$1)
+(docker push localhost:32000/cccs/assemblyline-ui:$1)
+(docker push localhost:32000/cccs/assemblyline-socketio:$1)
+(docker push localhost:32000/cccs/assemblyline-service-server:$1)
+
+# Build service-base
+(docker build . -t cccs/assemblyline-v4-service-base:$1 -f service-base.Dockerfile --build-arg build_no=$1)
+(docker tag cccs/assemblyline-v4-service-base:$1 cccs/assemblyline-v4-service-base:latest)

dev/k8s/service-base.Dockerfile

@@ -0,0 +1,13 @@
+ARG build_no=latest
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+# Setup environment varibles
+ENV PYTHONPATH $PYTHONPATH:/opt/al_service
+ENV SERVICE_API_HOST http://al_service_server:5003
+ENV SERVICE_API_AUTH_KEY ThisIsARandomAuthKey...ChangeMe!
+ENV CONTAINER_MODE true
+
+RUN mkdir -p /opt/al_service
+RUN touch /opt/al_service/__init__.py
+
+CMD ["python", "-m", "debugpy", "--listen", "localhost:5678", "-m", "/opt/alv4/assemblyline-v4-service/docker/process_handler.py"]

dev/k8s/service-server.Dockerfile

@@ -0,0 +1,4 @@
+ARG build_no=latest
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+CMD ["python", "-m", "debugpy", "--listen", "localhost:5678", "-m", "gunicorn", "assemblyline_service_server.patched:app", "--config=python:assemblyline_service_server.gunicorn_config", "--worker-class", "gevent"]

dev/k8s/socketio.Dockerfile

@@ -0,0 +1,4 @@
+ARG build_no=latest
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+CMD ["python", "-m", "debugpy", "--listen", "localhost:5678", "-m", "gunicorn", "-b", ":5002", "-w", "1", "-k", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "assemblyline_ui.socketsrv:app"]

dev/k8s/ui.Dockerfile

@@ -0,0 +1,4 @@
+ARG build_no=latest
+FROM localhost:32000/cccs/assemblyline:$build_no
+
+CMD ["python", "-m", "debugpy", "--listen", "localhost:5678", "-m", "gunicorn", "assemblyline_ui.patched:app", "--config=python:assemblyline_ui.gunicorn_config", "--worker-class", "gevent"]