From cf9116363afc639d0f3426b8a137301903bf89cd Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 18 Sep 2020 14:40:12 +0200 Subject: [PATCH] #296 Improve docker image --- .../cortex/services/DockerJobRunnerSrv.scala | 12 +- .../thp/cortex/services/JobRunnerSrv.scala | 3 +- conf/reference.conf | 2 + docker.sbt | 10 +- docker/cortex/.env | 1 + docker/cortex/docker-compose.yml | 10 +- package/docker/entrypoint | 228 ++++++++++-------- 7 files changed, 162 insertions(+), 104 deletions(-) create mode 100644 docker/cortex/.env diff --git a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala index df3424a0e..5931cac12 100644 --- a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala +++ b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala @@ -21,7 +21,13 @@ import org.thp.cortex.models._ import org.elastic4play.utils.RichFuture @Singleton -class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val system: ActorSystem) { +class DockerJobRunnerSrv( + client: DockerClient, + autoUpdate: Boolean, + jobBaseDirectory: Path, + dockerJobBaseDirectory: Path, + implicit val system: ActorSystem +) { @Inject() def this(config: Configuration, system: ActorSystem) = @@ -37,6 +43,8 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val .useProxy(config.getOptional[Boolean]("docker.useProxy").getOrElse(false)) .build(), config.getOptional[Boolean]("docker.autoUpdate").getOrElse(true), + Paths.get(config.get[String]("job.directory")), + Paths.get(config.get[String]("job.dockerDirectory")), system: ActorSystem ) @@ -60,7 +68,7 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val .builder() .appendBinds( Bind - .from(jobDirectory.toAbsolutePath.toString) + .from(dockerJobBaseDirectory.resolve(jobBaseDirectory.relativize(jobDirectory)).toAbsolutePath.toString) .to("/job") .readOnly(false) .build() diff --git a/app/org/thp/cortex/services/JobRunnerSrv.scala b/app/org/thp/cortex/services/JobRunnerSrv.scala index b939f56f1..018215d43 100644 --- a/app/org/thp/cortex/services/JobRunnerSrv.scala +++ b/app/org/thp/cortex/services/JobRunnerSrv.scala @@ -42,6 +42,7 @@ class JobRunnerSrv @Inject() ( val logger = Logger(getClass) lazy val analyzerExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") lazy val responderExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("responder") + val jobDirectory: Path = Paths.get(config.get[String]("job.directory")) private val runners: Seq[String] = config .getOptional[Seq[String]]("job.runners") @@ -89,7 +90,7 @@ class JobRunnerSrv @Inject() ( } private def prepareJobFolder(worker: Worker, job: Job): Future[Path] = { - val jobFolder = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), s"cortex-job-${job.id}-") + val jobFolder = Files.createTempDirectory(jobDirectory, s"cortex-job-${job.id}-") val inputJobFolder = Files.createDirectories(jobFolder.resolve("input")) Files.createDirectories(jobFolder.resolve("output")) diff --git a/conf/reference.conf b/conf/reference.conf index bd04bfaf1..7f457257b 100644 --- a/conf/reference.conf +++ b/conf/reference.conf @@ -12,6 +12,8 @@ cache { job { timeout = 30 minutes runners = [docker, process] + directory = ${java.io.tmpdir} + dockerDirectory = ${job.directory} } # HTTP filters diff --git a/docker.sbt b/docker.sbt index 9b3cfc801..b00495861 100644 --- a/docker.sbt +++ b/docker.sbt @@ -29,8 +29,16 @@ dockerCommands := Seq( Cmd("WORKDIR", "/opt/cortex"), // format: off Cmd("RUN", + "wget", "-q", "-O", "-", "https://download.docker.com/linux/static/stable/x86_64/docker-18.09.0.tgz", "|", + "tar", "-xzC", "/usr/local/bin/", "--strip-components", "1", "&&", + "addgroup", "--system", "dockremap", "&&", + "adduser", "--system", "--ingroup", "dockremap", "dockremap", "&&", + "addgroup", "--system", "docker", "&&", + "echo", "dockremap:165536:65536", ">>", "/etc/subuid", "&&", + "echo", "dockremap:165536:65536", ">>", "/etc/subgid", "&&", "apt", "update", "&&", "apt", "upgrade", "-y", "&&", + "apt", "install", "-y", "iptables", "lxc", "&&", "apt", "autoclean", "-y", "-q", "&&", "apt", "autoremove", "-y", "-q", "&&", "rm", "-rf", "/var/lib/apt/lists/*", "&&", @@ -46,9 +54,9 @@ dockerCommands := Seq( Cmd("ADD", "--chown=root:root", "opt", "/opt"), Cmd("ADD", "--chown=cortex:cortex", "var", "/var"), Cmd("ADD", "--chown=cortex:cortex", "etc", "/etc"), + Cmd("VOLUME", "/var/lib/docker"), ExecCmd("RUN", "chmod", "+x", "/opt/cortex/bin/cortex", "/opt/cortex/entrypoint"), Cmd("EXPOSE", "9001"), - Cmd("USER", "cortex"), ExecCmd("ENTRYPOINT", "/opt/cortex/entrypoint"), ExecCmd("CMD") ) diff --git a/docker/cortex/.env b/docker/cortex/.env new file mode 100644 index 000000000..d623e6efd --- /dev/null +++ b/docker/cortex/.env @@ -0,0 +1 @@ +job_directory=/tmp/cortex-jobs diff --git a/docker/cortex/docker-compose.yml b/docker/cortex/docker-compose.yml index f9a628502..be93a4cdd 100644 --- a/docker/cortex/docker-compose.yml +++ b/docker/cortex/docker-compose.yml @@ -8,10 +8,14 @@ services: - script.allowed_types=inline - thread_pool.search.queue_size=100000 - thread_pool.write.queue_size=10000 -path.repo: backup cortex: - image: thehiveproject/cortex:latest + image: thehiveproject/cortex:3.1.0-0.3RC1 + environment: + - job_directory=${job_directory} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${job_directory}:${job_directory} depends_on: - elasticsearch ports: - - "0.0.0.0:9001:9001" + - "0.0.0.0:9001:9001" \ No newline at end of file diff --git a/package/docker/entrypoint b/package/docker/entrypoint index 23646a41d..9d3b00cee 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -1,122 +1,156 @@ #!/bin/bash -ES_HOSTNAME=elasticsearch -CONFIG_SECRET=1 -CONFIG_ES=1 -CONFIG=1 -CONFIG_FILE=/etc/cortex/application.conf -ANALYZER_PATH=/opt/Cortex-Analyzers/analyzers +test "${no_config:-0}" == 1 +CONFIG=$? +test "${no_config_secret:-0}" == 1 +CONFIG_SECRET=$? +test "${no_config_es:-0}" == 1 +CONFIG_ES=$? +ES_URI=${es_uri:-} +ES_HOSTNAME=${es_hostname:-elasticsearch} +CONFIG_FILE=${config_file:-/etc/cortex/application.conf} +DEFAULT_ANALYZER_URL="https://download.thehive-project.org/analyzers.json" ANALYZER_URLS=() -RESPONDER_PATH=/opt/Cortex-Analyzers/responders +IFS=',' read -r -a ANALYZER_URLS <<< "${analyzer_urls:-$analyzer_url}" +DEFAULT_RESPONDER_URL="https://download.thehive-project.org/responders.json" RESPONDER_URLS=() -START_DOCKER=0 -SHOW_SECRET=0 +IFS=',' read -r -a RESPONDER_URLS <<< "${responder_urls:-$responder_url}" +START_DOCKER=${start_docker:-0} +SHOW_SECRET=${show_secret:-0} +DAEMON_USER=${daemon_user:-cortex} +JOB_DIRECTORY=${job_directory:-/tmp/cortex-jobs} +DOCKER_JOB_DIRECTORY=${docker_job_directory:-} function usage { - cat <<- _EOF_ - Available options: - --no-config | do not try to configure TheHive (add secret and elasticsearch) - --no-config-secret | do not add random secret to configuration - --no-config-es | do not add elasticsearch hosts to configuration - --es-uri | use this string to configure elasticsearch hosts (format: http(s)://host:port,host:port(/prefix)?querystring) - --es-hostname | resolve this hostname to find elasticseach instances - --secret | secret to secure sessions - --show-secret | show the generated secret - --analyzer-url | where analyzers are located (url or path) - --responder-url | where responders are located (url or path) - --start-docker | start a internal docker (inside container) to run analyzers/responders - _EOF_ - exit 1 + cat <<- _EOF_ + Available options: + --no-config | do not configure TheHive (add secret and elasticsearch) + --no-config-secret | do not add random secret to configuration + --no-config-es | do not add elasticsearch hosts to configuration + --es-uri | use this string to configure elasticsearch hosts (format: http(s)://host:port,host:port(/prefix)?querystring) + --es-hostname | resolve this hostname to find elasticsearch instances + --secret | secret to secure sessions + --show-secret | show the generated secret + --job-directory | use this directory to store job files + --docker-job-directory | indicate the job directory in the host (not inside container) + --analyzer-url | where analyzers are located (url or path) + --responder-url | where responders are located (url or path) + --start-docker | start a internal docker (inside container) to run analyzers/responders + --daemon-user | run cortex using this user +_EOF_ + exit 1 } STOP=0 while test $# -gt 0 -o $STOP = 1 do - case "$1" in - "--no-config") CONFIG=0;; - "--no-config-secret") CONFIG_SECRET=0;; - "--no-config-es") CONFIG_ES=0;; - "--es-hosts") echo "--es-hosts is deprecated, please use --es-uri" - usage;; - "--es-uri") shift; ES_URI=$1;; - "--es-hostname") shift; ES_HOSTNAME=$1;; - "--secret") shift; SECRET=$1;; - "--show-secret") SHOW_SECRET=1;; - "--analyzer-path") shift; ANALYZER_PATH=$1;; - "--responder-path") shift; RESPONDER_PATH=$1;; - "--analyzer-url") shift; ANALYZER_URLS+=$1;; - "--responder-url") shift; RESPONDER_URLS+=$1;; - "--start-docker") START_DOCKER=1;; - "--") STOP=1;; - *) echo "unrecognized option: $1"; usage;; - esac - shift + case "$1" in + "--no-config") CONFIG=0;; + "--no-config-secret") CONFIG_SECRET=0;; + "--no-config-es") CONFIG_ES=0;; + "--es-hosts") echo "--es-hosts is deprecated, please use --es-uri" + usage;; + "--es-uri") shift; ES_URI=$1;; + "--es-hostname") shift; ES_HOSTNAME=$1;; + "--secret") shift; SECRET=$1;; + "--show-secret") SHOW_SECRET=1;; + "--job-directory") shift; JOB_DIRECTORY=$1;; + "--docker-job-directory") shift; DOCKER_JOB_DIRECTORY=$1;; + "--analyzer-path") echo "--analyzer-path is deprecated, please use --analyzer-url" + shift; ANALYZER_URLS+=("$1");; + "--responder-path") echo "--responder-path is deprecated, please use --responder-url" + shift; RESPONDER_URLS+=("$1");; + "--analyzer-url") shift; ANALYZER_URLS+=("$1");; + "--responder-url") shift; RESPONDER_URLS+=("$1");; + "--start-docker") START_DOCKER=1;; + "--daemon-user") shift; DAEMON_USER=$1;; + "--") STOP=1;; + *) echo "unrecognized option: $1"; usage;; + esac + shift done if test $CONFIG = 1 then - CONFIG_FILE=$(mktemp).conf - if test $CONFIG_SECRET = 1 - then - if test -z "$SECRET" - then - SECRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1) - test $SHOW_SECRET = 1 && echo Using secret: $SECRET - fi - echo play.http.secret.key=\"$SECRET\" >> $CONFIG_FILE - fi + CONFIG_FILE=$(mktemp --tmpdir cortex-XXXXXX.conf) + if test $CONFIG_SECRET = 1 + then + if test -z "$SECRET" + then + SECRET=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1) + test $SHOW_SECRET = 1 && echo "Using secret: $SECRET" + fi + echo "play.http.secret.key=\"$SECRET\"" >> "$CONFIG_FILE" + fi - if test $CONFIG_ES = 1 - then - if test -z "$ES_URI" - then - function join_es_hosts { - echo -n $1:9200 - shift - printf "%s," "${@/#/:9200}" - } + if test $CONFIG_ES = 1 + then + if test -z "$ES_URI" + then + function join_es_hosts { + echo -n "$1:9200" + shift + printf "%s," "${@/#/:9200}" + } - ES=$(getent ahostsv4 $ES_HOSTNAME | awk '{ print $1 }' | sort -u) - if test -z "$ES" - then - echo "Warning automatic elasticsearch host config fails" - else - ES_URI=http://$(join_es_hosts $ES) - fi - fi - if test -n "$ES_URI" - then - echo Using elasticsearch uri: $ES_URI - echo search.uri=\"$ES_URI\" >> $CONFIG_FILE - else - echo elasticsearch host not configured - fi - fi + ES=$(getent ahostsv4 "$ES_HOSTNAME" | awk '{ print $1 }' | sort -u) + if test -z "$ES" + then + echo "Warning automatic elasticsearch host config fails" + else + ES_URI=http://$(join_es_hosts "$ES") + fi + fi + if test -n "$ES_URI" + then + echo "Using elasticsearch uri: $ES_URI" + echo "search.uri=\"$ES_URI\"" >> "$CONFIG_FILE" + else + echo elasticsearch host not configured + fi + fi - function join_urls { - echo -n \"$1\" - shift - for U do echo -n ,\"$U\"; done -# printf ",\"%s\"" $@ - } - test ${#ANALYZER_URLS} = 0 && ANALYZER_URLS+=$ANALYZER_PATH - test ${#RESPONDER_URLS} = 0 && RESPONDER_URLS+=$RESPONDER_PATH - - echo analyzer.urls=\[$(join_urls ${ANALYZER_URLS[@]})\] >> $CONFIG_FILE - echo responder.urls=\[$(join_urls ${RESPONDER_URLS[@]})\] >> $CONFIG_FILE + test -n "$JOB_DIRECTORY" && echo "job.directory=\"$JOB_DIRECTORY\"" >> "$CONFIG_FILE" + test -n "$DOCKER_JOB_DIRECTORY" && echo "job.dockerDirectory=\"$DOCKER_JOB_DIRECTORY\"" >> "$CONFIG_FILE" - echo 'include file("/etc/cortex/application.conf")' >> $CONFIG_FILE + function join_urls { + echo -n "\"$1\"" + shift + for U do echo -n ",\"$U\""; done +# printf ",\"%s\"" $@ + } + test ${#ANALYZER_URLS} = 0 && ANALYZER_URLS+=("$DEFAULT_ANALYZER_URL") + echo "analyzer.urls=[$(join_urls "${ANALYZER_URLS[@]}")]" >> "$CONFIG_FILE" + + test ${#RESPONDER_URLS} = 0 && RESPONDER_URLS+=("$DEFAULT_RESPONDER_URL") + echo "responder.urls=[$(join_urls "${RESPONDER_URLS[@]}")]" >> "$CONFIG_FILE" + + echo 'include file("/etc/cortex/application.conf")' >> "$CONFIG_FILE" fi -test $START_DOCKER = 1 && dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &> /dev/null & -DOCKER_PID=$! +if test $START_DOCKER = 1 +then + usermod --append --groups docker "$DAEMON_USER" + dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &> /dev/null & + DOCKER_PID=$! +fi echo config file is: -cat $CONFIG_FILE -/bin/sh -c "cd /opt/cortex; bin/cortex \ - -Dconfig.file=$CONFIG_FILE \ - -Dlogger.file=/etc/cortex/logback.xml \ - -Dpidfile.path=/dev/null \ - $@" +cat "$CONFIG_FILE" +chown -R "$DAEMON_USER" /var/log/cortex +chown -R "$DAEMON_USER" /etc/cortex +chown -R "$DAEMON_USER" "$CONFIG_FILE" +test -e /var/run/docker.sock && chown "$DAEMON_USER" /var/run/docker.sock +if test -n "$JOB_DIRECTORY" +then + mkdir -p "$JOB_DIRECTORY" + chown -R "$DAEMON_USER" "$JOB_DIRECTORY" +fi + +su "$DAEMON_USER" -c "bin/cortex \ + -Dconfig.file=$CONFIG_FILE \ + -Dlogger.file=/etc/cortex/logback.xml \ + -Dpidfile.path=/dev/null \ + $*" test $START_DOCKER = 1 && kill ${DOCKER_PID}