From 99d20763405b3ea2a827a12c2545a912170bc24f Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 3 Jun 2019 15:36:41 +0200 Subject: [PATCH] #175 Add docker daemon in Cortex docker image --- app/org/thp/cortex/controllers/JobCtrl.scala | 7 +-- .../cortex/services/DockerJobRunnerSrv.scala | 25 ++++---- docker.sbt | 19 +++++-- docker/cortex/docker-compose.yml | 5 +- package/docker/entrypoint | 57 +++++++++++++------ test/resources/analyzers/blocker/Dockerfile | 6 ++ test/resources/analyzers/blocker/blocker.sh | 6 ++ 7 files changed, 83 insertions(+), 42 deletions(-) create mode 100644 test/resources/analyzers/blocker/Dockerfile create mode 100755 test/resources/analyzers/blocker/blocker.sh diff --git a/app/org/thp/cortex/controllers/JobCtrl.scala b/app/org/thp/cortex/controllers/JobCtrl.scala index a9521526d..b43142e42 100644 --- a/app/org/thp/cortex/controllers/JobCtrl.scala +++ b/app/org/thp/cortex/controllers/JobCtrl.scala @@ -114,11 +114,8 @@ class JobCtrl @Inject()( "message" → artifact.message(), "tags" → artifact.tags(), "tlp" → artifact.tlp(), - "attachment" → Json.obj( - "contentType" → attachment.contentType, - "id" → attachment.id, - "name" → attachment.name, - "size" → attachment.size) + "attachment" → Json + .obj("contentType" → attachment.contentType, "id" → attachment.id, "name" → attachment.name, "size" → attachment.size) ) } .runWith(Sink.seq) diff --git a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala index 93d0fb475..ee1916531 100644 --- a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala +++ b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala @@ -46,10 +46,11 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val Try { logger.info(s"Docker is available:\n${client.info()}") true - }.getOrElse { - logger.info(s"Docker is not available") + }.recover { + case error ⇒ + logger.info(s"Docker is not available", error) false - } + }.get def run(jobDirectory: Path, dockerImage: String, job: Job, timeout: Option[FiniteDuration])(implicit ec: ExecutionContext): Future[Unit] = { import scala.collection.JavaConverters._ @@ -90,15 +91,15 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val client.waitContainer(containerCreation.id()) () }.andThen { - case r ⇒ - if (!Files.exists(jobDirectory.resolve("output").resolve("output.json"))) { - val message = r.fold(e ⇒ s"Docker creation error: ${e.getMessage}\n", _ ⇒ "") + - Try(client.logs(containerCreation.id(), LogsParam.stdout(), LogsParam.stderr()).readFully()) - .recover { case e ⇒ s"Container logs can't be read (${e.getMessage}" } - val report = Json.obj("success" → false, "errorMessage" → message) - Files.write(jobDirectory.resolve("output").resolve("output.json"), report.toString.getBytes(StandardCharsets.UTF_8)) - } - } + case r ⇒ + if (!Files.exists(jobDirectory.resolve("output").resolve("output.json"))) { + val message = r.fold(e ⇒ s"Docker creation error: ${e.getMessage}\n", _ ⇒ "") + + Try(client.logs(containerCreation.id(), LogsParam.stdout(), LogsParam.stderr()).readFully()) + .recover { case e ⇒ s"Container logs can't be read (${e.getMessage}" } + val report = Json.obj("success" → false, "errorMessage" → message) + Files.write(jobDirectory.resolve("output").resolve("output.json"), report.toString.getBytes(StandardCharsets.UTF_8)) + } + } timeout .fold(execution)(t ⇒ execution.withTimeout(t, client.stopContainer(containerCreation.id(), 3))) .andThen { diff --git a/docker.sbt b/docker.sbt index cc19a9f6e..a5451aa5d 100644 --- a/docker.sbt +++ b/docker.sbt @@ -23,17 +23,26 @@ mappings in Docker ~= (_.filterNot { }) dockerCommands ~= { dc => val (dockerInitCmds, dockerTailCmds) = dc - .collect { - case ExecCmd("RUN", "chown", _*) => ExecCmd("RUN", "chown", "-R", "daemon:root", ".") - case other => other + .flatMap { + case ExecCmd("RUN", "chown", _*) => Some(ExecCmd("RUN", "chown", "-R", "daemon:root", ".")) + case Cmd("USER", _) => None + case other => Some(other) } .splitAt(4) dockerInitCmds ++ Seq( Cmd("USER", "root"), ExecCmd("RUN", "bash", "-c", - "apt-get update && " + - "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev dnsutils && " + + "wget -q -O - https://download.docker.com/linux/static/stable/x86_64/docker-18.09.0.tgz | " + + "tar -xzC /usr/local/bin/ --strip-components 1 && " + + "addgroup --system dockremap && " + + "adduser --system --ingroup dockremap dockremap && " + + "addgroup --system docker && " + + "usermod --append --groups docker daemon &&" + + "echo 'dockremap:165536:65536' >> /etc/subuid && " + + "echo 'dockremap:165536:65536' >> /etc/subgid && " + + "apt-get update && " + + "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev dnsutils iptables && " + "pip2 install -U pip setuptools && " + "pip3 install -U pip setuptools && " + "hash -r && " + diff --git a/docker/cortex/docker-compose.yml b/docker/cortex/docker-compose.yml index 2ba0a692b..1dc8e47c5 100644 --- a/docker/cortex/docker-compose.yml +++ b/docker/cortex/docker-compose.yml @@ -1,13 +1,10 @@ version: "2" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + image: elasticsearch:6.8.0 environment: - http.host=0.0.0.0 - - transport.host=0.0.0.0 - - xpack.security.enabled=false - cluster.name=hive - - script.inline=true - thread_pool.index.queue_size=100000 - thread_pool.search.queue_size=100000 - thread_pool.bulk.queue_size=100000 diff --git a/package/docker/entrypoint b/package/docker/entrypoint index 5f78fe577..0737fed1c 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -6,7 +6,10 @@ CONFIG_ES=1 CONFIG=1 CONFIG_FILE=/etc/cortex/application.conf ANALYZER_PATH=/opt/Cortex-Analyzers/analyzers +ANALYZER_URLS=() RESPONDER_PATH=/opt/Cortex-Analyzers/responders +RESPONDER_URLS=() +START_DOCKER=0 function usage { cat <<- _EOF_ @@ -14,11 +17,12 @@ function usage { --no-config | do not try to configure TheHive (add secret and elasticsearch) --no-config-secret | do not add random secret to configuration --no-config-es | do not add elasticsearch hosts to configuration - --es-hosts | use this string to configure elasticsearch hosts (format: ["host1:9300","host2:9300"]) + --es-uri | use this string to configure elasticsearch hosts (format: http(s)://host:port,host:port(/prefix)?querystring) --es-hostname | resolve this hostname to find elasticseach instances --secret | secret to secure sessions - --analyzer-path | where analyzers are located - --responder-path | where responders are located + --analyzer-url | where analyzers are located (url or path) + --responder-url | where responders are located (url or path) + --start-docker | start a internal docker (inside container) to run analyzers/responders _EOF_ exit 1 } @@ -30,13 +34,18 @@ do "--no-config") CONFIG=0;; "--no-config-secret") CONFIG_SECRET=0;; "--no-config-es") CONFIG_ES=0;; - "--es-hosts") shift; ES_HOSTS=$1;; + "--es-hosts") echo "--es-hosts is deprecated, please use --es-uri" + usage;; + "--es-uri") shift; ES_URI=$1;; "--es-hostname") shift; ES_HOSTNAME=$1;; "--secret") shift; SECRET=$1;; "--analyzer-path") shift; ANALYZER_PATH=$1;; "--responder-path") shift; RESPONDER_PATH=$1;; + "--analyzer-url") shift; ANALYZER_URLS+=$1;; + "--responder-url") shift; RESPONDER_URLS+=$1;; + "--start-docker") START_DOCKER=1;; "--") STOP=1;; - *) usage + *) echo "unrecognized option: $1"; usage;; esac shift done @@ -56,12 +65,12 @@ then if test $CONFIG_ES = 1 then - if test -z "$ES_HOSTS" + if test -z "$ES_URI" then function join_es_hosts { - echo -n "[\"$1" + echo -n $1:9200 shift - printf "%s:9300\"]" "${@/#/:9300\",\"}" + printf "%s," "${@/#/:9200}" } ES=$(getent ahostsv4 $ES_HOSTNAME | awk '{ print $1 }' | sort -u) @@ -69,26 +78,42 @@ then then echo "Warning automatic elasticsearch host config fails" else - ES_HOSTS=$(join_es_hosts $ES) + ES_URI=http://$(join_es_hosts $ES) fi fi - if test -n "$ES_HOSTS" + if test -n "$ES_URI" then - echo Using elasticsearch host: $ES_HOSTS - echo search.host=$ES_HOSTS >> $CONFIG_FILE + echo Using elasticsearch uri: $ES_URI + echo search.uri=\"$ES_URI\" >> $CONFIG_FILE else echo elasticsearch host not configured fi fi - echo analyzer.path=[\"$ANALYZER_PATH\"] >> $CONFIG_FILE - echo responder.path=[\"$RESPONDER_PATH\"] >> $CONFIG_FILE + function join_urls { + echo -n \"$1\" + shift + for U do echo -n ,\"$U\"; done +# printf ",\"%s\"" $@ + } + test ${#ANALYZER_URLS} = 0 && ANALYZER_URLS+=$ANALYZER_PATH + test ${#RESPONDER_URLS} = 0 && RESPONDER_URLS+=$RESPONDER_PATH + + echo analyzer.urls=\[$(join_urls ${ANALYZER_URLS[@]})\] >> $CONFIG_FILE + echo responder.urls=\[$(join_urls ${RESPONDER_URLS[@]})\] >> $CONFIG_FILE echo 'include file("/etc/cortex/application.conf")' >> $CONFIG_FILE fi -exec bin/cortex \ +test $START_DOCKER = 1 && dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &> /dev/null & +DOCKER_PID=$! + +echo config file is: +cat $CONFIG_FILE +su -s /bin/sh -c "cd /opt/cortex; bin/cortex \ -Dconfig.file=$CONFIG_FILE \ -Dlogger.file=/etc/cortex/logback.xml \ -Dpidfile.path=/dev/null \ - $@ + $@" daemon + +test $START_DOCKER = 1 && kill ${DOCKER_PID} diff --git a/test/resources/analyzers/blocker/Dockerfile b/test/resources/analyzers/blocker/Dockerfile new file mode 100644 index 000000000..aa281134a --- /dev/null +++ b/test/resources/analyzers/blocker/Dockerfile @@ -0,0 +1,6 @@ +FROM debian:latest + +WORKDIR /analyzer +RUN apt update && apt install -y jq +COPY blocker.sh blocker/blocker.sh +ENTRYPOINT ["blocker/blocker.sh"] diff --git a/test/resources/analyzers/blocker/blocker.sh b/test/resources/analyzers/blocker/blocker.sh new file mode 100755 index 000000000..4ded379f6 --- /dev/null +++ b/test/resources/analyzers/blocker/blocker.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +while true +do + sleep 10 +done