From 5cdc028858a12050c80d30058e421fb63194ad24 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 3 Apr 2019 10:16:29 +0200 Subject: [PATCH] #175 Improve error messages --- app/org/thp/cortex/models/Worker.scala | 16 ++++++------ .../cortex/services/DockerJobRunnerSrv.scala | 25 ++++++++++++++++--- .../thp/cortex/services/JobRunnerSrv.scala | 2 +- conf/reference.conf | 4 +-- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/app/org/thp/cortex/models/Worker.scala b/app/org/thp/cortex/models/Worker.scala index 8b9a7660b..715a8aa07 100644 --- a/app/org/thp/cortex/models/Worker.scala +++ b/app/org/thp/cortex/models/Worker.scala @@ -28,17 +28,17 @@ object WorkerType extends Enumeration with HiveEnumeration { trait WorkerAttributes { _: AttributeDef ⇒ val workerId = attribute("_id", F.stringFmt, "Worker id", O.model) val name = attribute("name", F.stringFmt, "Worker name") - val vers = attribute("version", F.stringFmt, "Worker version") + val vers = attribute("version", F.stringFmt, "Worker version", O.readonly) val workerDefinitionId = attribute("workerDefinitionId", F.stringFmt, "Worker definition id", O.readonly) - val description = attribute("description", F.textFmt, "Worker description") - val author = attribute("author", F.textFmt, "Worker author") - val url = attribute("url", F.textFmt, "Worker url") - val license = attribute("license", F.textFmt, "Worker license") - val command = optionalAttribute("command", F.textFmt, "Worker command") - val dockerImage = optionalAttribute("dockerImage", F.textFmt, "Worker docker image") + val description = attribute("description", F.textFmt, "Worker description", O.readonly) + val author = attribute("author", F.textFmt, "Worker author", O.readonly) + val url = attribute("url", F.textFmt, "Worker url", O.readonly) + val license = attribute("license", F.textFmt, "Worker license", O.readonly) + val command = optionalAttribute("command", F.textFmt, "Worker command", O.readonly) + val dockerImage = optionalAttribute("dockerImage", F.textFmt, "Worker docker image", O.readonly) val dataTypeList = multiAttribute("dataTypeList", F.stringFmt, "List of data type this worker can manage") val configuration = attribute("configuration", F.rawFmt, "Configuration of the worker", O.sensitive) - val baseConfig = attribute("baseConfig", F.stringFmt, "Base configuration key") + val baseConfig = attribute("baseConfig", F.stringFmt, "Base configuration key", O.readonly) val rate = optionalAttribute("rate", F.numberFmt, "Number ") val rateUnit = optionalAttribute("rateUnit", F.enumFmt(RateUnit), "") val jobCache = optionalAttribute("jobCache", F.numberFmt, "") diff --git a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala index 6da1aadc1..848c1b4ea 100644 --- a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala +++ b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala @@ -1,14 +1,17 @@ package org.thp.cortex.services +import java.nio.charset.StandardCharsets import java.nio.file._ import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ ExecutionContext, Future } import scala.util.Try +import play.api.libs.json.Json import play.api.{ Configuration, Logger } import akka.actor.ActorSystem +import com.spotify.docker.client.DockerClient.LogsParam import com.spotify.docker.client.messages.HostConfig.Bind import com.spotify.docker.client.messages.{ ContainerConfig, HostConfig } import com.spotify.docker.client.{ DefaultDockerClient, DockerClient } @@ -41,7 +44,11 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val Try { logger.info(s"Docker is available:\n${client.info()}") true - }.getOrElse(false) + } + .getOrElse { + logger.info(s"Docker is not available") + false + } def run(jobDirectory: Path, dockerImage: String, job: Job, timeout: Option[FiniteDuration])(implicit ec: ExecutionContext): Future[Unit] = { import scala.collection.JavaConverters._ @@ -70,12 +77,24 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val s" volume : ${jobDirectory.toAbsolutePath}:/job" + Option(containerConfig.env()).fold("")(_.asScala.map("\n env : " + _).mkString)) - client.startContainer(containerCreation.id()) - val execution = Future { + client.startContainer(containerCreation.id()) client.waitContainer(containerCreation.id()) () } + .andThen { + case r ⇒ + if (!Files.exists(jobDirectory.resolve("output").resolve("output.json"))) { + val message = r.fold(e ⇒ s"Docker creation error: ${e.getMessage}\n", _ ⇒ "") + + Try(client.logs(containerCreation.id(), LogsParam.stdout(), LogsParam.stderr()).readFully()) + .recover { case e ⇒ s"Container logs can't be read (${e.getMessage}" } + val report = Json.obj( + "success" -> false, + "errorMessage" -> message) + Files.write(jobDirectory.resolve("output").resolve("output.json"), report.toString.getBytes(StandardCharsets.UTF_8)) + } + client.removeContainer(containerCreation.id()) + } timeout.fold(execution)(t ⇒ execution.withTimeout(t, client.stopContainer(containerCreation.id(), 3))) .andThen { case _ ⇒ client.removeContainer(containerCreation.id(), DockerClient.RemoveContainerParam.forceKill()) diff --git a/app/org/thp/cortex/services/JobRunnerSrv.scala b/app/org/thp/cortex/services/JobRunnerSrv.scala index c8cf39c42..b25a01396 100644 --- a/app/org/thp/cortex/services/JobRunnerSrv.scala +++ b/app/org/thp/cortex/services/JobRunnerSrv.scala @@ -101,7 +101,7 @@ class JobRunnerSrv @Inject() ( .map { case Some(file) ⇒ Json.obj( - "file" → file.toString, // FIXME set only the temporary file name + "file" → file.getFileName.toString, "filename" → job.attachment().get.name, "contentType" → job.attachment().get.contentType) case None if job.data().nonEmpty && job.tpe() == WorkerType.responder ⇒ diff --git a/conf/reference.conf b/conf/reference.conf index 94e2f2d30..d367a0110 100644 --- a/conf/reference.conf +++ b/conf/reference.conf @@ -93,7 +93,7 @@ audit.name = audit analyzer { # Directory that holds analyzers - path = [] + url = [] fork-join-executor { # Min number of threads available for analyze @@ -108,7 +108,7 @@ analyzer { responder { # Directory that holds responders - path = [] + url = [] fork-join-executor { # Min number of threads available for analyze