Merge branch 'feature/activeResponse' into develop

app/org/thp/cortex/controllers/AnalyzerConfigCtrl.scala

@@ -1,14 +1,13 @@
 package org.thp.cortex.controllers
 
 import javax.inject.{ Inject, Singleton }
-
 import scala.concurrent.{ ExecutionContext, Future }
 
 import play.api.libs.json.JsObject
 import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
 
-import org.thp.cortex.models.Roles
-import org.thp.cortex.services.{ AnalyzerConfigSrv, BaseConfig, UserSrv }
+import org.thp.cortex.models.{ BaseConfig, Roles }
+import org.thp.cortex.services.{ AnalyzerConfigSrv, UserSrv }
 
 import org.elastic4play.BadRequestError
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
@@ -29,7 +28,7 @@ class AnalyzerConfigCtrl @Inject() (
   }
 
   def list(): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request ⇒
-    analyzerConfigSrv.listForUser(request.userId)
+    analyzerConfigSrv.listConfigForUser(request.userId)
       .map { bc ⇒
         renderer.toOutput(OK, bc.sortWith {
           case (BaseConfig("global", _, _, _), _)               ⇒ true

app/org/thp/cortex/controllers/AnalyzerCtrl.scala

@@ -1,16 +1,15 @@
 package org.thp.cortex.controllers
 
 import javax.inject.{ Inject, Singleton }
-
 import scala.concurrent.{ ExecutionContext, Future }
 
-import play.api.libs.json.{ JsObject, Json }
+import play.api.libs.json.{ JsNumber, JsObject, JsString, Json }
 import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
 
 import akka.stream.Materializer
 import akka.stream.scaladsl.Sink
-import org.thp.cortex.models.{ Analyzer, AnalyzerDefinition, Roles }
-import org.thp.cortex.services.{ AnalyzerSrv, UserSrv }
+import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition }
+import org.thp.cortex.services.{ UserSrv, WorkerSrv }
 
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
 import org.elastic4play.models.JsonFormat.baseModelEntityWrites
@@ -19,7 +18,7 @@ import org.elastic4play.services.{ QueryDSL, QueryDef }
 
 @Singleton
 class AnalyzerCtrl @Inject() (
-    analyzerSrv: AnalyzerSrv,
+    workerSrv: WorkerSrv,
     userSrv: UserSrv,
     authenticated: Authenticated,
     fieldsBodyParser: FieldsBodyParser,
@@ -33,54 +32,56 @@ class AnalyzerCtrl @Inject() (
     val range = request.body.getString("range")
     val sort = request.body.getStrings("sort").getOrElse(Nil)
     val isAdmin = request.roles.contains(Roles.orgAdmin)
-    val (analyzers, analyzerTotal) = analyzerSrv.findForUser(request.userId, query, range, sort)
+    val (analyzers, analyzerTotal) = workerSrv.findAnalyzersForUser(request.userId, query, range, sort)
     val enrichedAnalyzers = analyzers.mapAsync(2)(analyzerJson(isAdmin))
     renderer.toOutput(OK, enrichedAnalyzers, analyzerTotal)
   }
 
   def get(analyzerId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒
     val isAdmin = request.roles.contains(Roles.orgAdmin)
-    analyzerSrv.getForUser(request.userId, analyzerId)
+    workerSrv.getForUser(request.userId, analyzerId)
       .flatMap(analyzerJson(isAdmin))
       .map(renderer.toOutput(OK, _))
   }
 
   private val emptyAnalyzerDefinitionJson = Json.obj(
-    "version" -> "0.0",
-    "description" -> "unknown",
-    "dataTypeList" -> Nil,
-    "author" -> "unknown",
-    "url" -> "unknown",
-    "license" -> "unknown")
-
-  private def analyzerJson(analyzer: Analyzer, analyzerDefinition: Option[AnalyzerDefinition]) = {
+    "version" → "0.0",
+    "description" → "unknown",
+    "dataTypeList" → Nil,
+    "author" → "unknown",
+    "url" → "unknown",
+    "license" → "unknown")
+
+  private def analyzerJson(analyzer: Worker, analyzerDefinition: Option[WorkerDefinition]) = {
     analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad ⇒
       Json.obj(
-        "version" -> ad.version,
-        "description" -> ad.description,
-        "author" -> ad.author,
-        "url" -> ad.url,
-        "license" -> ad.license,
-        "baseConfig" -> ad.baseConfiguration)
-    }
+        "maxTlp" → (analyzer.config \ "max_tlp").asOpt[JsNumber],
+        "maxPap" → (analyzer.config \ "max_pap").asOpt[JsNumber],
+        "version" → ad.version,
+        "description" → ad.description,
+        "author" → ad.author,
+        "url" → ad.url,
+        "license" → ad.license,
+        "baseConfig" → ad.baseConfiguration)
+    } + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId())) // For compatibility reason
   }
 
-  private def analyzerJson(isAdmin: Boolean)(analyzer: Analyzer): Future[JsObject] = {
-    analyzerSrv.getDefinition(analyzer.analyzerDefinitionId())
+  private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): Future[JsObject] = {
+    workerSrv.getDefinition(analyzer.workerDefinitionId())
       .map(analyzerDefinition ⇒ analyzerJson(analyzer, Some(analyzerDefinition)))
       .recover { case _ ⇒ analyzerJson(analyzer, None) }
       .map {
-        case a if isAdmin ⇒ a + ("configuration" -> Json.parse(analyzer.configuration()))
+        case a if isAdmin ⇒ a + ("configuration" → Json.parse(analyzer.configuration()))
         case a            ⇒ a
       }
   }
 
   def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒
     import org.elastic4play.services.QueryDSL._
-    analyzerSrv.findForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
+    workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
       ._1
       .mapAsyncUnordered(2) { analyzer ⇒
-        analyzerSrv.getDefinition(analyzer.analyzerDefinitionId())
+        workerSrv.getDefinition(analyzer.workerDefinitionId())
           .map(ad ⇒ analyzerJson(analyzer, Some(ad)))
       }
       .runWith(Sink.seq)
@@ -90,31 +91,33 @@ class AnalyzerCtrl @Inject() (
   def create(analyzerDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒
     for {
       organizationId ← userSrv.getOrganizationId(request.userId)
-      analyzer ← analyzerSrv.create(organizationId, analyzerDefinitionId, request.body)
-    } yield renderer.toOutput(CREATED, analyzer)
+      workerDefinition ← workerSrv.getDefinition(analyzerDefinitionId)
+      analyzer ← workerSrv.create(organizationId, workerDefinition, request.body)
+    } yield renderer.toOutput(CREATED, analyzerJson(analyzer, Some(workerDefinition)))
   }
 
   def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒
-    val (analyzers, analyzerTotal) = analyzerSrv.listDefinitions
+    val (analyzers, analyzerTotal) = workerSrv.listAnalyzerDefinitions
     renderer.toOutput(OK, analyzers, analyzerTotal)
   }
 
   def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒
-    analyzerSrv.rescan()
+    workerSrv.rescan()
     NoContent
   }
 
   def delete(analyzerId: String): Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒
     for {
-      analyzer ← analyzerSrv.getForUser(request.userId, analyzerId)
-      _ ← analyzerSrv.delete(analyzer)
+      analyzer ← workerSrv.getForUser(request.userId, analyzerId)
+      _ ← workerSrv.delete(analyzer)
     } yield NoContent
   }
 
   def update(analyzerId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒
     for {
-      analyzer ← analyzerSrv.getForUser(request.userId, analyzerId)
-      updatedAnalyzer ← analyzerSrv.update(analyzer, request.body)
-    } yield renderer.toOutput(OK, updatedAnalyzer)
+      analyzer ← workerSrv.getForUser(request.userId, analyzerId)
+      updatedAnalyzer ← workerSrv.update(analyzer, request.body)
+      updatedAnalyzerJson ← analyzerJson(isAdmin = true)(updatedAnalyzer)
+    } yield renderer.toOutput(OK, updatedAnalyzerJson)
   }
 }

app/org/thp/cortex/controllers/JobCtrl.scala

@@ -35,8 +35,8 @@ class JobCtrl @Inject() (
     implicit val mat: Materializer,
     implicit val actorSystem: ActorSystem) extends AbstractController(components) with Status {
 
-  def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], range: Option[String]): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
-    val (jobs, jobTotal) = jobSrv.listForUser(request.userId, dataTypeFilter, dataFilter, analyzerFilter, range)
+  def list(dataTypeFilter: Option[String], dataFilter: Option[String], workerFilter: Option[String], range: Option[String]): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+    val (jobs, jobTotal) = jobSrv.listForUser(request.userId, dataTypeFilter, dataFilter, workerFilter, range)
     renderer.toOutput(OK, jobs, jobTotal)
   }
 
@@ -84,31 +84,32 @@ class JobCtrl @Inject() (
             .collect {
               case artifact if artifact.data().isDefined ⇒
                 Json.obj(
-                  "data" -> artifact.data(),
-                  "dataType" -> artifact.dataType(),
-                  "message" -> artifact.message(),
-                  "tags" -> artifact.tags(),
-                  "tlp" -> artifact.tlp())
+                  "data" → artifact.data(),
+                  "dataType" → artifact.dataType(),
+                  "message" → artifact.message(),
+                  "tags" → artifact.tags(),
+                  "tlp" → artifact.tlp())
               case artifact if artifact.attachment().isDefined ⇒
                 artifact.attachment().fold(JsObject.empty) { a ⇒
                   Json.obj(
-                    "attachment" ->
+                    "attachment" →
                       Json.obj(
-                        "contentType" -> a.contentType,
-                        "id" -> a.id,
-                        "name" -> a.name,
-                        "size" -> a.size),
-                    "message" -> artifact.message(),
-                    "tags" -> artifact.tags(),
-                    "tlp" -> artifact.tlp())
+                        "contentType" → a.contentType,
+                        "id" → a.id,
+                        "name" → a.name,
+                        "size" → a.size),
+                    "message" → artifact.message(),
+                    "tags" → artifact.tags(),
+                    "tlp" → artifact.tlp())
                 }
             }
             .runWith(Sink.seq)
         } yield Json.obj(
-          "summary" -> Json.parse(report.summary()),
-          "full" -> Json.parse(report.full()),
-          "success" -> true,
-          "artifacts" -> artifacts)
+          "summary" → Json.parse(report.summary()),
+          "full" → Json.parse(report.full()),
+          "success" → true,
+          "artifacts" → artifacts,
+          "operations" → Json.parse(report.operations()))
       case JobStatus.Failure ⇒
         val errorMessage = job.errorMessage().getOrElse("")
         Future.successful(Json.obj(
@@ -120,7 +121,7 @@ class JobCtrl @Inject() (
       case JobStatus.Deleted    ⇒ Future.successful(JsString("Deleted"))
     })
       .map { report ⇒
-        Json.toJson(job).as[JsObject] + ("report" -> report)
+        Json.toJson(job).as[JsObject] + ("report" → report)
       }
   }
 
@@ -132,7 +133,7 @@ class JobCtrl @Inject() (
     jobSrv.getForUser(request.userId, jobId)
       .flatMap {
         case job if job.status() == JobStatus.InProgress || job.status() == JobStatus.Waiting ⇒
-          println(s"job status is ${job.status()} => wait")
+          println(s"job status is ${job.status()} ⇒ wait")
           val duration = Duration(atMost).asInstanceOf[FiniteDuration]
           implicit val timeout: Timeout = Timeout(duration)
           (auditActor ? Register(jobId, duration))
@@ -141,7 +142,7 @@ class JobCtrl @Inject() (
             .withTimeout(duration, ())
             .flatMap(_ ⇒ getJobWithReport(request.userId, jobId))
         case job ⇒
-          println(s"job status is ${job.status()} => send it directly")
+          println(s"job status is ${job.status()} ⇒ send it directly")
           getJobWithReport(request.userId, job)
       }
       .map(Ok(_))

app/org/thp/cortex/controllers/MispCtrl.scala

@@ -3,7 +3,7 @@ package org.thp.cortex.controllers
 import javax.inject.Inject
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
 import org.thp.cortex.models.Roles
-import org.thp.cortex.services.{ AnalyzerSrv, MispSrv }
+import org.thp.cortex.services.{ WorkerSrv, MispSrv }
 import play.api.Logger
 import play.api.libs.json.{ JsObject, JsValue }
 import play.api.mvc._
@@ -12,7 +12,7 @@ import scala.concurrent.{ ExecutionContext, Future }
 
 class MispCtrl @Inject() (
     mispSrv: MispSrv,
-    analyzerSrv: AnalyzerSrv,
+    analyzerSrv: WorkerSrv,
     authenticated: Authenticated,
     fieldsBodyParser: FieldsBodyParser,
     renderer: Renderer,

app/org/thp/cortex/controllers/ResponderConfigCtrl.scala

@@ -0,0 +1,48 @@
+package org.thp.cortex.controllers
+
+import scala.concurrent.{ ExecutionContext, Future }
+
+import play.api.libs.json.JsObject
+import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
+
+import javax.inject.{ Inject, Singleton }
+import org.thp.cortex.models.{ BaseConfig, Roles }
+import org.thp.cortex.services.{ ResponderConfigSrv, UserSrv }
+
+import org.elastic4play.BadRequestError
+import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
+
+@Singleton
+class ResponderConfigCtrl @Inject() (
+    responderConfigSrv: ResponderConfigSrv,
+    userSrv: UserSrv,
+    authenticated: Authenticated,
+    fieldsBodyParser: FieldsBodyParser,
+    renderer: Renderer,
+    components: ControllerComponents,
+    implicit val ec: ExecutionContext) extends AbstractController(components) {
+
+  def get(analyzerConfigName: String): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request ⇒
+    responderConfigSrv.getForUser(request.userId, analyzerConfigName)
+      .map(renderer.toOutput(OK, _))
+  }
+
+  def list(): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request ⇒
+    responderConfigSrv.listConfigForUser(request.userId)
+      .map { bc ⇒
+        renderer.toOutput(OK, bc.sortWith {
+          case (BaseConfig("global", _, _, _), _)               ⇒ true
+          case (_, BaseConfig("global", _, _, _))               ⇒ false
+          case (BaseConfig(a, _, _, _), BaseConfig(b, _, _, _)) ⇒ a.compareTo(b) < 0
+        })
+      }
+  }
+
+  def update(analyzerConfigName: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒
+    request.body.getValue("config").flatMap(_.asOpt[JsObject]) match {
+      case Some(config) ⇒ responderConfigSrv.updateOrCreate(request.userId, analyzerConfigName, config)
+        .map(renderer.toOutput(OK, _))
+      case None ⇒ Future.failed(BadRequestError("attribute config has invalid format"))
+    }
+  }
+}