Merge branch 'hotfix/3.0.7'

CHANGELOG.md

@@ -1,13 +1,26 @@
 # Change Log
 
-## [3.0.6](https://github.com/TheHive-Project/TheHive/tree/3.0.6)
+## [3.0.7](https://github.com/TheHive-Project/TheHive/tree/3.0.7) (2018-03-29)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.0.6...3.0.7)
 
+**Implemented enhancements:**
+- Delete Case [\#100](https://github.com/TheHive-Project/TheHive/issues/100)
+
+**Fixed bugs:**
+
+- Display only cortex servers available for each analyzer, in observable details page [\#513](https://github.com/TheHive-Project/TheHive/issues/513)
+- Can't save case template in 3.0.6 [\#502](https://github.com/TheHive-Project/TheHive/issues/502)
+
+## [3.0.6](https://github.com/TheHive-Project/TheHive/tree/3.0.6) (2018-03-08)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.0.5...3.0.6)
 
+**Implemented enhancements:**
+
+- Add compatibility with Cortex 2 [\#466](https://github.com/TheHive-Project/TheHive/issues/466)
+
 **Fixed bugs:**
 
 - Tasks are stripped when merging cases [\#489](https://github.com/TheHive-Project/TheHive/issues/489)
-- Add compatibility with Cortex 2 [\#466](https://github.com/TheHive-Project/TheHive/issues/466)
 
 ## [3.0.5](https://github.com/TheHive-Project/TheHive/tree/3.0.5) (2018-02-08)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.0.4...3.0.5)
@@ -17,7 +30,7 @@
  - Importing Template Button Non-Functional bug [\#404](https://github.com/TheHive-Project/TheHive/issues/404)
  - No reports available for "domain" type bug [\#409](https://github.com/TheHive-Project/TheHive/issues/409)
 
-## [3.0.4](https://github.com/TheHive-Project/TheHive/tree/3.0.4) (2018-02-05)
+## [3.0.4](https://github.com/TheHive-Project/TheHive/tree/3.0.4) (2018-02-06)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.0.3...3.0.4)
 
 **Implemented enhancements:**

docker/thehive/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "2"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.5.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0
     environment:
       - http.host=0.0.0.0
       - transport.host=0.0.0.0
@@ -13,8 +13,10 @@ services:
       - thread_pool.bulk.queue_size=100000
   cortex:
     image: certbdf/cortex:latest
+    depends_on:
+      - elasticsearch
     ports:
-      - "0.0.0.0:9001:9000"
+      - "0.0.0.0:9001:9001"
   thehive:
     image: certbdf/thehive:latest
     depends_on:

project/Dependencies.scala

@@ -20,6 +20,6 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.5"
+    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.6"
   }
 }

thehive-backend/app/controllers/CaseCtrl.scala

@@ -93,6 +93,12 @@ class CaseCtrl @Inject() (
       .map(_ ⇒ NoContent)
   }
 
+  @Timed
+  def realDelete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒
+    caseSrv.realDelete(id)
+      .map(_ ⇒ NoContent)
+  }
+
   @Timed
   def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
     val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])

thehive-backend/app/controllers/FlowCtrl.scala

@@ -8,15 +8,15 @@ import play.api.http.Status
 import play.api.mvc._
 
 import models.Roles
-import services.FlowSrv
+import services.AuditSrv
 
 import org.elastic4play.Timed
 import org.elastic4play.controllers.{ Authenticated, Renderer }
 import org.elastic4play.services.AuxSrv
 
 @Singleton
 class FlowCtrl @Inject() (
-    flowSrv: FlowSrv,
+    flowSrv: AuditSrv,
     auxSrv: AuxSrv,
     authenticated: Authenticated,
     renderer: Renderer,

thehive-backend/app/services/AlertSrv.scala

@@ -275,7 +275,18 @@ class AlertSrv(
     updateSrv(alert, Fields(Json.obj("case" → caze.id, "status" → AlertStatus.Imported)), modifyConfig)
   }
 
-  def delete(id: String)(implicit Context: AuthContext): Future[Alert] =
+  def unsetCase(alert: Alert, modifyConfig: ModifyConfig = ModifyConfig.default)(implicit authContext: AuthContext): Future[Alert] = {
+    val status = alert.status match {
+      case AlertStatus.New      ⇒ AlertStatus.New
+      case AlertStatus.Updated  ⇒ AlertStatus.New
+      case AlertStatus.Ignored  ⇒ AlertStatus.Ignored
+      case AlertStatus.Imported ⇒ AlertStatus.Ignored
+    }
+    logger.debug(s"Remove case association in alert ${alert.id} (${alert.title}")
+    updateSrv(alert, Fields(Json.obj("case" -> JsNull, "status" -> status)), modifyConfig)
+  }
+
+  def delete(id: String)(implicit authContext: AuthContext): Future[Alert] =
     deleteSrv[AlertModel, Alert](alertModel, id)
 
   def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Alert, NotUsed], Future[Long]) = {

thehive-backend/app/services/ArtifactSrv.scala

@@ -10,26 +10,33 @@ import play.api.libs.json.JsObject
 import play.api.libs.json.JsValue.jsValueToJsLookup
 
 import akka.NotUsed
-import akka.stream.scaladsl.Source
+import akka.stream.Materializer
+import akka.stream.scaladsl.{ Sink, Source }
 import models.{ CaseResolutionStatus, CaseStatus, _ }
 
 import org.elastic4play.ConflictError
 import org.elastic4play.controllers.Fields
-import org.elastic4play.database.ModifyConfig
+import org.elastic4play.database.{ DBRemove, ModifyConfig }
 import org.elastic4play.services._
 import org.elastic4play.utils.{ RichFuture, RichOr }
 
+case class RemoveJobsOf(artifactId: String) extends EventMessage
+
 @Singleton
 class ArtifactSrv @Inject() (
     artifactModel: ArtifactModel,
     caseModel: CaseModel,
+    auditSrv: AuditSrv,
+    eventSrv: EventSrv,
     createSrv: CreateSrv,
     getSrv: GetSrv,
     updateSrv: UpdateSrv,
     deleteSrv: DeleteSrv,
     findSrv: FindSrv,
     fieldsSrv: FieldsSrv,
-    implicit val ec: ExecutionContext) {
+    dbRemove: DBRemove,
+    implicit val ec: ExecutionContext,
+    implicit val mat: Materializer) {
 
   private[ArtifactSrv] lazy val logger = Logger(getClass)
 
@@ -93,9 +100,19 @@ class ArtifactSrv @Inject() (
     updateSrv.apply[ArtifactModel, Artifact](artifactModel, ids, fields, modifyConfig)
   }
 
-  def delete(id: String)(implicit Context: AuthContext): Future[Artifact] =
+  def delete(id: String)(implicit authContext: AuthContext): Future[Artifact] =
     deleteSrv[ArtifactModel, Artifact](artifactModel, id)
 
+  def realDelete(artifact: Artifact): Future[Unit] = {
+    for {
+      _ ← auditSrv.findFor(artifact, Some("all"), Nil)._1
+        .mapAsync(1)(auditSrv.realDelete)
+        .runWith(Sink.ignore)
+      _ = eventSrv.publish(RemoveJobsOf(artifact.id))
+      _ ← dbRemove(artifact)
+    } yield ()
+  }
+
   def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Artifact, NotUsed], Future[Long]) = {
     findSrv[ArtifactModel, Artifact](artifactModel, queryDef, range, sortBy)
   }

thehive-backend/app/services/AuditSrv.scala

@@ -2,18 +2,21 @@ package services
 
 import javax.inject.{ Inject, Singleton }
 
-import scala.concurrent.ExecutionContext
+import scala.concurrent.{ ExecutionContext, Future }
 
 import play.api.Logger
-import play.api.libs.json.{ JsObject, Json }
+import play.api.libs.json.{ JsObject, JsValue, Json }
 
+import org.elastic4play.database.DBRemove
+import akka.NotUsed
 import akka.actor.Actor
+import akka.stream.scaladsl.Source
 import models.{ Audit, AuditModel }
 
 import org.elastic4play.controllers.Fields
 import org.elastic4play.models.{ Attribute, BaseEntity, BaseModelDef }
 import org.elastic4play.services._
-import org.elastic4play.utils.Instance
+import org.elastic4play.utils.{ Instance, RichJson }
 
 trait AuditedModel { self: BaseModelDef ⇒
   def attributes: Seq[Attribute[_]]
@@ -34,6 +37,62 @@ trait AuditedModel { self: BaseModelDef ⇒
   }
 }
 
+@Singleton
+class AuditSrv @Inject() (
+    auditModel: AuditModel,
+    modelSrv: ModelSrv,
+    auxSrv: AuxSrv,
+    dBRemove: DBRemove,
+    findSrv: FindSrv,
+    implicit val ec: ExecutionContext) {
+
+  private[AuditSrv] lazy val logger = Logger(getClass)
+
+  def apply(rootId: Option[String], count: Int): (Source[JsObject, NotUsed], Future[Long]) = {
+    import org.elastic4play.services.QueryDSL._
+
+    val streamableEntities = modelSrv.list.collect {
+      case m: AuditedModel if m.modelName != "user" ⇒ m.modelName
+    }
+
+    val filter = rootId match {
+      case Some(rid) ⇒ and("rootId" ~= rid, "base" ~= true, "objectType" in (streamableEntities: _*))
+      case None      ⇒ and("base" ~= true, "objectType" in (streamableEntities: _*))
+    }
+    val (src, total) = findSrv[AuditModel, Audit](auditModel, filter, Some(s"0-$count"), Seq("-startDate"))
+    val entities = src.mapAsync(5) { audit ⇒
+      val fSummary = findSrv(auditModel, and("requestId" ~= audit.requestId(), "objectType" in (streamableEntities: _*)), groupByField("objectType", groupByField("operation", selectCount)))
+        .map { json ⇒
+          json.collectValues {
+            case objectType: JsObject ⇒ objectType.collectValues {
+              case operation: JsObject ⇒ (operation \ "count").as[JsValue]
+            }
+          }
+        }
+      val fObj = auxSrv.apply(audit.objectType(), audit.objectId(), 10, withStats = false, removeUnaudited = true)
+
+      for {
+        summary ← fSummary
+        obj ← fObj
+      } yield JsObject(Seq("base" → (audit.toJson + ("object" → obj)), "summary" → summary))
+    }
+    (entities, total)
+  }
+
+  def realDelete(audit: Audit): Future[Unit] = {
+    dBRemove(audit).map(_ ⇒ ())
+  }
+
+  def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Audit, NotUsed], Future[Long]) = {
+    findSrv[AuditModel, Audit](auditModel, queryDef, range, sortBy)
+  }
+
+  def findFor(entity: BaseEntity, range: Option[String], sortBy: Seq[String]): (Source[Audit, NotUsed], Future[Long]) = {
+    import org.elastic4play.services.QueryDSL._
+    findSrv[AuditModel, Audit](auditModel, and("objectId" ~= entity.id, "objectType" ~= entity.model.modelName), range, sortBy)
+  }
+}
+
 @Singleton
 class AuditActor @Inject() (
     auditModel: AuditModel,

thehive-backend/app/services/CaseSrv.scala

@@ -1,6 +1,6 @@
 package services
 
-import javax.inject.{ Inject, Singleton }
+import javax.inject.{ Inject, Provider, Singleton }
 
 import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.Try
@@ -10,7 +10,8 @@ import play.api.libs.json.Json.toJsFieldJsValueWrapper
 import play.api.libs.json._
 
 import akka.NotUsed
-import akka.stream.scaladsl.Source
+import akka.stream.Materializer
+import akka.stream.scaladsl.{ Sink, Source }
 import models._
 
 import org.elastic4play.InternalError
@@ -23,39 +24,49 @@ class CaseSrv(
     maxSimilarCases: Int,
     caseModel: CaseModel,
     artifactModel: ArtifactModel,
-    taskModel: TaskModel,
+    taskSrv: TaskSrv,
+    auditSrv: AuditSrv,
+    alertSrvProvider: Provider[AlertSrv],
     createSrv: CreateSrv,
     artifactSrv: ArtifactSrv,
     getSrv: GetSrv,
     updateSrv: UpdateSrv,
     deleteSrv: DeleteSrv,
     findSrv: FindSrv,
-    implicit val ec: ExecutionContext) {
+    implicit val ec: ExecutionContext,
+    implicit val mat: Materializer) {
 
   @Inject() def this(
       configuration: Configuration,
       caseModel: CaseModel,
       artifactModel: ArtifactModel,
-      taskModel: TaskModel,
+      taskSrv: TaskSrv,
+      auditSrv: AuditSrv,
+      alertSrvProvider: Provider[AlertSrv],
       createSrv: CreateSrv,
       artifactSrv: ArtifactSrv,
       getSrv: GetSrv,
       updateSrv: UpdateSrv,
       deleteSrv: DeleteSrv,
       findSrv: FindSrv,
-      ec: ExecutionContext) = this(
+      ec: ExecutionContext,
+      mat: Materializer) = this(
     configuration.getOptional[Int]("maxSimilarCases").getOrElse(100),
     caseModel,
     artifactModel,
-    taskModel,
+    taskSrv,
+    auditSrv,
+    alertSrvProvider,
     createSrv,
     artifactSrv,
     getSrv,
     updateSrv,
     deleteSrv,
     findSrv,
-    ec)
+    ec,
+    mat)
 
+  private lazy val alertSrv = alertSrvProvider.get
   private[CaseSrv] lazy val logger = Logger(getClass)
 
   def applyTemplate(template: CaseTemplate, originalFields: Fields): Fields = {
@@ -93,7 +104,7 @@ class CaseSrv(
         val taskFields = fields.getValues("tasks").collect {
           case task: JsObject ⇒ Fields(task)
         } ++ template.map(_.tasks().map(Fields(_))).getOrElse(Nil)
-        createSrv[TaskModel, Task, Case](taskModel, taskFields.map(caze → _))
+        taskSrv.create(caze, taskFields)
           .map(_ ⇒ caze)
       }
   }
@@ -117,9 +128,31 @@ class CaseSrv(
     updateSrv[CaseModel, Case](caseModel, ids, fields, modifyConfig)
   }
 
-  def delete(id: String)(implicit Context: AuthContext): Future[Case] =
+  def delete(id: String)(implicit authContext: AuthContext): Future[Case] =
     deleteSrv[CaseModel, Case](caseModel, id)
 
+  def realDelete(id: String)(implicit authContext: AuthContext): Future[Unit] = {
+    get(id).flatMap(realDelete)
+  }
+
+  def realDelete(caze: Case)(implicit authContext: AuthContext): Future[Unit] = {
+    import org.elastic4play.services.QueryDSL._
+    for {
+      _ ← taskSrv.find(withParent(caze), Some("all"), Nil)._1
+        .mapAsync(1)(taskSrv.realDelete)
+        .runWith(Sink.ignore)
+      _ ← artifactSrv.find(withParent(caze), Some("all"), Nil)._1
+        .mapAsync(1)(artifactSrv.realDelete)
+        .runWith(Sink.ignore)
+      _ ← auditSrv.findFor(caze, Some("all"), Nil)._1
+        .mapAsync(1)(auditSrv.realDelete)
+        .runWith(Sink.ignore)
+      _ = alertSrv.find("case" ~= caze.id, Some("all"), Nil)._1
+        .mapAsync(1)(alertSrv.unsetCase(_))
+      _ ← deleteSrv.realDelete(caze)
+    } yield ()
+  }
+
   def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Case, NotUsed], Future[Long]) = {
     findSrv[CaseModel, Case](caseModel, queryDef, range, sortBy)
   }
@@ -129,9 +162,9 @@ class CaseSrv(
   def getStats(id: String): Future[JsObject] = {
     import org.elastic4play.services.QueryDSL._
     for {
-      taskStats ← findSrv(taskModel, and(
+      taskStats ← taskSrv.stats(and(
         "_parent" ~= id,
-        "status" in ("Waiting", "InProgress", "Completed")), groupByField("status", selectCount))
+        "status" in ("Waiting", "InProgress", "Completed")), Seq(groupByField("status", selectCount)))
       artifactStats ← findSrv(
         artifactModel,
         and("_parent" ~= id, "status" ~= "Ok"),