#1236 Add similar cases in alert output

dto/src/main/scala/org/thp/thehive/dto/v0/Alert.scala

@@ -2,7 +2,7 @@ package org.thp.thehive.dto.v0
 
 import java.util.Date
 
-import play.api.libs.json.{JsObject, Json, OWrites, Reads}
+import play.api.libs.json._
 
 import org.thp.scalligraph.controllers.WithParser
 
@@ -27,6 +27,28 @@ object InputAlert {
   implicit val writes: OWrites[InputAlert] = Json.writes[InputAlert]
 }
 
+case class OutputSimilarCase(
+    _id: String,
+    id: String,
+    caseId: Int, // number
+    title: String,
+    severity: Int,
+    startDate: Date,
+    endDate: Option[Date] = None,
+    resolutionStatus: Option[String] = None,
+    tags: Set[String] = Set.empty,
+    tlp: Int,
+    status: String,
+    similarIOCCount: Int,
+    iocCount: Int,
+    similarArtifactCount: Int,
+    artifactCount: Int
+)
+
+object OutputSimilarCase {
+  implicit val format: OFormat[OutputSimilarCase] = Json.format[OutputSimilarCase]
+}
+
 case class OutputAlert(
     _id: String,
     id: String,
@@ -51,7 +73,8 @@ case class OutputAlert(
     `case`: Option[String],
     customFields: JsObject,
     caseTemplate: Option[String] = None,
-    artifacts: Seq[OutputObservable] = Nil
+    artifacts: Seq[OutputObservable] = Nil,
+    similarCases: Seq[OutputSimilarCase]
 )
 
 object OutputAlert {
@@ -81,6 +104,7 @@ object OutputAlert {
       customFields <- (json \ "customFields").validate[JsObject]
       caseTemplate <- (json \ "caseTemplate").validateOpt[String]
       artifacts    <- (json \ "artifacts").validate[Seq[OutputObservable]]
+      similarCases <- (json \ "similarCases").validate[Seq[OutputSimilarCase]]
     } yield OutputAlert(
       _id,
       id,
@@ -105,7 +129,8 @@ object OutputAlert {
       case0,
       customFields,
       caseTemplate,
-      artifacts
+      artifacts,
+      similarCases
     )
   }
   implicit val writes: OWrites[OutputAlert] = OWrites[OutputAlert] { outputAlert =>

dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala

@@ -49,7 +49,8 @@ case class OutputObservable(
     sighted: Boolean,
     message: Option[String],
     reports: JsObject,
-    stats: JsObject
+    stats: JsObject,
+    seen: Option[Boolean]
 )
 
 object OutputObservable {

thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala

@@ -2,22 +2,26 @@ package org.thp.thehive.controllers.v0
 
 import java.util.Base64
 
+import scala.collection.JavaConverters._
 import scala.util.{Failure, Success, Try}
 
 import play.api.Logger
+import play.api.libs.json.{JsArray, JsObject, Json}
 import play.api.mvc.{Action, AnyContent, Results}
 
-import gremlin.scala.Graph
+import gremlin.scala.{__, By, Graph, Key, StepLabel, Vertex}
+import io.scalaland.chimney.dsl._
 import javax.inject.{Inject, Singleton}
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers.{Entrypoint, FString, FieldsParser}
 import org.thp.scalligraph.models.Database
 import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperty, Query}
-import org.thp.scalligraph.steps.PagedResult
+import org.thp.scalligraph.services._
 import org.thp.scalligraph.steps.StepsOps._
-import org.thp.scalligraph.{RichSeq, _}
+import org.thp.scalligraph.steps.{PagedResult, Traversal}
+import org.thp.scalligraph.{InvalidFormatAttributeError, RichJMap, RichSeq}
 import org.thp.thehive.controllers.v0.Conversion._
-import org.thp.thehive.dto.v0.{InputAlert, InputObservable}
+import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase}
 import org.thp.thehive.models._
 import org.thp.thehive.services._
 
@@ -93,18 +97,92 @@ class AlertCtrl @Inject() (
         } yield Results.Created((richAlert -> richObservables).toJson)
       }
 
+  def alertSimilarityRenderer(implicit authContext: AuthContext, db: Database, graph: Graph): AlertSteps => Traversal[JsArray, JsArray] = {
+    alertSteps =>
+      val observableLabel = StepLabel[Vertex]()
+      val caseLabel       = StepLabel[Vertex]()
+      Traversal(
+        alertSteps
+          .observables
+          .similar
+          .as(observableLabel)
+          .`case`
+          .as(caseLabel)
+          .raw
+          .select(observableLabel.name, caseLabel.name)
+          .fold
+          .map { resultMapList =>
+            val similarCases = resultMapList
+              .asScala
+              .map { m =>
+                val cid = m.getValue(caseLabel).id()
+                val ioc = m.getValue(observableLabel).value[Boolean]("ioc")
+                cid -> ioc
+              }
+              .groupBy(_._1)
+              .map {
+                case (cid, cidIoc) =>
+                  val iocStats = cidIoc.groupBy(_._2).mapValues(_.size)
+                  val (caseVertex, observableCount, resolutionStatus) = caseSrv
+                    .getByIds(cid.toString)
+                    .project(
+                      _(By[Vertex]())
+                        .and(By(new CaseSteps(__[Vertex]).observables(authContext).groupCount(By(Key[Boolean]("ioc"))).raw))
+                        .and(By(__[Vertex].outTo[CaseResolutionStatus].values[String]("value").fold))
+                    )
+                    .head()
+                  val case0 = caseVertex
+                    .as[Case]
+                  val similarCase = case0
+                    .asInstanceOf[Case]
+                    .into[OutputSimilarCase]
+                    .withFieldConst(_.artifactCount, observableCount.getOrDefault(false, 0L).toInt)
+                    .withFieldConst(_.iocCount, observableCount.getOrDefault(true, 0L).toInt)
+                    .withFieldConst(_.similarArtifactCount, iocStats.getOrElse(false, 0))
+                    .withFieldConst(_.similarIOCCount, iocStats.getOrElse(true, 0))
+                    .withFieldConst(_.resolutionStatus, atMostOneOf[String](resolutionStatus))
+                    .withFieldComputed(_.status, _.status.toString)
+                    .withFieldConst(_.id, case0._id)
+                    .withFieldConst(_._id, case0._id)
+                    .withFieldRenamed(_.number, _.caseId)
+                    .transform
+                  Json.toJson(similarCase)
+              }
+            JsArray(similarCases.toSeq)
+          }
+      )
+  }
+
   def get(alertId: String): Action[AnyContent] =
     entrypoint("get alert")
+      .extract("similarity", FieldsParser[Boolean].optional.on("similarity"))
       .authRoTransaction(db) { implicit request => implicit graph =>
-        alertSrv
-          .get(alertId)
-          .visible
-          .richAlert
-          .getOrFail()
-          .map { richAlert =>
-            val alertWithObservables: (RichAlert, Seq[RichObservable]) = richAlert -> alertSrv.get(richAlert.alert).observables.richObservable.toList
-            Results.Ok(alertWithObservables.toJson)
-          }
+        val similarity: Option[Boolean] = request.body("similarity")
+        val alert =
+          alertSrv
+            .get(alertId)
+            .visible
+        if (similarity.contains(true))
+          alert
+            .richAlertWithCustomRenderer(alertSimilarityRenderer(request, db, graph))
+            .getOrFail()
+            .map {
+              case (richAlert, similarCases) =>
+                val alertWithObservables: (RichAlert, Seq[RichObservable]) =
+                  richAlert -> alertSrv.get(richAlert.alert).observables.richObservableWithSeen.toList
+
+                Results.Ok(alertWithObservables.toJson.as[JsObject] + ("similarCases" -> similarCases))
+            }
+        else
+          alert
+            .richAlert
+            .getOrFail()
+            .map { richAlert =>
+              val alertWithObservables: (RichAlert, Seq[RichObservable]) =
+                richAlert -> alertSrv.get(richAlert.alert).observables.richObservable.toList
+              Results.Ok(alertWithObservables.toJson)
+            }
+
       }
 
   def update(alertId: String): Action[AnyContent] =

thehive/app/org/thp/thehive/controllers/v0/Conversion.scala

@@ -56,6 +56,7 @@ object Conversion {
             case (_, false)    => "Updated"
           }
       )
+      .withFieldConst(_.similarCases, Nil)
       .transform
   )
 
@@ -82,6 +83,7 @@ object Conversion {
             }
         )
         .withFieldConst(_.artifacts, richAlertWithObservables._2.map(_.toOutput))
+        .withFieldConst(_.similarCases, Nil)
         .transform
     )
 

thehive/app/org/thp/thehive/models/Observable.scala

@@ -26,6 +26,7 @@ case class RichObservable(
     data: Option[Data with Entity],
     attachment: Option[Attachment with Entity],
     tags: Seq[Tag with Entity],
+    seen: Option[Boolean],
     extensions: Seq[KeyValue with Entity],
     reportTags: Seq[ReportTag with Entity]
 ) {

thehive/app/org/thp/thehive/services/AlertSrv.scala

@@ -15,7 +15,7 @@ import org.thp.scalligraph.models._
 import org.thp.scalligraph.query.PropertyUpdater
 import org.thp.scalligraph.services._
 import org.thp.scalligraph.steps.StepsOps._
-import org.thp.scalligraph.steps.{Traversal, VertexSteps}
+import org.thp.scalligraph.steps.{Traversal, TraversalLike, VertexSteps}
 import org.thp.scalligraph.{CreateError, EntitySteps, InternalError, RichJMap, RichOptionTry, RichSeq}
 import org.thp.thehive.controllers.v1.Conversion._
 import org.thp.thehive.models._
@@ -372,6 +372,40 @@ class AlertSteps(raw: GremlinScala[Vertex])(implicit db: Database, graph: Graph)
 
   def observables: ObservableSteps = new ObservableSteps(raw.outTo[AlertObservable])
 
+  def richAlertWithCustomRenderer[A](
+      entityRenderer: AlertSteps => TraversalLike[_, A]
+  )(implicit authContext: AuthContext): Traversal[(RichAlert, A), (RichAlert, A)] =
+    Traversal(
+      raw
+        .project(
+          _.apply(By[Vertex]())
+            .and(By(__[Vertex].outTo[AlertOrganisation].values[String]("name").fold))
+            .and(By(__[Vertex].outTo[AlertTag].fold))
+            .and(By(__[Vertex].outToE[AlertCustomField].inV().path.fold))
+            .and(By(__[Vertex].outTo[AlertCase].id().fold))
+            .and(By(__[Vertex].outTo[AlertCaseTemplate].values[String]("name").fold))
+            .and(By(entityRenderer(newInstance(__[Vertex])).raw))
+        )
+        .map {
+          case (alert, organisation, tags, customFields, caseId, caseTemplate, renderedEntity) =>
+            val customFieldValues = (customFields: JList[Path])
+              .asScala
+              .map(_.asScala.takeRight(2).toList.asInstanceOf[List[Element]])
+              .map {
+                case List(acf, cf) => RichCustomField(cf.as[CustomField], acf.as[AlertCustomField])
+                case _             => throw InternalError("Not possible")
+              }
+            RichAlert(
+              alert.as[Alert],
+              onlyOneOf[String](organisation),
+              tags.asScala.map(_.as[Tag]),
+              customFieldValues,
+              atMostOneOf[AnyRef](caseId).map(_.toString),
+              atMostOneOf[String](caseTemplate)
+            ) -> renderedEntity
+        }
+    )
+
   def richAlert: Traversal[RichAlert, RichAlert] =
     Traversal(
       raw

thehive/app/org/thp/thehive/services/ObservableSrv.scala

@@ -66,7 +66,7 @@ class ObservableSrv @Inject() (
       _                 <- observableAttachmentSrv.create(ObservableAttachment(), createdObservable, attachment)
       tags              <- addTags(createdObservable, tagNames)
       ext               <- addExtensions(createdObservable, extensions)
-    } yield RichObservable(createdObservable, `type`, None, Some(attachment), tags, ext, Nil)
+    } yield RichObservable(createdObservable, `type`, None, Some(attachment), tags, None, ext, Nil)
 
   def create(observable: Observable, `type`: ObservableType with Entity, dataValue: String, tagNames: Set[String], extensions: Seq[KeyValue])(
       implicit graph: Graph,
@@ -79,7 +79,7 @@ class ObservableSrv @Inject() (
       _                 <- observableDataSrv.create(ObservableData(), createdObservable, data)
       tags              <- addTags(createdObservable, tagNames)
       ext               <- addExtensions(createdObservable, extensions)
-    } yield RichObservable(createdObservable, `type`, Some(data), None, tags, ext, Nil)
+    } yield RichObservable(createdObservable, `type`, Some(data), None, tags, None, ext, Nil)
 
   def addTags(observable: Observable with Entity, tags: Set[String])(implicit graph: Graph, authContext: AuthContext): Try[Seq[Tag with Entity]] = {
     val currentTags = get(observable)
@@ -215,6 +215,35 @@ class ObservableSteps(raw: GremlinScala[Vertex])(implicit db: Database, graph: G
               atMostOneOf[Vertex](data).map(_.as[Data]),
               atMostOneOf[Vertex](attachment).map(_.as[Attachment]),
               tags.asScala.map(_.as[Tag]),
+              None,
+              extensions.asScala.map(_.as[KeyValue]),
+              reportTags.asScala.map(_.as[ReportTag])
+            )
+        }
+    )
+
+  def richObservableWithSeen(implicit authContext: AuthContext): Traversal[RichObservable, RichObservable] =
+    Traversal(
+      raw
+        .project(
+          _.apply(By[Vertex]())
+            .and(By(__[Vertex].outTo[ObservableObservableType].fold))
+            .and(By(__[Vertex].outTo[ObservableData].fold))
+            .and(By(__[Vertex].outTo[ObservableAttachment].fold))
+            .and(By(__[Vertex].outTo[ObservableTag].fold))
+            .and(By(new ObservableSteps(__[Vertex]).similar.visible.raw.limit(1).count))
+            .and(By(__[Vertex].outTo[ObservableKeyValue].fold))
+            .and(By(__[Vertex].outTo[ObservableReportTag].fold))
+        )
+        .map {
+          case (observable, tpe, data, attachment, tags, count, extensions, reportTags) =>
+            RichObservable(
+              observable.as[Observable],
+              onlyOneOf[Vertex](tpe).as[ObservableType],
+              atMostOneOf[Vertex](data).map(_.as[Data]),
+              atMostOneOf[Vertex](attachment).map(_.as[Attachment]),
+              tags.asScala.map(_.as[Tag]),
+              Some(count != 0),
               extensions.asScala.map(_.as[KeyValue]),
               reportTags.asScala.map(_.as[ReportTag])
             )
@@ -244,6 +273,7 @@ class ObservableSteps(raw: GremlinScala[Vertex])(implicit db: Database, graph: G
               atMostOneOf[Vertex](data).map(_.as[Data]),
               atMostOneOf[Vertex](attachment).map(_.as[Attachment]),
               tags.asScala.map(_.as[Tag]),
+              None,
               extensions.asScala.map(_.as[KeyValue]),
               reportTags.asScala.map(_.as[ReportTag])
             ) -> renderedEntity

thehive/test/org/thp/thehive/services/ObservableSrvTest.scala

@@ -40,7 +40,7 @@ class ObservableSrvTest extends PlaySpecification with TestAppBuilder {
                 _                 <- printTiming("linkData")(observableSrv.observableDataSrv.create(ObservableData(), createdObservable, data))
                 tags              <- printTiming("createTags")(observableSrv.addTags(createdObservable, tagNames))
                 //          ext               <- observableSrv.addExtensions(createdObservable, extensions)
-              } yield RichObservable(createdObservable, tpe, Some(data), None, tags, Nil, Nil)
+              } yield RichObservable(createdObservable, tpe, Some(data), None, tags, None, Nil, Nil)
             }
           }
         }