From 75a9e90c268ff628f210ed27da736ab96fb5d770 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 11 Mar 2020 12:16:02 +0100 Subject: [PATCH] #1236 Add similar cases in alert output --- ScalliGraph | 2 +- .../scala/org/thp/thehive/dto/v0/Alert.scala | 31 +++++- .../org/thp/thehive/dto/v0/Observable.scala | 3 +- .../thehive/controllers/v0/AlertCtrl.scala | 104 +++++++++++++++--- .../thehive/controllers/v0/Conversion.scala | 2 + .../org/thp/thehive/models/Observable.scala | 1 + .../org/thp/thehive/services/AlertSrv.scala | 36 +++++- .../thp/thehive/services/ObservableSrv.scala | 34 +++++- .../thehive/services/ObservableSrvTest.scala | 2 +- 9 files changed, 193 insertions(+), 22 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 395121bf38..56e67b99f0 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 395121bf38efdd3aa9aa37ecd925a46a29b459da +Subproject commit 56e67b99f03aca37b6c4748bcc07d2739e8143f1 diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Alert.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Alert.scala index 7ca6a54f3e..41df1ef251 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Alert.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Alert.scala @@ -2,7 +2,7 @@ package org.thp.thehive.dto.v0 import java.util.Date -import play.api.libs.json.{JsObject, Json, OWrites, Reads} +import play.api.libs.json._ import org.thp.scalligraph.controllers.WithParser @@ -27,6 +27,28 @@ object InputAlert { implicit val writes: OWrites[InputAlert] = Json.writes[InputAlert] } +case class OutputSimilarCase( + _id: String, + id: String, + caseId: Int, // number + title: String, + severity: Int, + startDate: Date, + endDate: Option[Date] = None, + resolutionStatus: Option[String] = None, + tags: Set[String] = Set.empty, + tlp: Int, + status: String, + similarIOCCount: Int, + iocCount: Int, + similarArtifactCount: Int, + artifactCount: Int +) + +object OutputSimilarCase { + implicit val format: OFormat[OutputSimilarCase] = Json.format[OutputSimilarCase] +} + case class OutputAlert( _id: String, id: String, @@ -51,7 +73,8 @@ case class OutputAlert( `case`: Option[String], customFields: JsObject, caseTemplate: Option[String] = None, - artifacts: Seq[OutputObservable] = Nil + artifacts: Seq[OutputObservable] = Nil, + similarCases: Seq[OutputSimilarCase] ) object OutputAlert { @@ -81,6 +104,7 @@ object OutputAlert { customFields <- (json \ "customFields").validate[JsObject] caseTemplate <- (json \ "caseTemplate").validateOpt[String] artifacts <- (json \ "artifacts").validate[Seq[OutputObservable]] + similarCases <- (json \ "similarCases").validate[Seq[OutputSimilarCase]] } yield OutputAlert( _id, id, @@ -105,7 +129,8 @@ object OutputAlert { case0, customFields, caseTemplate, - artifacts + artifacts, + similarCases ) } implicit val writes: OWrites[OutputAlert] = OWrites[OutputAlert] { outputAlert => diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala index cb51e84467..8c8775a12f 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala @@ -49,7 +49,8 @@ case class OutputObservable( sighted: Boolean, message: Option[String], reports: JsObject, - stats: JsObject + stats: JsObject, + seen: Option[Boolean] ) object OutputObservable { diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index a4e3bb6853..1ccdd051f4 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -2,22 +2,26 @@ package org.thp.thehive.controllers.v0 import java.util.Base64 +import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} import play.api.Logger +import play.api.libs.json.{JsArray, JsObject, Json} import play.api.mvc.{Action, AnyContent, Results} -import gremlin.scala.Graph +import gremlin.scala.{__, By, Graph, Key, StepLabel, Vertex} +import io.scalaland.chimney.dsl._ import javax.inject.{Inject, Singleton} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{Entrypoint, FString, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperty, Query} -import org.thp.scalligraph.steps.PagedResult +import org.thp.scalligraph.services._ import org.thp.scalligraph.steps.StepsOps._ -import org.thp.scalligraph.{RichSeq, _} +import org.thp.scalligraph.steps.{PagedResult, Traversal} +import org.thp.scalligraph.{InvalidFormatAttributeError, RichJMap, RichSeq} import org.thp.thehive.controllers.v0.Conversion._ -import org.thp.thehive.dto.v0.{InputAlert, InputObservable} +import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase} import org.thp.thehive.models._ import org.thp.thehive.services._ @@ -93,18 +97,92 @@ class AlertCtrl @Inject() ( } yield Results.Created((richAlert -> richObservables).toJson) } + def alertSimilarityRenderer(implicit authContext: AuthContext, db: Database, graph: Graph): AlertSteps => Traversal[JsArray, JsArray] = { + alertSteps => + val observableLabel = StepLabel[Vertex]() + val caseLabel = StepLabel[Vertex]() + Traversal( + alertSteps + .observables + .similar + .as(observableLabel) + .`case` + .as(caseLabel) + .raw + .select(observableLabel.name, caseLabel.name) + .fold + .map { resultMapList => + val similarCases = resultMapList + .asScala + .map { m => + val cid = m.getValue(caseLabel).id() + val ioc = m.getValue(observableLabel).value[Boolean]("ioc") + cid -> ioc + } + .groupBy(_._1) + .map { + case (cid, cidIoc) => + val iocStats = cidIoc.groupBy(_._2).mapValues(_.size) + val (caseVertex, observableCount, resolutionStatus) = caseSrv + .getByIds(cid.toString) + .project( + _(By[Vertex]()) + .and(By(new CaseSteps(__[Vertex]).observables(authContext).groupCount(By(Key[Boolean]("ioc"))).raw)) + .and(By(__[Vertex].outTo[CaseResolutionStatus].values[String]("value").fold)) + ) + .head() + val case0 = caseVertex + .as[Case] + val similarCase = case0 + .asInstanceOf[Case] + .into[OutputSimilarCase] + .withFieldConst(_.artifactCount, observableCount.getOrDefault(false, 0L).toInt) + .withFieldConst(_.iocCount, observableCount.getOrDefault(true, 0L).toInt) + .withFieldConst(_.similarArtifactCount, iocStats.getOrElse(false, 0)) + .withFieldConst(_.similarIOCCount, iocStats.getOrElse(true, 0)) + .withFieldConst(_.resolutionStatus, atMostOneOf[String](resolutionStatus)) + .withFieldComputed(_.status, _.status.toString) + .withFieldConst(_.id, case0._id) + .withFieldConst(_._id, case0._id) + .withFieldRenamed(_.number, _.caseId) + .transform + Json.toJson(similarCase) + } + JsArray(similarCases.toSeq) + } + ) + } + def get(alertId: String): Action[AnyContent] = entrypoint("get alert") + .extract("similarity", FieldsParser[Boolean].optional.on("similarity")) .authRoTransaction(db) { implicit request => implicit graph => - alertSrv - .get(alertId) - .visible - .richAlert - .getOrFail() - .map { richAlert => - val alertWithObservables: (RichAlert, Seq[RichObservable]) = richAlert -> alertSrv.get(richAlert.alert).observables.richObservable.toList - Results.Ok(alertWithObservables.toJson) - } + val similarity: Option[Boolean] = request.body("similarity") + val alert = + alertSrv + .get(alertId) + .visible + if (similarity.contains(true)) + alert + .richAlertWithCustomRenderer(alertSimilarityRenderer(request, db, graph)) + .getOrFail() + .map { + case (richAlert, similarCases) => + val alertWithObservables: (RichAlert, Seq[RichObservable]) = + richAlert -> alertSrv.get(richAlert.alert).observables.richObservableWithSeen.toList + + Results.Ok(alertWithObservables.toJson.as[JsObject] + ("similarCases" -> similarCases)) + } + else + alert + .richAlert + .getOrFail() + .map { richAlert => + val alertWithObservables: (RichAlert, Seq[RichObservable]) = + richAlert -> alertSrv.get(richAlert.alert).observables.richObservable.toList + Results.Ok(alertWithObservables.toJson) + } + } def update(alertId: String): Action[AnyContent] = diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index 1a35fbbc95..b6d6786d77 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -56,6 +56,7 @@ object Conversion { case (_, false) => "Updated" } ) + .withFieldConst(_.similarCases, Nil) .transform ) @@ -82,6 +83,7 @@ object Conversion { } ) .withFieldConst(_.artifacts, richAlertWithObservables._2.map(_.toOutput)) + .withFieldConst(_.similarCases, Nil) .transform ) diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 3d99456264..e35a474903 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -26,6 +26,7 @@ case class RichObservable( data: Option[Data with Entity], attachment: Option[Attachment with Entity], tags: Seq[Tag with Entity], + seen: Option[Boolean], extensions: Seq[KeyValue with Entity], reportTags: Seq[ReportTag with Entity] ) { diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 41fc697b73..99e20839f5 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -15,7 +15,7 @@ import org.thp.scalligraph.models._ import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.steps.StepsOps._ -import org.thp.scalligraph.steps.{Traversal, VertexSteps} +import org.thp.scalligraph.steps.{Traversal, TraversalLike, VertexSteps} import org.thp.scalligraph.{CreateError, EntitySteps, InternalError, RichJMap, RichOptionTry, RichSeq} import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.models._ @@ -372,6 +372,40 @@ class AlertSteps(raw: GremlinScala[Vertex])(implicit db: Database, graph: Graph) def observables: ObservableSteps = new ObservableSteps(raw.outTo[AlertObservable]) + def richAlertWithCustomRenderer[A]( + entityRenderer: AlertSteps => TraversalLike[_, A] + )(implicit authContext: AuthContext): Traversal[(RichAlert, A), (RichAlert, A)] = + Traversal( + raw + .project( + _.apply(By[Vertex]()) + .and(By(__[Vertex].outTo[AlertOrganisation].values[String]("name").fold)) + .and(By(__[Vertex].outTo[AlertTag].fold)) + .and(By(__[Vertex].outToE[AlertCustomField].inV().path.fold)) + .and(By(__[Vertex].outTo[AlertCase].id().fold)) + .and(By(__[Vertex].outTo[AlertCaseTemplate].values[String]("name").fold)) + .and(By(entityRenderer(newInstance(__[Vertex])).raw)) + ) + .map { + case (alert, organisation, tags, customFields, caseId, caseTemplate, renderedEntity) => + val customFieldValues = (customFields: JList[Path]) + .asScala + .map(_.asScala.takeRight(2).toList.asInstanceOf[List[Element]]) + .map { + case List(acf, cf) => RichCustomField(cf.as[CustomField], acf.as[AlertCustomField]) + case _ => throw InternalError("Not possible") + } + RichAlert( + alert.as[Alert], + onlyOneOf[String](organisation), + tags.asScala.map(_.as[Tag]), + customFieldValues, + atMostOneOf[AnyRef](caseId).map(_.toString), + atMostOneOf[String](caseTemplate) + ) -> renderedEntity + } + ) + def richAlert: Traversal[RichAlert, RichAlert] = Traversal( raw diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 387c8a6cab..de7f1d2950 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -66,7 +66,7 @@ class ObservableSrv @Inject() ( _ <- observableAttachmentSrv.create(ObservableAttachment(), createdObservable, attachment) tags <- addTags(createdObservable, tagNames) ext <- addExtensions(createdObservable, extensions) - } yield RichObservable(createdObservable, `type`, None, Some(attachment), tags, ext, Nil) + } yield RichObservable(createdObservable, `type`, None, Some(attachment), tags, None, ext, Nil) def create(observable: Observable, `type`: ObservableType with Entity, dataValue: String, tagNames: Set[String], extensions: Seq[KeyValue])( implicit graph: Graph, @@ -79,7 +79,7 @@ class ObservableSrv @Inject() ( _ <- observableDataSrv.create(ObservableData(), createdObservable, data) tags <- addTags(createdObservable, tagNames) ext <- addExtensions(createdObservable, extensions) - } yield RichObservable(createdObservable, `type`, Some(data), None, tags, ext, Nil) + } yield RichObservable(createdObservable, `type`, Some(data), None, tags, None, ext, Nil) def addTags(observable: Observable with Entity, tags: Set[String])(implicit graph: Graph, authContext: AuthContext): Try[Seq[Tag with Entity]] = { val currentTags = get(observable) @@ -215,6 +215,35 @@ class ObservableSteps(raw: GremlinScala[Vertex])(implicit db: Database, graph: G atMostOneOf[Vertex](data).map(_.as[Data]), atMostOneOf[Vertex](attachment).map(_.as[Attachment]), tags.asScala.map(_.as[Tag]), + None, + extensions.asScala.map(_.as[KeyValue]), + reportTags.asScala.map(_.as[ReportTag]) + ) + } + ) + + def richObservableWithSeen(implicit authContext: AuthContext): Traversal[RichObservable, RichObservable] = + Traversal( + raw + .project( + _.apply(By[Vertex]()) + .and(By(__[Vertex].outTo[ObservableObservableType].fold)) + .and(By(__[Vertex].outTo[ObservableData].fold)) + .and(By(__[Vertex].outTo[ObservableAttachment].fold)) + .and(By(__[Vertex].outTo[ObservableTag].fold)) + .and(By(new ObservableSteps(__[Vertex]).similar.visible.raw.limit(1).count)) + .and(By(__[Vertex].outTo[ObservableKeyValue].fold)) + .and(By(__[Vertex].outTo[ObservableReportTag].fold)) + ) + .map { + case (observable, tpe, data, attachment, tags, count, extensions, reportTags) => + RichObservable( + observable.as[Observable], + onlyOneOf[Vertex](tpe).as[ObservableType], + atMostOneOf[Vertex](data).map(_.as[Data]), + atMostOneOf[Vertex](attachment).map(_.as[Attachment]), + tags.asScala.map(_.as[Tag]), + Some(count != 0), extensions.asScala.map(_.as[KeyValue]), reportTags.asScala.map(_.as[ReportTag]) ) @@ -244,6 +273,7 @@ class ObservableSteps(raw: GremlinScala[Vertex])(implicit db: Database, graph: G atMostOneOf[Vertex](data).map(_.as[Data]), atMostOneOf[Vertex](attachment).map(_.as[Attachment]), tags.asScala.map(_.as[Tag]), + None, extensions.asScala.map(_.as[KeyValue]), reportTags.asScala.map(_.as[ReportTag]) ) -> renderedEntity diff --git a/thehive/test/org/thp/thehive/services/ObservableSrvTest.scala b/thehive/test/org/thp/thehive/services/ObservableSrvTest.scala index abe76baf0e..28da3dc459 100644 --- a/thehive/test/org/thp/thehive/services/ObservableSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/ObservableSrvTest.scala @@ -40,7 +40,7 @@ class ObservableSrvTest extends PlaySpecification with TestAppBuilder { _ <- printTiming("linkData")(observableSrv.observableDataSrv.create(ObservableData(), createdObservable, data)) tags <- printTiming("createTags")(observableSrv.addTags(createdObservable, tagNames)) // ext <- observableSrv.addExtensions(createdObservable, extensions) - } yield RichObservable(createdObservable, tpe, Some(data), None, tags, Nil, Nil) + } yield RichObservable(createdObservable, tpe, Some(data), None, tags, None, Nil, Nil) } } }