diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala index 69cf1d92fc..2e2722b805 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala @@ -98,7 +98,7 @@ class ActionOperationSrv @Inject() ( c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_)) obsType <- observableTypeSrv.getOrFail(EntityIdOrName(dataType)) richObservable <- observableSrv.create( - Observable(Some(dataMessage), 2, ioc = false, sighted = false), + Observable(Some(dataMessage), 2, ioc = false, sighted = false, ignoreSimilarity = None), obsType, dataMessage, Set.empty[String], diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala index 1a1608bd07..5f1256fd91 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala @@ -28,6 +28,7 @@ object Conversion { .withFieldComputed(_.tlp, _.tlp) .withFieldConst(_.ioc, false) .withFieldConst(_.sighted, false) + .withFieldConst(_.ignoreSimilarity, None) .transform } diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala index c63f0b01de..369ead4aa9 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala @@ -17,7 +17,8 @@ case class InputObservable( tlp: Option[Int] = None, tags: Set[String] = Set.empty, ioc: Option[Boolean] = None, - sighted: Option[Boolean] = None + sighted: Option[Boolean] = None, + ignoreSimilarity: Option[Boolean] = None ) object InputObservable { @@ -51,7 +52,8 @@ case class OutputObservable( reports: JsObject, stats: JsObject, seen: Option[Boolean], - `case`: Option[OutputCase] + `case`: Option[OutputCase], + ignoreSimilarity: Option[Boolean] ) object OutputObservable { diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala index e94bcf09c8..4211b762f9 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala @@ -17,7 +17,8 @@ case class InputObservable( tlp: Option[Int] = None, tags: Set[String] = Set.empty, ioc: Option[Boolean] = None, - sighted: Option[Boolean] = None + sighted: Option[Boolean] = None, + ignoreSimilarity: Option[Boolean] = None ) object InputObservable { @@ -48,7 +49,8 @@ case class OutputObservable( sighted: Boolean, reports: JsObject, message: Option[String], - extraData: JsObject + extraData: JsObject, + ignoreSimilarity: Option[Boolean] ) object OutputObservable { diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index 880f31d881..77b45faac6 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -107,7 +107,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp, ioc, sighted), + Observable(message, tlp, ioc, sighted, None), Seq(mainOrganisation), dataType, tags, @@ -228,7 +228,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false), + Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false, ignoreSimilarity = None), Nil, dataType, tags, @@ -448,7 +448,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp, ioc, sighted), + Observable(message, tlp, ioc, sighted, ignoreSimilarity = None), Seq(mainOrganisation), dataType, tags, diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index 1f57a57a3d..95e8d64ecd 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -110,7 +110,7 @@ class MispImportSrv @Inject() ( ) List( ( - Observable(attribute.comment, 0, ioc = false, sighted = false), + Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Right(attribute.data.get) @@ -122,7 +122,7 @@ class MispImportSrv @Inject() ( ) List( ( - Observable(attribute.comment, 0, ioc = false, sighted = false), + Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Left(attribute.value) @@ -140,7 +140,7 @@ class MispImportSrv @Inject() ( s"attribute ${attribute.category}:${attribute.`type`} (${attribute.tags}) is converted to observable $observableType with tags $additionalTags" ) ( - Observable(attribute.comment, 0, ioc = false, sighted = false), + Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Left(value) diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 8278610233..407a2604a0 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -123,7 +123,7 @@ class ObservableCtrl @Inject() ( val observables = observableSrv .get(EntityIdOrName(observableId)) .visible - .similar + .filteredSimilar .visible .richObservableWithCustomRenderer(observableLinkRenderer) .toSeq diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala index 6589aa9d9b..5076cdac72 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala @@ -19,7 +19,7 @@ trait ObservableRenderer { def observableStatsRenderer(implicit authContext: AuthContext ): Traversal.V[Observable] => Traversal[JsObject, JMap[JBoolean, JLong], Converter[JsObject, JMap[JBoolean, JLong]]] = - _.similar + _.filteredSimilar .visible .groupCount(_.byValue(_.ioc)) .domainMap { stats => diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index c4d7fe18d6..910afb8f73 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -62,7 +62,7 @@ class ObservableCtrl @Inject() ( ), Query[Traversal.V[Observable], Traversal.V[Observable]]( "similar", - (observableSteps, authContext) => observableSteps.similar.visible(authContext) + (observableSteps, authContext) => observableSteps.filteredSimilar.visible(authContext) ), Query[Traversal.V[Observable], Traversal.V[Case]]("case", (observableSteps, _) => observableSteps.`case`) ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala index 54b9e3cfd0..8bef007b51 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala @@ -20,7 +20,7 @@ trait ObservableRenderer { def seenStats(implicit authContext: AuthContext ): Traversal.V[Observable] => Traversal[JsValue, JMap[JBoolean, JLong], Converter[JsValue, JMap[JBoolean, JLong]]] = - _.similar + _.filteredSimilar .visible .groupCount(_.byValue(_.ioc)) .domainMap { stats => diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 31dd2d149a..ae4d2715ce 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -18,7 +18,7 @@ case class ObservableData() case class ObservableTag() @BuildVertexEntity -case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean) +case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean, ignoreSimilarity: Option[Boolean]) case class RichObservable( observable: Observable with Entity, @@ -39,6 +39,7 @@ case class RichObservable( def tlp: Int = observable.tlp def ioc: Boolean = observable.ioc def sighted: Boolean = observable.sighted + def ignoreSimilarity: Option[Boolean] = observable.ignoreSimilarity def dataOrAttachment: Either[Data with Entity, Attachment with Entity] = data.toLeft(attachment.get) } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index f994c29280..62683434d6 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -68,10 +68,12 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .noop // .addIndex("Tag", IndexType.unique, "namespace", "predicate", "value") .noop // .addIndex("Audit", IndexType.basic, "requestId", "mainAction") .rebuildIndexes + // release 4.0.0 .updateGraph("Remove cases with a Deleted status", "Case") { traversal => traversal.unsafeHas("status", "Deleted").remove() Success(()) } + .addProperty[Option[Boolean]]("Observable", "ignoreSimilarity") val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 0ac7644654..b4f40a6036 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -271,28 +271,30 @@ class AlertSrv @Inject() ( updatedCase <- mergeInCase(alert, case0) } yield updatedCase - def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = { - auditSrv.mergeAudits { - val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" + def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = + auditSrv + .mergeAudits { + val description = `case`.description + s"\n \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}" - for { - _ <- markAsRead(alert._id) - _ <- importObservables(alert, `case`) - _ <- importCustomFields(alert, `case`) - _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") - _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) - // No audit for markAsRead and observables - // Audits for customFields, description and tags - c <- caseSrv.getOrFail(`case`._id) - details <- Success(Json.obj( - "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), - "description" -> c.description, - "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString)) - ) - } yield details - } (details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) + for { + _ <- markAsRead(alert._id) + _ <- importObservables(alert, `case`) + _ <- importCustomFields(alert, `case`) + _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case") + _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet) + // No audit for markAsRead and observables + // Audits for customFields, description and tags + c <- caseSrv.getOrFail(`case`._id) + details <- Success( + Json.obj( + "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson), + "description" -> c.description, + "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString) + ) + ) + } yield details + }(details => auditSrv.alertToCase.merge(alert, `case`, Some(details))) .flatMap(_ => caseSrv.getOrFail(`case`._id)) - } def importObservables(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, @@ -335,10 +337,7 @@ class AlertSrv @Inject() ( .toIterator .toTry { richCustomField => caseSrv - .setOrCreateCustomField(`case`, - richCustomField.customField._id, - richCustomField.value, - richCustomField.customFieldValue.order) + .setOrCreateCustomField(`case`, richCustomField.customField._id, richCustomField.value, richCustomField.customFieldValue.order) } .map(_ => ()) @@ -401,7 +400,7 @@ object AlertOps { authContext: AuthContext ): Traversal[(RichCase, SimilarStats), JMap[String, Any], Converter[(RichCase, SimilarStats), JMap[String, Any]]] = { val similarObservables = observables - .similar + .filteredSimilar .visible maybeCaseFilter .fold(similarObservables)(caseFilter => similarObservables.filter(o => caseFilter(o.`case`))) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 0c0caf292f..8c6a22b303 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -162,7 +162,7 @@ class CaseSrv @Inject() ( ): Try[Unit] = { val alreadyExistInThatCase = observableSrv .get(richObservable.observable) - .similar + .filteredSimilar .visible .`case` .hasId(`case`._id) diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index fa61aa98e0..2a156ec43f 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -274,7 +274,7 @@ object ObservableOps { .by(_.data.fold) .by(_.attachments.fold) .by(_.tags.fold) - .by(_.similar.visible.limit(1).count) + .by(_.filteredSimilar.visible.limit(1).count) .by(_.keyValues.fold) .by(_.reportTags.fold) ) @@ -302,7 +302,7 @@ object ObservableOps { .by(_.data.fold) .by(_.attachments.fold) .by(_.tags.fold) - .by(_.similar.visible.limit(1).count) + .by(_.filteredSimilar.visible.limit(1).count) .by(_.keyValues.fold) .by(_.reportTags.fold) .by(entityRenderer) @@ -333,6 +333,12 @@ object ObservableOps { if (tags.nonEmpty) traversal.outE[ObservableTag].filter(_.otherV.hasId(tags.map(_._id).toSeq: _*)).remove() + def filteredSimilar: Traversal.V[Observable] = + traversal + .hasNot(_.ignoreSimilarity, true) + .similar + .hasNot(_.ignoreSimilarity, true) + def similar: Traversal.V[Observable] = { val originLabel = StepLabel.v[Observable] traversal @@ -341,7 +347,7 @@ object ObservableOps { _.out[ObservableData] .in[ObservableData], _.out[ObservableAttachment] - .in[ObservableAttachment] + .in[ObservableAttachment] // FIXME this doesn't work. Link must be done with attachmentId ) .where(JP.without(originLabel.name)) .dedup diff --git a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala index f63bc0e3e6..3af6601d61 100644 --- a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala @@ -103,7 +103,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { for { observableType <- app[ObservableTypeSrv].getOrFail(EntityName("domain")) observable <- app[ObservableSrv].create( - observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true), + observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None), `type` = observableType, dataValue = "perdu.com", tagNames = Set("tag10"), diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index 75a105d0b9..de36479d95 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -279,7 +279,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { val newObs = app[Database].tryTransaction { implicit graph => app[ObservableSrv].create( - Observable(Some("if you feel lost"), 1, ioc = false, sighted = true), + Observable(Some("if you feel lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None), app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Case").get, "lost.com", Set[String](), diff --git a/thehive/test/org/thp/thehive/services/DataSrvTest.scala b/thehive/test/org/thp/thehive/services/DataSrvTest.scala index 054b578e89..922926dd91 100644 --- a/thehive/test/org/thp/thehive/services/DataSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/DataSrvTest.scala @@ -22,7 +22,7 @@ class DataSrvTest extends PlaySpecification with TestAppBuilder { "get related observables" in testApp { app => app[Database].tryTransaction { implicit graph => app[ObservableSrv].create( - Observable(Some("love"), 1, ioc = false, sighted = true), + Observable(Some("love"), 1, ioc = false, sighted = true, ignoreSimilarity = None), app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Observable").get, "love.com", Set("tagX"),