#1625 Add "ignoreSimilarity" attribute in observable

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala

@@ -98,7 +98,7 @@ class ActionOperationSrv @Inject() (
           c       <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_))
           obsType <- observableTypeSrv.getOrFail(EntityIdOrName(dataType))
           richObservable <- observableSrv.create(
-            Observable(Some(dataMessage), 2, ioc = false, sighted = false),
+            Observable(Some(dataMessage), 2, ioc = false, sighted = false, ignoreSimilarity = None),
             obsType,
             dataMessage,
             Set.empty[String],

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala

@@ -28,6 +28,7 @@ object Conversion {
         .withFieldComputed(_.tlp, _.tlp)
         .withFieldConst(_.ioc, false)
         .withFieldConst(_.sighted, false)
+        .withFieldConst(_.ignoreSimilarity, None)
         .transform
   }
 

dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala

@@ -17,7 +17,8 @@ case class InputObservable(
     tlp: Option[Int] = None,
     tags: Set[String] = Set.empty,
     ioc: Option[Boolean] = None,
-    sighted: Option[Boolean] = None
+    sighted: Option[Boolean] = None,
+    ignoreSimilarity: Option[Boolean] = None
 )
 
 object InputObservable {
@@ -51,7 +52,8 @@ case class OutputObservable(
     reports: JsObject,
     stats: JsObject,
     seen: Option[Boolean],
-    `case`: Option[OutputCase]
+    `case`: Option[OutputCase],
+    ignoreSimilarity: Option[Boolean]
 )
 
 object OutputObservable {

dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala

@@ -17,7 +17,8 @@ case class InputObservable(
     tlp: Option[Int] = None,
     tags: Set[String] = Set.empty,
     ioc: Option[Boolean] = None,
-    sighted: Option[Boolean] = None
+    sighted: Option[Boolean] = None,
+    ignoreSimilarity: Option[Boolean] = None
 )
 
 object InputObservable {
@@ -48,7 +49,8 @@ case class OutputObservable(
     sighted: Boolean,
     reports: JsObject,
     message: Option[String],
-    extraData: JsObject
+    extraData: JsObject,
+    ignoreSimilarity: Option[Boolean]
 )
 
 object OutputObservable {

migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala

@@ -107,7 +107,7 @@ trait Conversion {
           )
     } yield InputObservable(
       metaData,
-      Observable(message, tlp, ioc, sighted),
+      Observable(message, tlp, ioc, sighted, None),
       Seq(mainOrganisation),
       dataType,
       tags,
@@ -228,7 +228,7 @@ trait Conversion {
             )
       } yield InputObservable(
         metaData,
-        Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false),
+        Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false, ignoreSimilarity = None),
         Nil,
         dataType,
         tags,
@@ -448,7 +448,7 @@ trait Conversion {
           )
       } yield InputObservable(
         metaData,
-        Observable(message, tlp, ioc, sighted),
+        Observable(message, tlp, ioc, sighted, ignoreSimilarity = None),
         Seq(mainOrganisation),
         dataType,
         tags,

misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala

@@ -110,7 +110,7 @@ class MispImportSrv @Inject() (
           )
           List(
             (
-              Observable(attribute.comment, 0, ioc = false, sighted = false),
+              Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None),
               observableType,
               attribute.tags.map(_.name).toSet ++ additionalTags,
               Right(attribute.data.get)
@@ -122,7 +122,7 @@ class MispImportSrv @Inject() (
           )
           List(
             (
-              Observable(attribute.comment, 0, ioc = false, sighted = false),
+              Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None),
               observableType,
               attribute.tags.map(_.name).toSet ++ additionalTags,
               Left(attribute.value)
@@ -140,7 +140,7 @@ class MispImportSrv @Inject() (
                   s"attribute ${attribute.category}:${attribute.`type`} (${attribute.tags}) is converted to observable $observableType with tags $additionalTags"
                 )
                 (
-                  Observable(attribute.comment, 0, ioc = false, sighted = false),
+                  Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None),
                   observableType,
                   attribute.tags.map(_.name).toSet ++ additionalTags,
                   Left(value)

thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala

@@ -123,7 +123,7 @@ class ObservableCtrl @Inject() (
         val observables = observableSrv
           .get(EntityIdOrName(observableId))
           .visible
-          .similar
+          .filteredSimilar
           .visible
           .richObservableWithCustomRenderer(observableLinkRenderer)
           .toSeq

thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala

@@ -19,7 +19,7 @@ trait ObservableRenderer {
   def observableStatsRenderer(implicit
       authContext: AuthContext
   ): Traversal.V[Observable] => Traversal[JsObject, JMap[JBoolean, JLong], Converter[JsObject, JMap[JBoolean, JLong]]] =
-    _.similar
+    _.filteredSimilar
       .visible
       .groupCount(_.byValue(_.ioc))
       .domainMap { stats =>

thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala

@@ -62,7 +62,7 @@ class ObservableCtrl @Inject() (
     ),
     Query[Traversal.V[Observable], Traversal.V[Observable]](
       "similar",
-      (observableSteps, authContext) => observableSteps.similar.visible(authContext)
+      (observableSteps, authContext) => observableSteps.filteredSimilar.visible(authContext)
     ),
     Query[Traversal.V[Observable], Traversal.V[Case]]("case", (observableSteps, _) => observableSteps.`case`)
   )

thehive/app/org/thp/thehive/controllers/v1/ObservableRenderer.scala

@@ -20,7 +20,7 @@ trait ObservableRenderer {
   def seenStats(implicit
       authContext: AuthContext
   ): Traversal.V[Observable] => Traversal[JsValue, JMap[JBoolean, JLong], Converter[JsValue, JMap[JBoolean, JLong]]] =
-    _.similar
+    _.filteredSimilar
       .visible
       .groupCount(_.byValue(_.ioc))
       .domainMap { stats =>

thehive/app/org/thp/thehive/models/Observable.scala

@@ -18,7 +18,7 @@ case class ObservableData()
 case class ObservableTag()
 
 @BuildVertexEntity
-case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean)
+case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean, ignoreSimilarity: Option[Boolean])
 
 case class RichObservable(
     observable: Observable with Entity,
@@ -39,6 +39,7 @@ case class RichObservable(
   def tlp: Int                                                           = observable.tlp
   def ioc: Boolean                                                       = observable.ioc
   def sighted: Boolean                                                   = observable.sighted
+  def ignoreSimilarity: Option[Boolean]                                  = observable.ignoreSimilarity
   def dataOrAttachment: Either[Data with Entity, Attachment with Entity] = data.toLeft(attachment.get)
 }
 

thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala

@@ -68,10 +68,12 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     .noop // .addIndex("Tag", IndexType.unique, "namespace", "predicate", "value")
     .noop // .addIndex("Audit", IndexType.basic, "requestId", "mainAction")
     .rebuildIndexes
+    // release 4.0.0
     .updateGraph("Remove cases with a Deleted status", "Case") { traversal =>
       traversal.unsafeHas("status", "Deleted").remove()
       Success(())
     }
+    .addProperty[Option[Boolean]]("Observable", "ignoreSimilarity")
 
   val reflectionClasses = new Reflections(
     new ConfigurationBuilder()

thehive/app/org/thp/thehive/services/AlertSrv.scala

@@ -271,28 +271,30 @@ class AlertSrv @Inject() (
       updatedCase <- mergeInCase(alert, case0)
     } yield updatedCase
 
-  def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] = {
-    auditSrv.mergeAudits {
-      val description = `case`.description + s"\n  \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}"
+  def mergeInCase(alert: Alert with Entity, `case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Case with Entity] =
+    auditSrv
+      .mergeAudits {
+        val description = `case`.description + s"\n  \n#### Merged with alert #${alert.sourceRef} ${alert.title}\n\n${alert.description.trim}"
 
-      for {
-        _ <- markAsRead(alert._id)
-        _ <- importObservables(alert, `case`)
-        _ <- importCustomFields(alert, `case`)
-        _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case")
-        _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet)
-        // No audit for markAsRead and observables
-        // Audits for customFields, description and tags
-        c <- caseSrv.getOrFail(`case`._id)
-        details <- Success(Json.obj(
-          "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson),
-          "description" -> c.description,
-          "tags" -> caseSrv.get(`case`).tags.toSeq.map(_.toString))
-        )
-      } yield details
-    } (details => auditSrv.alertToCase.merge(alert, `case`, Some(details)))
+        for {
+          _ <- markAsRead(alert._id)
+          _ <- importObservables(alert, `case`)
+          _ <- importCustomFields(alert, `case`)
+          _ <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case")
+          _ <- caseSrv.addTags(`case`, get(alert).tags.toSeq.map(_.toString).toSet)
+          // No audit for markAsRead and observables
+          // Audits for customFields, description and tags
+          c <- caseSrv.getOrFail(`case`._id)
+          details <- Success(
+            Json.obj(
+              "customFields" -> get(alert).richCustomFields.toSeq.map(_.toOutput.toJson),
+              "description"  -> c.description,
+              "tags"         -> caseSrv.get(`case`).tags.toSeq.map(_.toString)
+            )
+          )
+        } yield details
+      }(details => auditSrv.alertToCase.merge(alert, `case`, Some(details)))
       .flatMap(_ => caseSrv.getOrFail(`case`._id))
-  }
 
   def importObservables(alert: Alert with Entity, `case`: Case with Entity)(implicit
       graph: Graph,
@@ -335,10 +337,7 @@ class AlertSrv @Inject() (
       .toIterator
       .toTry { richCustomField =>
         caseSrv
-          .setOrCreateCustomField(`case`,
-            richCustomField.customField._id,
-            richCustomField.value,
-            richCustomField.customFieldValue.order)
+          .setOrCreateCustomField(`case`, richCustomField.customField._id, richCustomField.value, richCustomField.customFieldValue.order)
       }
       .map(_ => ())
 
@@ -401,7 +400,7 @@ object AlertOps {
         authContext: AuthContext
     ): Traversal[(RichCase, SimilarStats), JMap[String, Any], Converter[(RichCase, SimilarStats), JMap[String, Any]]] = {
       val similarObservables = observables
-        .similar
+        .filteredSimilar
         .visible
       maybeCaseFilter
         .fold(similarObservables)(caseFilter => similarObservables.filter(o => caseFilter(o.`case`)))

thehive/app/org/thp/thehive/services/CaseSrv.scala

@@ -162,7 +162,7 @@ class CaseSrv @Inject() (
   ): Try[Unit] = {
     val alreadyExistInThatCase = observableSrv
       .get(richObservable.observable)
-      .similar
+      .filteredSimilar
       .visible
       .`case`
       .hasId(`case`._id)

thehive/app/org/thp/thehive/services/ObservableSrv.scala

@@ -274,7 +274,7 @@ object ObservableOps {
             .by(_.data.fold)
             .by(_.attachments.fold)
             .by(_.tags.fold)
-            .by(_.similar.visible.limit(1).count)
+            .by(_.filteredSimilar.visible.limit(1).count)
             .by(_.keyValues.fold)
             .by(_.reportTags.fold)
         )
@@ -302,7 +302,7 @@ object ObservableOps {
             .by(_.data.fold)
             .by(_.attachments.fold)
             .by(_.tags.fold)
-            .by(_.similar.visible.limit(1).count)
+            .by(_.filteredSimilar.visible.limit(1).count)
             .by(_.keyValues.fold)
             .by(_.reportTags.fold)
             .by(entityRenderer)
@@ -333,6 +333,12 @@ object ObservableOps {
       if (tags.nonEmpty)
         traversal.outE[ObservableTag].filter(_.otherV.hasId(tags.map(_._id).toSeq: _*)).remove()
 
+    def filteredSimilar: Traversal.V[Observable] =
+      traversal
+        .hasNot(_.ignoreSimilarity, true)
+        .similar
+        .hasNot(_.ignoreSimilarity, true)
+
     def similar: Traversal.V[Observable] = {
       val originLabel = StepLabel.v[Observable]
       traversal
@@ -341,7 +347,7 @@ object ObservableOps {
           _.out[ObservableData]
             .in[ObservableData],
           _.out[ObservableAttachment]
-            .in[ObservableAttachment]
+            .in[ObservableAttachment] // FIXME this doesn't work. Link must be done with attachmentId
         )
         .where(JP.without(originLabel.name))
         .dedup

thehive/test/org/thp/thehive/services/AlertSrvTest.scala

@@ -103,7 +103,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder {
         for {
           observableType <- app[ObservableTypeSrv].getOrFail(EntityName("domain"))
           observable <- app[ObservableSrv].create(
-            observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true),
+            observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None),
             `type` = observableType,
             dataValue = "perdu.com",
             tagNames = Set("tag10"),

thehive/test/org/thp/thehive/services/CaseSrvTest.scala

@@ -279,7 +279,7 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder {
 
         val newObs = app[Database].tryTransaction { implicit graph =>
           app[ObservableSrv].create(
-            Observable(Some("if you feel lost"), 1, ioc = false, sighted = true),
+            Observable(Some("if you feel lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None),
             app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Case").get,
             "lost.com",
             Set[String](),

thehive/test/org/thp/thehive/services/DataSrvTest.scala

@@ -22,7 +22,7 @@ class DataSrvTest extends PlaySpecification with TestAppBuilder {
     "get related observables" in testApp { app =>
       app[Database].tryTransaction { implicit graph =>
         app[ObservableSrv].create(
-          Observable(Some("love"), 1, ioc = false, sighted = true),
+          Observable(Some("love"), 1, ioc = false, sighted = true, ignoreSimilarity = None),
           app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Observable").get,
           "love.com",
           Set("tagX"),