Use index to search imported alerts

migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala

@@ -731,7 +731,7 @@ class Output @Inject() (
       val `case` = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption)
       for {
         organisation <- getOrganisation(inputAlert.organisation)
-        createdAlert <- alertSrv.createEntity(inputAlert.alert.copy(organisationId = organisation._id, caseId = `case`.map(_._id)))
+        createdAlert <- alertSrv.createEntity(inputAlert.alert.copy(organisationId = organisation._id, caseId = `case`.fold(EntityId.empty)(_._id)))
         tags = inputAlert.alert.tags.flatMap(getTag(_, organisation._id.value).toOption)
         _    = updateMetaData(createdAlert, inputAlert.metaData)
         _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation)

misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala

@@ -162,7 +162,7 @@ class MispExportSrv @Inject() (
           read = false,
           follow = true,
           tags = Nil,
-          caseId = Some(`case`._id)
+          caseId = `case`._id
         )
       }
       createdAlert <- alertSrv.create(alert.copy(lastSyncDate = new Date(0L)), org, Set.empty, Nil, None)

misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala

@@ -73,7 +73,7 @@ class MispImportSrv @Inject() (
           follow = true,
           organisationId = organisationId,
           tags = event.tags.map(_.name),
-          caseId = None
+          caseId = EntityId.empty
         )
       }
 

misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/MispImportSrvTest.scala

@@ -6,7 +6,7 @@ import org.thp.misp.dto.{Event, Organisation, Tag, User}
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models.{Database, DummyUserSrv}
 import org.thp.scalligraph.traversal.TraversalOps._
-import org.thp.scalligraph.{AppBuilder, EntityName}
+import org.thp.scalligraph.{AppBuilder, EntityId, EntityName}
 import org.thp.thehive.TestAppBuilder
 import org.thp.thehive.models.{Alert, Permissions}
 import org.thp.thehive.services.AlertOps._
@@ -94,7 +94,7 @@ class MispImportSrvTest(implicit ec: ExecutionContext) extends PlaySpecification
               follow = true,
               tags = Seq("TH-test", "TH-test-2"),
               organisationId = alert.organisationId,
-              caseId = None
+              caseId = EntityId.empty
             )
           )
         }

thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala

@@ -390,19 +390,19 @@ class PublicAlert @Inject() (
 
   def statusFilter(status: String): Traversal.V[Alert] => Traversal.V[Alert] =
     status match {
-      case "New"      => _.hasNot(_.caseId).has(_.read, false)
-      case "Updated"  => _.has(_.caseId).has(_.read, false)
-      case "Ignored"  => _.hasNot(_.caseId).has(_.read, true)
-      case "Imported" => _.has(_.caseId).has(_.read, true)
+      case "New"      => _.isEmptyId(_.caseId).has(_.read, false)
+      case "Updated"  => _.nonEmptyId(_.caseId).has(_.read, false)
+      case "Ignored"  => _.isEmptyId(_.caseId).has(_.read, true)
+      case "Imported" => _.nonEmptyId(_.caseId).has(_.read, true)
       case _          => _.empty
     }
 
   def statusNotFilter(status: String): Traversal.V[Alert] => Traversal.V[Alert] =
     status match {
-      case "New"      => _.or(_.has(_.caseId), _.has(_.read, true))
-      case "Updated"  => _.or(_.hasNot(_.caseId), _.has(_.read, true))
-      case "Ignored"  => _.or(_.has(_.caseId), _.has(_.read, false))
-      case "Imported" => _.or(_.hasNot(_.caseId), _.has(_.read, false))
+      case "New"      => _.or(_.nonEmptyId(_.caseId), _.has(_.read, true))
+      case "Updated"  => _.or(_.isEmptyId(_.caseId), _.has(_.read, true))
+      case "Ignored"  => _.or(_.nonEmptyId(_.caseId), _.has(_.read, false))
+      case "Imported" => _.or(_.isEmptyId(_.caseId), _.has(_.read, false))
       case _          => identity
     }
 
@@ -503,8 +503,8 @@ class PublicAlert @Inject() (
             predicate.fold(
               b => if (b) alertTraversal else alertTraversal.empty,
               p =>
-                if (p.getValue) alertTraversal.has(_.caseId)
-                else alertTraversal.hasNot(_.caseId)
+                if (p.getValue) alertTraversal.nonEmptyId(_.caseId)
+                else alertTraversal.isEmptyId(_.caseId)
             )
           )
           .readonly

thehive/app/org/thp/thehive/controllers/v0/Conversion.scala

@@ -1,6 +1,7 @@
 package org.thp.thehive.controllers.v0
 
 import io.scalaland.chimney.dsl._
+import org.thp.scalligraph.EntityId
 import org.thp.scalligraph.auth.{AuthContext, Permission, PermissionDesc}
 import org.thp.scalligraph.controllers.Renderer
 import org.thp.scalligraph.models.Entity
@@ -104,7 +105,7 @@ object Conversion {
         .withFieldConst(_.lastSyncDate, new Date)
         .withFieldConst(_.follow, true)
         .withFieldConst(_.tags, inputAlert.tags.toSeq)
-        .withFieldConst(_.caseId, None)
+        .withFieldConst(_.caseId, EntityId.empty)
         .transform
   }
 

thehive/app/org/thp/thehive/controllers/v1/Conversion.scala

@@ -1,6 +1,7 @@
 package org.thp.thehive.controllers.v1
 
 import io.scalaland.chimney.dsl._
+import org.thp.scalligraph.EntityId
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers.Renderer
 import org.thp.scalligraph.models.Entity
@@ -79,7 +80,7 @@ object Conversion {
         .withFieldConst(_.lastSyncDate, new Date)
         .withFieldConst(_.follow, true)
         .withFieldConst(_.tags, inputAlert.tags.toSeq)
-        .withFieldConst(_.caseId, None)
+        .withFieldConst(_.caseId, EntityId.empty)
         .transform
   }
 

thehive/app/org/thp/thehive/controllers/v1/Properties.scala

@@ -95,8 +95,8 @@ class Properties @Inject() (
             predicate.fold(
               b => if (b) alertTraversal else alertTraversal.empty,
               p =>
-                if (p.getValue) alertTraversal.has(_.caseId)
-                else alertTraversal.hasNot(_.caseId)
+                if (p.getValue) alertTraversal.nonEmptyId(_.caseId)
+                else alertTraversal.isEmptyId(_.caseId)
             )
           )
           .readonly

thehive/app/org/thp/thehive/models/Alert.scala

@@ -70,8 +70,8 @@ case class Alert(
     follow: Boolean,
     tags: Seq[String],
     /* filled by the service */
-    organisationId: EntityId = EntityId(""),
-    caseId: Option[EntityId] = None
+    organisationId: EntityId = EntityId.empty,
+    caseId: EntityId = EntityId.empty // empty string means no related case (index optimisation)
 )
 
 case class RichAlert(

thehive/app/org/thp/thehive/models/Log.scala

@@ -17,7 +17,7 @@ case class Log(
     message: String,
     date: Date,
     /* filled by the service */
-    taskId: EntityId = EntityId(""),
+    taskId: EntityId = EntityId.empty,
     organisationIds: Set[EntityId] = Set.empty
 )
 

thehive/app/org/thp/thehive/models/Observable.scala

@@ -40,7 +40,7 @@ case class Observable(
     /* filled by the service */
     data: Option[String] = None,
     attachmentId: Option[String] = None,
-    relatedId: EntityId = EntityId(""),
+    relatedId: EntityId = EntityId.empty,
     organisationIds: Set[EntityId] = Set.empty
 )
 

thehive/app/org/thp/thehive/models/Task.scala

@@ -42,7 +42,7 @@ case class Task(
     dueDate: Option[Date],
     /* filled by the service */
     assignee: Option[String],
-    relatedId: EntityId = EntityId(""),
+    relatedId: EntityId = EntityId.empty,
     organisationIds: Set[EntityId] = Set.empty
 )
 

thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala

@@ -126,21 +126,23 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     }
     .updateGraph("Add each tag to its Organisation's FreeTags taxonomy", "Tag") { tags =>
       tags
+        .unsafeHas("namespace", TextP.notStartingWith("_freetags_"))
         .project(
-          _.by.by(
-            _.unionFlat(
-              _.in("CaseTag").in("ShareCase").in("OrganisationShare"),
-              _.in("ObservableTag").unionFlat(_.in("ShareObservable").in("OrganisationShare"), _.in("AlertObservable").out("AlertOrganisation")),
-              _.in("AlertTag").out("AlertOrganisation"),
-              _.in("CaseTemplateTag").out("CaseTemplateOrganisation")
+          _.by
+            .by(
+              _.unionFlat(
+                _.in("CaseTag").in("ShareCase").in("OrganisationShare"),
+                _.in("ObservableTag").unionFlat(_.in("ShareObservable").in("OrganisationShare"), _.in("AlertObservable").out("AlertOrganisation")),
+                _.in("AlertTag").out("AlertOrganisation"),
+                _.in("CaseTemplateTag").out("CaseTemplateOrganisation")
+              )
+                .dedup
+                .sort(_.by("_createdAt", Order.desc))
+                .limit(1)
+                .out("OrganisationTaxonomy")
+                .unsafeHas("namespace", TextP.startingWith("_freetags_"))
+                .option
             )
-              .dedup
-              .sort(_.by("_createdAt", Order.desc))
-              .limit(1)
-              .out("OrganisationTaxonomy")
-              .unsafeHas("namespace", TextP.startingWith("_freetags_"))
-              .option
-          )
         )
         .foreach {
           case (tag, Some(freetagsTaxo)) =>
@@ -197,6 +199,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     .addProperty[Option[EntityId]]("Alert", "caseId")
     .updateGraph("Add tags, organisationId and caseId in alerts", "Alert") { traversal =>
       traversal
+        .unsafeHasNot("organisationId")
         .project(
           _.by
             .by(_.out("AlertTag").valueMap("namespace", "predicate", "value").fold)
@@ -290,6 +293,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     .addProperty[Set[EntityId]]("Log", "organisationIds")
     .updateGraph("Add taskId and organisationIds data in logs", "Log") { traversal =>
       traversal
+        .unsafeHasNot("organisationIds")
         .project(
           _.by
             .by(_.in("TaskLog")._id.option)
@@ -310,6 +314,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     .addProperty[Set[EntityId]]("Observable", "organisationIds")
     .updateGraph("Add dataType, tags, data, relatedId and organisationIds data in observables", "Observable") { traversal =>
       traversal
+        .unsafeHasNot("organisationIds")
         .project(
           _.by
             .by(_.out("ObservableObservableType").property("name", UMapping.string).option)
@@ -354,6 +359,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     .addProperty[EntityId]("Task", "relatedId")
     .updateGraph("Add assignee, relatedId and organisationIds data in tasks", "Task") { traversal =>
       traversal
+        .unsafeHasNot("organisationIds")
         .project(
           _.by
             .by(_.out("TaskUser").property("login", UMapping.string).option)
@@ -408,25 +414,24 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
 //        .foreach {
 //          case (vertex, caseId) =>
 //            caseId.foreach(cid => vertex.property("caseId", cid.value))
-//          case _ =>
 //        }
 //      Success(())
 //    }
     .noop
     //=====[release 4.1.1]=====
-    .updateGraph("Set caseId in imported alerts", "Alert") { traversal =>
-      traversal
-        .project(
-          _.by
-            .by(_.out("AlertCase")._id.option)
-        )
-        .foreach {
-          case (vertex, caseId) =>
-            caseId.foreach(cid => vertex.property("caseId", cid.value))
-          case _ =>
-        }
-      Success(())
-    }
+//    .updateGraph("Set caseId in imported alerts", "Alert") { traversal =>
+//      traversal
+//        .project(
+//          _.by
+//            .by(_.out("AlertCase")._id.option)
+//        )
+//        .foreach {
+//          case (vertex, caseId) =>
+//            caseId.foreach(cid => vertex.property("caseId", cid.value))
+//        }
+//      Success(())
+//    }
+    .noop
     .updateGraph("Set taskId in logs", "Log") { traversal =>
       traversal
         .project(_.by.by(_.in("TaskLog")._id.option))
@@ -444,6 +449,19 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
     .removeIndex("Case", IndexType.basic, "status")
     .removeIndex("Task", IndexType.basic, "status")
     .removeIndex("Alert", IndexType.basic, "type", "source", "sourceRef")
+    //=====[release 4.1.3]=====
+    .updateGraph("Set caseId in imported alerts", "Alert") { traversal =>
+      traversal
+        .project(
+          _.by
+            .by(_.out("AlertCase")._id.option)
+        )
+        .foreach {
+          case (vertex, caseId) =>
+            vertex.property("caseId", caseId.fold("")(_.value))
+        }
+      Success(())
+    }
 
   val reflectionClasses = new Reflections(
     new ConfigurationBuilder()

thehive/app/org/thp/thehive/services/AlertSrv.scala

@@ -262,7 +262,7 @@ class AlertSrv @Inject() (
           createdCase <- caseSrv.create(case0, assignee, organisation, customField, caseTemplate, Nil)
           _           <- importObservables(alert.alert, createdCase.`case`)
           _           <- alertCaseSrv.create(AlertCase(), alert.alert, createdCase.`case`)
-          _           <- get(alert.alert).update(_.caseId, Some(createdCase._id)).getOrFail("Alert")
+          _           <- get(alert.alert).update(_.caseId, createdCase._id).getOrFail("Alert")
           _           <- markAsRead(alert._id)
           _ = integrityCheckActor ! EntityAdded("Alert")
         } yield createdCase
@@ -291,7 +291,7 @@ class AlertSrv @Inject() (
             _ <- importCustomFields(alert, `case`)
             _ <- caseSrv.addTags(`case`, alert.tags.toSet)
             _ <- alertCaseSrv.create(AlertCase(), alert, `case`)
-            _ <- get(alert).update(_.caseId, Some(`case`._id)).getOrFail("Alert")
+            _ <- get(alert).update(_.caseId, `case`._id).getOrFail("Alert")
             c <- caseSrv.get(`case`).update(_.description, description).getOrFail("Case")
             details <- Success(
               Json.obj(
@@ -402,10 +402,10 @@ object AlertOps {
       else traversal.empty
 
     def imported: Traversal[Boolean, Boolean, IdentityConverter[Boolean]] =
-      traversal.choose(_.has(_.caseId), onTrue = true, onFalse = false)
+      traversal.choose(_.nonEmptyId(_.caseId), onTrue = true, onFalse = false)
 
     def isImported: Boolean =
-      traversal.has(_.caseId).exists
+      traversal.nonEmptyId(_.caseId).exists
 
     def importDate: Traversal[Date, Date, Converter[Date, Date]] =
       traversal.outE[AlertCase].value(_._createdAt)

thehive/app/org/thp/thehive/services/CaseSrv.scala

@@ -365,7 +365,7 @@ class CaseSrv @Inject() (
             _ <-
               get(c)
                 .alert
-                .update(_.caseId, Some(richCase._id))
+                .update(_.caseId, richCase._id)
                 .toSeq
                 .toTry(alertSrv.alertCaseSrv.create(AlertCase(), _, richCase.`case`))
             _ <-

thehive/test/org/thp/thehive/DatabaseBuilder.scala

@@ -215,7 +215,7 @@ class DatabaseBuilder @Inject() (
             .foreach {
               case (alert, organisationId, caseId) =>
                 alert.tags.foreach(tag => tagSrv.getOrCreate(tag).flatMap(alertSrv.alertTagSrv.create(AlertTag(), alert, _)).get)
-                alertSrv.get(alert).update(_.organisationId, organisationId).update(_.caseId, caseId).iterate()
+                alertSrv.get(alert).update(_.organisationId, organisationId).update(_.caseId, caseId.getOrElse(EntityId.empty)).iterate()
             }
 
           observableSrv

thehive/test/org/thp/thehive/services/AlertSrvTest.scala

@@ -3,7 +3,7 @@ package org.thp.thehive.services
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models._
 import org.thp.scalligraph.traversal.TraversalOps._
-import org.thp.scalligraph.{EntityIdOrName, EntityName}
+import org.thp.scalligraph.{EntityId, EntityIdOrName, EntityName}
 import org.thp.thehive.TestAppBuilder
 import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
@@ -40,7 +40,7 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder {
             follow = false,
             organisationId = organisation._id,
             tags = Seq("tag1", "tag2"),
-            caseId = None
+            caseId = EntityId.empty
           ),
           organisation,
           Set("tag1", "tag2"),