Merge branch 'develop-th4' into feature/update-observable-type

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Change Log
 
+## [4.1.9](https://github.com/TheHive-Project/TheHive/milestone/78) (2021-07-23)
+
+**Implemented enhancements:**
+
+- [Enhancement] Add button for index rebuilding [\#2144](https://github.com/TheHive-Project/TheHive/issues/2144)
+
+**Fixed bugs:**
+
+- [Bug] MISP sync delete existing observables when updating existing Alert [\#2134](https://github.com/TheHive-Project/TheHive/issues/2134)
+- [Bug] Livestream emptied of audit logs after TheHive reboot [\#2135](https://github.com/TheHive-Project/TheHive/issues/2135)
+- [Bug] AddTagToCase operation does not work [\#2136](https://github.com/TheHive-Project/TheHive/issues/2136)
+
 ## [4.1.8](https://github.com/TheHive-Project/TheHive/milestone/77) (2021-07-19)
 
 **Implemented enhancements:**

README.md

@@ -66,7 +66,7 @@ TheHive comes with a special multi-tenancy support. It allows the following stra
 ### RBAC
 TheHive comes with a set of permissions and several pre-configured user profiles: 
 
-- `admin`: full administrative permissions on the plateform ; can't manage any Cases or other data related to investigations;
+- `admin`: full administrative permissions on the platform ; can't manage any Cases or other data related to investigations;
 - `org-admin`: manage users and all organisation-level configuration, can create and edit Cases, Tasks, Observables and run Analyzers and Responders;
 - `analyst`: can create and edit _Cases_, _Tasks_, _Observables_ and run _Analyzers_ & _Responders_;
 - `read-only`: Can only read, Cases, Tasks and Observables details;

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala

@@ -72,8 +72,13 @@ class ActionCtrl(
       }
 }
 
-class PublicAction(actionSrv: ActionSrv, override val organisationSrv: OrganisationSrv, override val customFieldSrv: CustomFieldSrv, db: Database)
-    extends PublicData
+class PublicAction(
+    actionSrv: ActionSrv,
+    override val organisationSrv: OrganisationSrv,
+    override val customFieldSrv: CustomFieldSrv,
+    override val customFieldValueSrv: CustomFieldValueSrv,
+    db: Database
+) extends PublicData
     with CortexOps
     with ActionOps {
 
@@ -105,13 +110,12 @@ class PublicAction(actionSrv: ActionSrv, override val organisationSrv: Organisat
   override val extraQueries: Seq[ParamQuery[_]] = Seq(actionsQuery)
   override val publicProperties: PublicProperties =
     PublicPropertyListBuilder[Action]
-      .property("responderId", UMapping.string)(_.field.readonly)
+      .property("responderId", UMapping.string)(_.rename("workerId").readonly)
       .property("objectType", UMapping.string)(_.select(_.context.domainMap(o => fromObjectType(o._label))).readonly)
       .property("status", UMapping.string)(_.field.readonly)
       .property("startDate", UMapping.date)(_.field.readonly)
       .property("objectId", db.idMapping)(_.select(_.out[ActionContext]._id).readonly)
-      .property("responderName", UMapping.string.optional)(_.field.readonly)
+      .property("responderName", UMapping.string.optional)(_.rename("workerName").readonly)
       .property("cortexId", UMapping.string.optional)(_.field.readonly)
-      .property("tlp", UMapping.int.optional)(_.field.readonly)
       .build
 }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala

@@ -109,7 +109,6 @@ class PublicAnalyzerTemplate(analyzerTemplateSrv: AnalyzerTemplateSrv) extends P
   override val outputQuery: Query = Query.output[AnalyzerTemplate with Entity]
   override val publicProperties: PublicProperties = PublicPropertyListBuilder[AnalyzerTemplate]
     .property("analyzerId", UMapping.string)(_.rename("workerId").readonly)
-    .property("reportType", UMapping.string)(_.field.readonly)
     .property("content", UMapping.string)(_.field.updatable)
     .build
 }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexModelDescription.scala

@@ -1,5 +1,6 @@
 package org.thp.thehive.connector.cortex.controllers.v0
 
+import org.thp.scalligraph.models.IndexType
 import org.thp.thehive.controllers.ModelDescription
 import org.thp.thehive.services.{EntityDescription, PropertyDescription}
 import play.api.Logger
@@ -9,10 +10,10 @@ class CortexModelDescription(publicAction: PublicAction, publicAnalyzerTemplate:
 
   override val logger: Logger = Logger(getClass)
   val metadata = Seq(
-    PropertyDescription("createdBy", "user"),
-    PropertyDescription("createdAt", "date"),
-    PropertyDescription("updatedBy", "user"),
-    PropertyDescription("updatedAt", "date")
+    PropertyDescription("createdBy", "user", indexType = IndexType.standard),
+    PropertyDescription("createdAt", "date", indexType = IndexType.standard),
+    PropertyDescription("updatedBy", "user", indexType = IndexType.standard),
+    PropertyDescription("updatedAt", "date", indexType = IndexType.standard)
   )
 
   override def entityDescriptions: Seq[EntityDescription] =
@@ -21,21 +22,19 @@ class CortexModelDescription(publicAction: PublicAction, publicAnalyzerTemplate:
         "action",
         "/connector/cortex/action",
         "listAction",
-        publicAction.publicProperties.list.flatMap(propToDesc("action", _)) ++ metadata
+        publicAction.publicProperties.list.flatMap(propertyDescription("action", _)) ++ metadata
       ),
       EntityDescription(
         "case_artifact_job",
         "/connector/cortex/job",
         "listJob",
-        publicJob.publicProperties.list.flatMap(propToDesc("case_artifact_job", _)) ++ metadata
+        publicJob.publicProperties.list.flatMap(propertyDescription("case_artifact_job", _)) ++ metadata
       ),
       EntityDescription(
         "analyzer_template",
         "/connector/cortex/analyzer/template/",
         "listAnalyzerTemplate",
-        publicAnalyzerTemplate.publicProperties.list.flatMap(propToDesc("analyzer_template", _)) ++ metadata
+        publicAnalyzerTemplate.publicProperties.list.flatMap(propertyDescription("analyzer_template", _)) ++ metadata
       )
     )
-
-  override def customDescription(model: String, propertyName: String): Option[Seq[PropertyDescription]] = None
 }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala

@@ -59,15 +59,18 @@ class CortexQueryExecutor(
   override val version: (Int, Int) = 0 -> 1
 }
 
-class CortexParentIdInputFilter(parentId: String) extends InputQuery[Traversal.Unk, Traversal.Unk] with CortexOps with TheHiveOpsNoDeps {
+class CortexParentIdInputFilter(parentId: String) extends InputFilter with CortexOps with TheHiveOpsNoDeps {
   override def apply(
       publicProperties: PublicProperties,
       traversalType: ru.Type,
       traversal: Traversal.Unk,
       authContext: AuthContext
   ): Traversal.Unk =
     if (traversalType =:= ru.typeOf[Traversal.V[Job]])
-      traversal.asInstanceOf[Traversal.V[Job]].filter(_.observable.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk]
+      if (isNegate)
+        traversal.asInstanceOf[Traversal.V[Job]].filterNot(_.observable.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk]
+      else
+        traversal.asInstanceOf[Traversal.V[Job]].filter(_.observable.get(EntityIdOrName(parentId))).asInstanceOf[Traversal.Unk]
     else throw BadRequestError(s"$traversalType hasn't parent")
 }
 
@@ -76,40 +79,52 @@ class CortexParentIdInputFilter(parentId: String) extends InputQuery[Traversal.U
   *
   * @param parentFilter the query
   */
-class CortexParentQueryInputFilter(parentFilter: InputQuery[Traversal.Unk, Traversal.Unk])
-    extends InputQuery[Traversal.Unk, Traversal.Unk]
-    with CortexOps {
+class CortexParentQueryInputFilter(parentFilter: InputFilter) extends InputFilter with CortexOps {
   override def apply(
       publicProperties: PublicProperties,
       traversalType: ru.Type,
       traversal: Traversal.Unk,
       authContext: AuthContext
   ): Traversal.Unk =
     if (traversalType =:= ru.typeOf[Traversal.V[Job]])
-      traversal
-        .asInstanceOf[Traversal.V[Job]]
-        .filter { t =>
-          parentFilter(publicProperties, ru.typeOf[Traversal.V[Observable]], t.observable.asInstanceOf[Traversal.Unk], authContext)
-        }
-        .asInstanceOf[Traversal.Unk]
+      if (isNegate)
+        traversal
+          .asInstanceOf[Traversal.V[Job]]
+          .filterNot { t =>
+            parentFilter(publicProperties, ru.typeOf[Traversal.V[Observable]], t.observable.asInstanceOf[Traversal.Unk], authContext)
+          }
+          .asInstanceOf[Traversal.Unk]
+      else
+        traversal
+          .asInstanceOf[Traversal.V[Job]]
+          .filter { t =>
+            parentFilter(publicProperties, ru.typeOf[Traversal.V[Observable]], t.observable.asInstanceOf[Traversal.Unk], authContext)
+          }
+          .asInstanceOf[Traversal.Unk]
     else throw BadRequestError(s"$traversalType hasn't parent")
 }
 
-class CortexChildQueryInputFilter(childType: String, childFilter: InputQuery[Traversal.Unk, Traversal.Unk])
-    extends InputQuery[Traversal.Unk, Traversal.Unk]
-    with CortexOps {
+class CortexChildQueryInputFilter(childType: String, childFilter: InputFilter) extends InputFilter with CortexOps {
   override def apply(
       publicProperties: PublicProperties,
       traversalType: ru.Type,
       traversal: Traversal.Unk,
       authContext: AuthContext
   ): Traversal.Unk =
     if (traversalType =:= ru.typeOf[Traversal.V[Observable]] && childType == "case_artifact_job")
-      traversal
-        .asInstanceOf[Traversal.V[Observable]]
-        .filter { t =>
-          childFilter(publicProperties, ru.typeOf[Traversal.V[Job]], t.jobs.asInstanceOf[Traversal.Unk], authContext)
-        }
-        .asInstanceOf[Traversal.Unk]
+      if (isNegate)
+        traversal
+          .asInstanceOf[Traversal.V[Observable]]
+          .filterNot { t =>
+            childFilter(publicProperties, ru.typeOf[Traversal.V[Job]], t.jobs.asInstanceOf[Traversal.Unk], authContext)
+          }
+          .asInstanceOf[Traversal.Unk]
+      else
+        traversal
+          .asInstanceOf[Traversal.V[Observable]]
+          .filter { t =>
+            childFilter(publicProperties, ru.typeOf[Traversal.V[Job]], t.jobs.asInstanceOf[Traversal.Unk], authContext)
+          }
+          .asInstanceOf[Traversal.Unk]
     else throw BadRequestError(s"$traversalType hasn't child of type $childType")
 }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala

@@ -22,7 +22,8 @@ class ActionOperationSrv(
     alertSrv: AlertSrv,
     logSrv: LogSrv,
     organisationSrv: OrganisationSrv,
-    userSrv: UserSrv
+    userSrv: UserSrv,
+    customFieldSrv: CustomFieldSrv
 ) extends CortexOps {
   private[ActionOperationSrv] lazy val logger: Logger = Logger(getClass)
 
@@ -67,8 +68,9 @@ class ActionOperationSrv(
 
       case AddCustomFields(name, _, value) =>
         for {
-          c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddCustomFields without case")))(Success(_))
-          _ <- caseSrv.setOrCreateCustomField(c, EntityIdOrName(name), Some(value), None)
+          c  <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddCustomFields without case")))(Success(_))
+          cf <- customFieldSrv.getOrFail(EntityIdOrName(name))
+          _  <- caseSrv.updateOrCreateCustomField(c, cf, value, None)
         } yield updateOperation(operation)
 
       case CloseTask() =>

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala

@@ -16,7 +16,7 @@ import org.thp.thehive.connector.cortex.models._
 import org.thp.thehive.connector.cortex.services.Conversion._
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.models._
-import org.thp.thehive.services.{CustomFieldSrv, LogSrv, OrganisationSrv, TheHiveOps}
+import org.thp.thehive.services.{CustomFieldSrv, CustomFieldValueSrv, LogSrv, OrganisationSrv, TheHiveOps}
 import play.api.libs.json.{JsObject, JsString, Json, OWrites}
 
 import java.util.{Date, Map => JMap}
@@ -32,6 +32,7 @@ class ActionSrv(
     clientsConfig: ConfigItem[Seq[CortexClientConfig], Seq[CortexClient]],
     override val organisationSrv: OrganisationSrv,
     override val customFieldSrv: CustomFieldSrv,
+    override val customFieldValueSrv: CustomFieldValueSrv,
     implicit val schema: Schema,
     implicit val db: Database,
     implicit val ec: ExecutionContext,
@@ -194,13 +195,13 @@ class ActionSrv(
     */
   def relatedCase(id: EntityId)(implicit graph: Graph): Option[Case with Entity] =
     for {
-      richAction  <- startTraversal.getByIds(id).richAction.getOrFail("Action").toOption
+      richAction  <- startTraversal.getByIds(id).richAction.headOption
       relatedCase <- entityHelper.parentCase(richAction.context)
     } yield relatedCase
 
   def relatedTask(id: EntityId)(implicit graph: Graph): Option[Task with Entity] =
     for {
-      richAction  <- startTraversal.getByIds(id).richAction.getOrFail("Action").toOption
+      richAction  <- startTraversal.getByIds(id).richAction.headOption
       relatedTask <- entityHelper.parentTask(richAction.context)
     } yield relatedTask
 

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala

@@ -17,7 +17,8 @@ class EntityHelper(
     observableSrv: ObservableSrv,
     logSrv: LogSrv,
     override val organisationSrv: OrganisationSrv,
-    override val customFieldSrv: CustomFieldSrv
+    override val customFieldSrv: CustomFieldSrv,
+    override val customFieldValueSrv: CustomFieldValueSrv
 ) extends TheHiveOps {
 
   lazy val logger: Logger = Logger(getClass)
@@ -53,11 +54,12 @@ class EntityHelper(
     */
   def parentCase(entity: Entity)(implicit graph: Graph): Option[Case with Entity] =
     entity._label match {
-      case "Task"  => taskSrv.get(entity).`case`.headOption
-      case "Case"  => caseSrv.get(entity).headOption
-      case "Log"   => logSrv.get(entity).`case`.headOption
-      case "Alert" => None
-      case _       => None
+      case "Task"       => taskSrv.get(entity).`case`.headOption
+      case "Case"       => caseSrv.get(entity).headOption
+      case "Observable" => observableSrv.get(entity).`case`.headOption
+      case "Log"        => logSrv.get(entity).`case`.headOption
+      case "Alert"      => None
+      case _            => None
     }
 
   /**

cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/TestAppBuilder.scala

@@ -36,7 +36,7 @@ trait TestAppBuilder extends LogFileConfig {
             |]
             |""".stripMargin
         )
-      ).withFallback(TestApplication.appWithoutDatabase.configuration)
+      ).withFallback(TestApplication.configuration)
 
       override val thehiveModule: TheHiveModule = buildTheHiveModule(this)
       injectModule(thehiveModule)
@@ -47,15 +47,15 @@ trait TestAppBuilder extends LogFileConfig {
   def destroyApp(app: TestApplication with WithTheHiveModule with WithCortexModule): Unit = {
     destroyTheHiveModule(app.thehiveModule)
     destroyCortexModule(app.cortexModule)
+    app.terminate()
   }
 
   def testApp[A](body: TestApplication with WithTheHiveModule with WithCortexModule => A): A =
     JanusDatabaseProvider
-      .withDatabase(databaseName, buildDatabase, TestApplication.appWithoutDatabase.actorSystem) { db =>
+      .withDatabase(databaseName, buildDatabase, TestApplication.actorSystemForDB) { db =>
         val app = buildApp(db)
-        val res = body(app)
-        destroyApp(app)
-        res
+        try body(app)
+        finally destroyApp(app)
       }
       .get
 }

cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/ActionSrvTest.scala

@@ -89,7 +89,7 @@ class ActionSrvTest extends PlaySpecification with TestAppBuilder with TheHiveOp
       import app.cortexModule._
       import app.thehiveModule._
 
-      TheHiveOps(organisationSrv, customFieldSrv) { ops =>
+      TheHiveOps(organisationSrv, customFieldSrv, customFieldValueSrv) { ops =>
         import ops.AlertOpsDefs
 
         implicit val entityWrites: OWrites[Entity] = actionCtrl.entityWrites

cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/EntityHelperTest.scala

@@ -67,7 +67,7 @@ class EntityHelperTest extends PlaySpecification with TestAppBuilder with TheHiv
       import app.cortexModule._
       import app.thehiveModule._
 
-      TheHiveOps(organisationSrv, customFieldSrv) { ops =>
+      TheHiveOps(organisationSrv, customFieldSrv, customFieldValueSrv) { ops =>
         import ops.AlertOpsDefs
 
         database.roTransaction { implicit graph =>