#1552 WIP Cleaned customFields ordering

dto/src/main/scala/org/thp/thehive/dto/v1/Case.scala

@@ -19,7 +19,7 @@ case class InputCase(
     summary: Option[String] = None,
     user: Option[String] = None,
     @WithParser(InputCustomFieldValue.parser)
-    customFieldValue: Seq[InputCustomFieldValue] = Nil
+    customFieldValues: Seq[InputCustomFieldValue] = Nil
 )
 
 object InputCase {

dto/src/main/scala/org/thp/thehive/dto/v1/CustomFieldValue.scala

@@ -48,25 +48,23 @@ object InputCustomFieldValue {
     case (_, FObject(fields)) =>
       fields
         .toSeq
-        .zipWithIndex
         .validatedBy {
-          case ((name, valueField), i) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, Some(i)))
+          case (name, valueField) => valueParser(valueField).map(v => InputCustomFieldValue(name, v, None))
         }
         .map(_.toSeq)
     case (_, FSeq(list)) =>
       list
-        .zipWithIndex
         .validatedBy {
-        case (cf: FObject, i) =>
-          val order = FieldsParser.int(cf.get("order")).getOrElse(i)
+        case cf: FObject =>
+          val order = FieldsParser.int(cf.get("order")).toOption
           for {
             name  <- FieldsParser.string(cf.get("name"))
             value <- valueParser(cf.get("value"))
-          } yield InputCustomFieldValue(name, value, Some(order))
-        case (other, i) =>
+          } yield InputCustomFieldValue(name, value, order)
+        case other =>
           Bad(
             One(
-              InvalidFormatAttributeError(s"customField[$i]", "CustomFieldValue", Set.empty, other)
+              InvalidFormatAttributeError(s"customField", "CustomFieldValue", Set.empty, other)
             )
           )
       }
@@ -82,6 +80,7 @@ object InputCustomFieldValue {
       case InputCustomFieldValue(name, None, _)             => name -> JsNull
       case InputCustomFieldValue(name, other, _)            => sys.error(s"The custom field $name has invalid value: $other (${other.getClass})")
     }
+    // TODO Change JsObject to JsArray ?
     JsObject(fields)
   }
 }

migration/src/main/scala/org/thp/thehive/migration/dto/InputCaseTemplate.scala

@@ -1,11 +1,12 @@
 package org.thp.thehive.migration.dto
 
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models.CaseTemplate
 
 case class InputCaseTemplate(
     metaData: MetaData,
     caseTemplate: CaseTemplate,
     organisation: String,
     tags: Set[String],
-    customFields: Seq[(String, Option[Any], Option[Int])]
+    customFields: Seq[InputCustomFieldValue]
 )

migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala

@@ -8,6 +8,7 @@ import akka.util.ByteString
 import org.thp.scalligraph.utils.Hash
 import org.thp.thehive.connector.cortex.models.{Action, Job, JobStatus}
 import org.thp.thehive.controllers.v0
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.migration.dto._
 import org.thp.thehive.models._
 import play.api.libs.functional.syntax._
@@ -338,12 +339,12 @@ trait Conversion {
       tags    = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty)
       metrics = (json \ "metrics").asOpt[JsObject].getOrElse(JsObject.empty)
       metricsValue = metrics.value.map {
-        case (name, value) => (name, Some(value), None)
+        case (name, value) => InputCustomFieldValue(name, Some(value), None)
       }
       customFields <- (json \ "customFields").validateOpt[JsObject]
       customFieldsValue = customFields.getOrElse(JsObject.empty).value.map {
         case (name, value) =>
-          (
+          InputCustomFieldValue(
             name,
             Some((value \ "string") orElse (value \ "boolean") orElse (value \ "number") orElse (value \ "date") getOrElse JsNull),
             (value \ "order").asOpt[Int]

migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala

@@ -16,6 +16,7 @@ import org.thp.scalligraph.traversal.Traversal
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.thehive.connector.cortex.models.{CortexSchemaDefinition, TheHiveCortexSchemaProvider}
 import org.thp.thehive.connector.cortex.services.{ActionSrv, JobSrv}
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.migration
 import org.thp.thehive.migration.IdMapping
 import org.thp.thehive.migration.dto._
@@ -467,7 +468,7 @@ class Output @Inject() (
         richCaseTemplate <- caseTemplateSrv.create(inputCaseTemplate.caseTemplate, organisation, tags, Nil, Nil)
         _ = updateMetaData(richCaseTemplate.caseTemplate, inputCaseTemplate.metaData)
         _ = inputCaseTemplate.customFields.foreach {
-          case (name, value, order) =>
+          case InputCustomFieldValue(name, value, order) =>
             (for {
               cf  <- getCustomField(name)
               ccf <- CustomFieldType.map(cf.`type`).setValue(CaseTemplateCustomField(order = order), value)
@@ -678,7 +679,7 @@ class Output @Inject() (
         _     <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), alert, organisation)
         _     <- caseTemplate.map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), alert, ct)).flip
         _     <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t))
-        _     <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, name, value, None) }
+        _     <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, InputCustomFieldValue(name, value, None)) }
         _ = updateMetaData(alert, inputAlert.metaData)
         _ = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _))
       } yield IdMapping(inputAlert.metaData.id, alert._id)

thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala

@@ -14,6 +14,7 @@ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal}
 import org.thp.scalligraph.{AuthorizationError, BadRequestError, EntityId, EntityIdOrName, EntityName, InvalidFormatAttributeError, RichSeq}
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase}
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
 import org.thp.thehive.services.AlertOps._
 import org.thp.thehive.services.CaseOps._
@@ -52,7 +53,7 @@ class AlertCtrl @Inject() (
         val caseTemplateName: Option[String]  = request.body("caseTemplate")
         val inputAlert: InputAlert            = request.body("alert")
         val observables: Seq[InputObservable] = request.body("observables")
-        val customFields                      = inputAlert.customFields.map(c => (c.name, c.value, c.order))
+        val customFields                      = inputAlert.customFields.map(c => InputCustomFieldValue(c.name, c.value, c.order))
         val caseTemplate                      = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption)
         for {
           organisation <-
@@ -428,7 +429,7 @@ class PublicAlert @Inject() (
         case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) =>
           for {
             c <- alertSrv.getByIds(EntityId(vertex.id))(graph).getOrFail("Alert")
-            _ <- alertSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext)
+            _ <- alertSrv.setOrCreateCustomField(c, InputCustomFieldValue(name, Some(value), None))(graph, authContext)
           } yield Json.obj(s"customField.$name" -> value)
         case (FPathElem(_, FPathEmpty), values: JsObject, vertex, _, graph, authContext) =>
           for {

thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala

@@ -12,6 +12,7 @@ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal}
 import org.thp.scalligraph.{RichSeq, _}
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.{InputCase, InputTask}
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
 import org.thp.thehive.services.CaseOps._
 import org.thp.thehive.services.CaseTemplateOps._
@@ -46,7 +47,7 @@ class CaseCtrl @Inject() (
         val caseTemplateName: Option[String] = request.body("caseTemplate")
         val inputCase: InputCase             = request.body("case")
         val inputTasks: Seq[InputTask]       = request.body("tasks")
-        val customFields                     = inputCase.customFields.map(c => (c.name, c.value, c.order))
+        val customFields                     = inputCase.customFields.map(c => InputCustomFieldValue(c.name, c.value, c.order))
         for {
           organisation <-
             userSrv

thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala

@@ -10,7 +10,7 @@ import org.thp.scalligraph.query._
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal}
 import org.thp.thehive.controllers.v1.Conversion._
-import org.thp.thehive.dto.v1.InputAlert
+import org.thp.thehive.dto.v1.{InputAlert, InputCustomFieldValue}
 import org.thp.thehive.models._
 import org.thp.thehive.services.AlertOps._
 import org.thp.thehive.services.CaseTemplateOps._
@@ -85,7 +85,7 @@ class AlertCtrl @Inject() (
         val caseTemplate                     = caseTemplateName.flatMap(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.headOption)
         for {
           organisation <- userSrv.current.organisations(Permissions.manageAlert).getOrFail("Organisation")
-          customFields = inputAlert.customFieldValue.map(cf => (cf.name, cf.value, cf.order))
+          customFields = inputAlert.customFieldValue.map(cf => InputCustomFieldValue(cf.name, cf.value, cf.order))
           richAlert <- alertSrv.create(inputAlert.toAlert, organisation, inputAlert.tags, customFields, caseTemplate)
         } yield Results.Created(richAlert.toJson)
       }

thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala

@@ -72,7 +72,6 @@ class CaseCtrl @Inject() (
         val inputTasks: Seq[InputTask]       = request.body("tasks")
         for {
           caseTemplate <- caseTemplateName.map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).visible.richCaseTemplate.getOrFail("CaseTemplate")).flip
-          customFields = inputCase.customFieldValue.map(cf => (cf.name, cf.value, cf.order))
           organisation <- userSrv.current.organisations(Permissions.manageCase).get(request.organisation).getOrFail("Organisation")
           user         <- inputCase.user.fold[Try[Option[User with Entity]]](Success(None))(u => userSrv.getOrFail(EntityIdOrName(u)).map(Some.apply))
           tags         <- inputCase.tags.toTry(tagSrv.getOrCreate)
@@ -81,7 +80,7 @@ class CaseCtrl @Inject() (
             user,
             organisation,
             tags.toSet,
-            customFields,
+            inputCase.customFieldValues,
             caseTemplate,
             inputTasks.map(t => t.toTask -> t.assignee.flatMap(u => userSrv.get(EntityIdOrName(u)).headOption))
           )

thehive/app/org/thp/thehive/controllers/v1/Conversion.scala

@@ -131,7 +131,7 @@ object Conversion {
         status = inputCase.status,
         summary = inputCase.summary orElse caseTemplate.summary,
         user = inputCase.user,
-        customFieldValue = inputCase.customFieldValue
+        customFieldValues = inputCase.customFieldValues
       )
   }
 

thehive/app/org/thp/thehive/controllers/v1/Properties.scala

@@ -9,6 +9,7 @@ import org.thp.scalligraph.query.{PublicProperties, PublicPropertyListBuilder}
 import org.thp.scalligraph.traversal.Converter
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{BadRequestError, EntityIdOrName, RichSeq}
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
 import org.thp.thehive.services.AlertOps._
 import org.thp.thehive.services.AuditOps._
@@ -96,7 +97,7 @@ class Properties @Inject() (
         case (FPathElem(_, FPathElem(name, _)), value, vertex, _, graph, authContext) =>
           for {
             c <- alertSrv.getOrFail(vertex)(graph)
-            _ <- alertSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext)
+            _ <- alertSrv.setOrCreateCustomField(c, InputCustomFieldValue(name, Some(value), None))(graph, authContext)
           } yield Json.obj(s"customField.$name" -> value)
         case _ => Failure(BadRequestError("Invalid custom fields format"))
       })

thehive/app/org/thp/thehive/services/AlertSrv.scala

@@ -14,6 +14,7 @@ import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{Converter, IdentityConverter, StepLabel, Traversal}
 import org.thp.scalligraph.{CreateError, EntityId, EntityIdOrName, RichOptionTry, RichSeq}
 import org.thp.thehive.controllers.v1.Conversion._
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
 import org.thp.thehive.services.AlertOps._
 import org.thp.thehive.services.CaseOps._
@@ -55,7 +56,7 @@ class AlertSrv @Inject() (
       alert: Alert,
       organisation: Organisation with Entity,
       tagNames: Set[String],
-      customFields: Seq[(String, Option[Any], Option[Int])],
+      customFields: Seq[InputCustomFieldValue],
       caseTemplate: Option[CaseTemplate with Entity]
   )(implicit
       graph: Graph,
@@ -67,7 +68,7 @@ class AlertSrv @Inject() (
       alert: Alert,
       organisation: Organisation with Entity,
       tags: Seq[Tag with Entity],
-      customFields: Seq[(String, Option[Any], Option[Int])],
+      customFields: Seq[InputCustomFieldValue],
       caseTemplate: Option[CaseTemplate with Entity]
   )(implicit
       graph: Graph,
@@ -82,7 +83,7 @@ class AlertSrv @Inject() (
         _            <- alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation)
         _            <- caseTemplate.map(ct => alertCaseTemplateSrv.create(AlertCaseTemplate(), createdAlert, ct)).flip
         _            <- tags.toTry(t => alertTagSrv.create(AlertTag(), createdAlert, t))
-        cfs          <- customFields.toTry { case (name, value, order) => createCustomField(createdAlert, name, value, order) }
+        cfs          <- customFields.toTry { simpleCf: InputCustomFieldValue => createCustomField(createdAlert, simpleCf) }
         richAlert = RichAlert(createdAlert, organisation.name, tags, cfs, None, caseTemplate.map(_.name), 0)
         _ <- auditSrv.alert.create(createdAlert, richAlert.toJson)
       } yield richAlert
@@ -170,25 +171,23 @@ class AlertSrv @Inject() (
 
   def createCustomField(
       alert: Alert with Entity,
-      name: String,
-      value: Option[Any],
-      order: Option[Int]
+      simpleCf: InputCustomFieldValue
   )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] =
     for {
-      cf   <- customFieldSrv.getOrFail(EntityIdOrName(name))
-      ccf  <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), value).map(_.order_=(order))
+      cf   <- customFieldSrv.getOrFail(EntityIdOrName(simpleCf.name))
+      ccf  <- CustomFieldType.map(cf.`type`).setValue(AlertCustomField(), simpleCf.value).map(_.order_=(simpleCf.order))
       ccfe <- alertCustomFieldSrv.create(ccf, alert, cf)
     } yield RichCustomField(cf, ccfe)
 
-  def setOrCreateCustomField(alert: Alert with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit
+  def setOrCreateCustomField(alert: Alert with Entity, simpleCf: InputCustomFieldValue)(implicit
       graph: Graph,
       authContext: AuthContext
   ): Try[Unit] = {
-    val cfv = get(alert).customFields(customFieldName)
+    val cfv = get(alert).customFields(simpleCf.name)
     if (cfv.clone().exists)
-      cfv.setValue(value)
+      cfv.setValue(simpleCf.value)
     else
-      createCustomField(alert, customFieldName, value, order).map(_ => ())
+      createCustomField(alert, simpleCf).map(_ => ())
   }
 
   def getCustomField(alert: Alert with Entity, customFieldName: String)(implicit graph: Graph): Option[RichCustomField] =
@@ -206,7 +205,7 @@ class AlertSrv @Inject() (
       .filterNot(rcf => customFieldNames.contains(rcf.name))
       .foreach(rcf => get(alert).customFields(rcf.name).remove())
     customFieldValues
-      .toTry { case (cf, v) => setOrCreateCustomField(alert, cf.name, Some(v), None) }
+      .toTry { case (cf, v) => setOrCreateCustomField(alert, InputCustomFieldValue(cf.name, Some(v), None)) }
       .map(_ => ())
   }
 
@@ -244,7 +243,7 @@ class AlertSrv @Inject() (
           .caseTemplate
           .map(ct => caseTemplateSrv.get(EntityIdOrName(ct)).richCaseTemplate.getOrFail("CaseTemplate"))
           .flip
-      customField = alert.customFields.map(f => (f.name, f.value, f.order))
+      customField = alert.customFields.map(f => InputCustomFieldValue(f.name, f.value, f.order))
       case0 = Case(
         number = 0,
         title = caseTemplate.flatMap(_.titlePrefix).getOrElse("") + alert.title,

thehive/app/org/thp/thehive/services/CaseSrv.scala

@@ -15,6 +15,7 @@ import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal}
 import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName, RichOptionTry, RichSeq}
 import org.thp.thehive.controllers.v1.Conversion._
+import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
 import org.thp.thehive.services.CaseOps._
 import org.thp.thehive.services.CustomFieldOps._
@@ -62,7 +63,7 @@ class CaseSrv @Inject() (
       user: Option[User with Entity],
       organisation: Organisation with Entity,
       tags: Set[Tag with Entity],
-      customFields: Seq[(String, Option[Any], Option[Int])],
+      customFields: Seq[InputCustomFieldValue],
       caseTemplate: Option[RichCaseTemplate],
       additionalTasks: Seq[(Task, Option[User with Entity])]
   )(implicit graph: Graph, authContext: AuthContext): Try[RichCase] =
@@ -72,27 +73,37 @@ class CaseSrv @Inject() (
       _           <- caseUserSrv.create(CaseUser(), createdCase, assignee)
       _           <- shareSrv.shareCase(owner = true, createdCase, organisation, profileSrv.orgAdmin)
       _           <- caseTemplate.map(ct => caseCaseTemplateSrv.create(CaseCaseTemplate(), createdCase, ct.caseTemplate)).flip
+
       createdTasks <- caseTemplate.fold(additionalTasks)(_.tasks.map(t => t.task -> t.assignee)).toTry {
         case (task, owner) => taskSrv.create(task, owner)
       }
       _ <- createdTasks.toTry(t => shareSrv.shareTask(t, createdCase, organisation))
-      caseTemplateCustomFields =
-        caseTemplate
-          .fold[Seq[RichCustomField]](Nil)(_.customFields)
-          .map(cf => (cf.name, cf.value, cf.order))
-      uniqueFields = caseTemplateCustomFields.filter {
-        case (name, _, _) => !customFields.map(c => c._1).contains(name)
-      }
-      cfs <- (uniqueFields ++ customFields).toTry {
-        case (name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order)
+
+      caseTemplateCf = caseTemplate
+        .fold[Seq[RichCustomField]](Seq())(_.customFields)
+        .map(cf => InputCustomFieldValue(cf.name, cf.value, cf.order))
+      cfs <- cleanCustomFields(caseTemplateCf, customFields).toTry {
+        case InputCustomFieldValue(name, value, order) => createCustomField(createdCase, EntityIdOrName(name), value, order)
       }
+
       caseTemplateTags = caseTemplate.fold[Seq[Tag with Entity]](Nil)(_.tags)
       allTags          = tags ++ caseTemplateTags
       _ <- allTags.toTry(t => caseTagSrv.create(CaseTag(), createdCase, t))
+
       richCase = RichCase(createdCase, allTags.toSeq, None, None, Some(assignee.login), cfs, authContext.permissions)
       _ <- auditSrv.`case`.create(createdCase, richCase.toJson)
     } yield richCase
 
+  private def cleanCustomFields(caseTemplateCf: Seq[InputCustomFieldValue], caseCf: Seq[InputCustomFieldValue]): Seq[InputCustomFieldValue] = {
+    val uniqueFields = caseTemplateCf.filter {
+      case InputCustomFieldValue(name, _, _) => !caseCf.map(c => c.name).contains(name)
+    }
+    (caseCf ++ uniqueFields)
+      .sortBy(cf => (cf.order.isEmpty, cf.order))
+      .zipWithIndex
+      .map { case (InputCustomFieldValue(name, value, _), i) => InputCustomFieldValue(name, value, Some(i)) }
+  }
+
   def nextCaseNumber(implicit graph: Graph): Int = startTraversal.getLast.headOption.fold(0)(_.number) + 1
 
   override def exists(e: Case)(implicit graph: Graph): Boolean = startTraversal.getByNumber(e.number).exists

thehive/test/org/thp/thehive/controllers/v0/AlertCtrlTest.scala

@@ -275,8 +275,8 @@ class AlertCtrlTest extends PlaySpecification with TestAppBuilder {
       summary = None,
       owner = Some("[email protected]"),
       customFields = Json.obj(
-        "boolean1" -> Json.obj("boolean" -> JsNull, "order" -> 2),
-        "string1"  -> Json.obj("string" -> "string1 custom field", "order" -> 1)
+        "boolean1" -> Json.obj("boolean" -> JsNull, "order" -> 1),
+        "string1"  -> Json.obj("string" -> "string1 custom field", "order" -> 0)
       ),
       stats = Json.obj()
     )