#67 Database migration (v8)

thehive-backend/app/models/Artifact.scala

@@ -40,6 +40,7 @@ trait ArtifactAttributes { _: AttributeDef ⇒
   val tags = multiAttribute("tags", F.stringFmt, "Artifact tags")
   val ioc = attribute("ioc", F.booleanFmt, "Artifact is an IOC", false)
   val status = attribute("status", F.enumFmt(ArtifactStatus), "Status of the artifact", ArtifactStatus.Ok)
+  val reports = attribute("reports", F.textFmt, "Json object that contains all short reports", "{}", O.unaudited)
 }
 
 @Singleton

thehive-backend/app/models/Case.scala

@@ -41,14 +41,14 @@ trait CaseAttributes { _: AttributeDef ⇒
   val title = attribute("title", F.textFmt, "Title of the case")
   val description = attribute("description", F.textFmt, "Description of the case")
   val severity = attribute("severity", F.numberFmt, "Severity if the case is an incident (0-5)", 3L)
+  val owner = attribute("owner", F.stringFmt, "Owner of the case")
   val startDate = attribute("startDate", F.dateFmt, "Creation date", new Date)
   val endDate = optionalAttribute("endDate", F.dateFmt, "Resolution date")
   val tags = multiAttribute("tags", F.stringFmt, "Case tags")
   val flag = attribute("flag", F.booleanFmt, "Flag of the case", false)
   val tlp = attribute("tlp", F.numberFmt, "TLP level", -1L)
   val status = attribute("status", F.enumFmt(CaseStatus), "Status of the case", CaseStatus.Open)
   val metrics = optionalAttribute("metrics", F.metricsFmt, "List of metrics")
-  val isIncident = attribute("isIncident", F.booleanFmt, "Indicates if this case is an incident (true positive)", false)
   val resolutionStatus = optionalAttribute("resolutionStatus", F.enumFmt(CaseResolutionStatus), "Resolution status of the case")
   val impactStatus = optionalAttribute("impactStatus", F.enumFmt(CaseImpactStatus), "Impact status of the case")
   val summary = optionalAttribute("summary", F.textFmt, "Summary of the case, to be provided when closing a case")

thehive-backend/app/models/Migration.scala

@@ -9,31 +9,25 @@ import scala.concurrent.{ ExecutionContext, Future }
 import scala.math.BigDecimal.int2bigDecimal
 import scala.util.Try
 
-import akka.NotUsed
 import akka.stream.Materializer
-import akka.stream.scaladsl.{ Sink, Source }
 
 import play.api.Logger
-import play.api.libs.json.{ JsArray, JsBoolean, JsDefined }
-import play.api.libs.json.{ JsNumber, JsObject, JsString, JsValue }
-import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup
+import play.api.libs.json.{ JsBoolean, JsNumber, JsObject, JsString, JsValue }
+import play.api.libs.json.{ Json, Reads }
 import play.api.libs.json.JsValue.jsValueToJsLookup
-import play.api.libs.json.Json
-import play.api.libs.json.Json.toJsFieldJsValueWrapper
 
-import org.elastic4play.JsonFormat.dateFormat
 import org.elastic4play.models.BaseModelDef
 import org.elastic4play.services.{ DBLists, DatabaseState, MigrationOperations, Operation }
 import org.elastic4play.utils
-import org.elastic4play.utils.{ Hasher, RichFuture, RichJson }
+import org.elastic4play.utils.RichJson
 
 class Migration @Inject() (
     models: ISet[BaseModelDef],
     dblists: DBLists,
     implicit val ec: ExecutionContext,
     implicit val materializer: Materializer) extends MigrationOperations {
-  import Operation._
-  val log = Logger(getClass)
+  import org.elastic4play.services.Operation._
+  val logger = Logger(getClass)
 
   override def beginMigration(version: Int) = Future.successful(())
 
@@ -47,213 +41,48 @@ class Migration @Inject() (
   }
 
   override val operations: PartialFunction[DatabaseState, Seq[Operation]] = {
-    case previousState @ DatabaseState(1) ⇒
-      // replace content-type by full metadata (from apache tika) in entity that contain attachment
-      /*
-      mapEntity("event_artifact", "task_log") { entity =>
-        (entity \ "hash").asOpt[String] match {
-          case Some(hash) => entity - "contentType" + ("metadata" -> dataStore.getMetadata(hash, (entity \ "attachmentName").as[String]))
-          case None => entity
-        }
-      },
-      */
-      // remove sensitive information (attributes that start with @) in audit details
+    case previousState @ DatabaseState(7) ⇒
       Seq(
-        mapAttribute("audit", "details") {
-          case JsString(details) ⇒
-            Try(Json.parse(details)).map({
-              case JsObject(fields) ⇒ JsObject(fields.filterNot(_._1.startsWith("@")))
-              case a: JsValue       ⇒ a
-            }).getOrElse(JsObject(Seq("value" → JsString(details))))
-          case details ⇒ JsObject(Seq("value" → details))
-        },
-        // don't use document version to store sequence value
-        //        Operation((req: String => Source[JsObject, NotUsed]) => {
-        //          case tableName @ "sequence" =>
-        //            for {
-        //              s <- req(tableName)
-        //              id <- (s \ "id").asOpt[String]
-        //              previousValue = previousState.getEntity(tableName, id).await
-        //              version <- previousValue \ "version" toOption
-        //            } yield s - "dummy" + ("value" -> version)
-        //          case other => req(other)
-        //        }),
+        renameAttribute("reportTemplate", "analyzerId", "analyzers"), // reportTemplate refers only one analyzer
+        renameAttribute("reportTemplate", "reportType", "flavor"), // rename flavor into reportType
 
-        // rename entities
-        mapAttribute("sequence", "id") {
-          case JsString("event") ⇒ JsString("case")
-          case x                 ⇒ x
-        },
-        renameEntity("event", "case"),
-        renameAttribute("case", "caseId", "eventId"),
-        renameEntity("event_artifact", "case_artifact"),
-        renameEntity("artifact_job", "case_artifact_job"),
-        renameEntity("job_log", "case_artifact_job_log"),
-        renameEntity("event_task", "case_task"),
-        renameEntity("task_log", "case_task_log"),
-        renameAttribute("user", "name", "full-name"),
-        // Tag key as private and rootId as non-private
-        renameAttribute("user", "@key", "key"),
-        renameAttribute("audit", "rootId", "@rootId"),
-        // Use user login as entity id
-        mapEntity("user") { user ⇒
-          user \ "login" match {
-            case JsDefined(login: JsString) ⇒ user - "id" - "login" + ("id" → login)
-            case _                          ⇒ user
-          }
+        removeAttribute("case", "isIncident"), // this information is now stored in resolutionStatus
+        mapEntity("case") { c ⇒ // add case owner
+          val owner = (c \ "createdBy")
+            .asOpt[JsString]
+            .getOrElse(JsString("init"))
+          c + ("owner" → owner)
         },
-        // replace user id by user login in Audit
-        mapEntity("audit") { audit ⇒
-          val userMapping = previousState.source("user")
-            .mapConcat { user ⇒
-              (for {
-                id ← (user \ "id").asOpt[String]
-                login ← (user \ "login").asOpt[String]
-              } yield id → login).toList
-            }
-            .runWith(Sink.seq)
-            .await
-            .toMap
 
-          (for {
-            objectType ← (audit \ "objectType").asOpt[String]
-            if objectType == "user"
-            id ← (audit \ "objectId").asOpt[String]
-            login ← userMapping.get(id)
-          } yield audit + ("objectId" → JsString(login))) getOrElse audit
-        })
+        removeEntity("analyzer")(_ ⇒ false), // analyzer is now stored in cortex
 
-    case previousState @ DatabaseState(2) ⇒
-      Seq(
-        // Add flag in task
-        addAttribute("case_task", "flag" → JsBoolean(false)),
-        // Add flag in Case
-        addAttribute("case", "flag" → JsBoolean(false)),
-        // Set TLP if attribute is not present in Artifact
-        addAttributeIfAbsent("case_artifact", "tlp" → JsNumber(-1)),
-        // Add IOC and Label in Artifact
-        addAttribute("case_artifact", "ioc" → JsBoolean(false), "labels" → JsArray(Nil)),
-        // Rename command by @command in analyzer
-        renameAttribute("analyzer", "@command", "command"),
-        // Fix attribute type in job reports
-        mapEntity("case_artifact_job") { job ⇒
-          val analyzerId = (job \ "analyzerId").asOpt[String].getOrElse {
-            //log.error("Job entity has invalid attributes : analyzerId is missing : " + job)
-            "unknown"
-          }
-          rename(job, "error_message", "report" :: analyzerId :: "value" :: Nil)
-        },
-        // Transform comma separated list by array
-        mapAttribute("user", "roles") {
-          case JsString(roles) ⇒ JsArray(roles.split(",").toSeq.map(JsString))
-          case x               ⇒ x
-        },
-        mapAttribute("case", "tags") {
-          case JsString(tags) ⇒ JsArray(tags.split(",").toSeq.map(JsString))
-          case x              ⇒ x
-        })
-    case previousState @ DatabaseState(3) ⇒
-      // artifact data convert from textFmt into stringFmt (become not analyzed)  
-      // no operation required
-      Nil
-    case previousState @ DatabaseState(4) ⇒
-      // saved filesize from datastore in order to insert it in artifacts & logs
-      var attachmentSizes = Map.empty[String, Long]
+        addAttribute("case_artifact", "reports" → JsString("{}")), // add short reports in artifact
 
-      Seq(
-        removeEntity("data") { data ⇒
-          (for {
-            fileSize ← (data \ "fileSize").asOpt[Long]
-            id ← (data \ "id").asOpt[String]
-          } yield {
+        addAttribute("case_task", "order" → JsNumber(0)), // add task order
 
-            attachmentSizes += id → fileSize
-            false
-          }) getOrElse (true)
-        },
-        //      attachment is now an object (no more attributes for metadata nor hash)
-        mapEntity("case_artifact", "case_task_log") { obj ⇒
-          (obj \ "hash").asOpt[String].fold(obj) { hash ⇒
-            val attachmentSize = attachmentSizes.getOrElse(hash, 0L)
-            val attachmentName = (obj \ "attachmentName").asOpt[String].getOrElse("noname")
-            val contentType = (obj \ "metadata" \ "Content-Type").asOpt[String].getOrElse("application/octet-stream")
-            obj - "hash" - "metadata" - "attachmentName" + (
-              "attachment" → Json.obj(
-                "id" → hash,
-                "hashes" → JsArray(Nil),
-                "name" → attachmentName,
-                "size" → attachmentSize,
-                "contentType" → contentType))
-          }
-        },
-        // convert dblist format
-        // before 1 entry for each dblist, after 1 entry for each dblist item
-        Operation((f: String ⇒ Source[JsObject, NotUsed]) ⇒ {
-          case table @ "dblist" ⇒ f(table).mapConcat { list ⇒
-            for {
-              listName ← (list \ "id").asOpt[String].toList
-              items ← (list \ listName).asOpt[Seq[JsValue]].toList
-              item ← items
-              id = Hasher("MD5").fromString(item.toString).head.toString
-            } yield JsObject(Seq("id" → JsString(id), "dblist" → JsString(listName), "value" → JsString(item.toString)))
-          }
-          case other ⇒ f(other)
-        }),
-        // Add resolutionStatus and summary attributes to Case entity
-        addAttribute("case", "resolutionStatus" → JsString("Unknown"), "summary" → JsString("")),
-        // Set case TLP to AMBER(2) by default instead of not specified(-1)
-        mapAttribute("case", "tlp") {
-          case JsNumber(x) if x == -1 ⇒ JsNumber(2)
-          case x                      ⇒ x
-        },
-        // Set observable TLP to AMBER(2) by default instead of not specified(-1)
-        mapAttribute("case_artifact", "tlp") {
-          case JsNumber(x) if x == -1 ⇒ JsNumber(2)
-          case x                      ⇒ x
-        })
-
-    case previousState @ DatabaseState(5) ⇒
-      Seq(
-        renameAttribute("case_artifact", "tags", "labels"),
-        renameAttribute("user", "password", "@password"),
-        renameAttribute("user", "key", "@key"),
-        renameAttribute("data", "binary", "data"),
-        removeAttribute("data", "chunkCount", "fileSize"),
-        renameAttribute("sequence", "counter", "value"),
-        removeAttribute(_ ⇒ true, "$routing"),
-        removeAttribute("case", "ioc"),
-        mapAttribute("case", "resolutionStatus") {
-          case JsString("Unknown")     ⇒ JsString("Indeterminate")
-          case JsString("NotIncident") ⇒ JsString("Other")
-          case x                       ⇒ x
-        },
-        mapAttribute("case_artifact_job", "report") { report ⇒ JsString(report.toString) },
-        removeAttribute("analyzer", "@baseConfig", "@command", "@config", "@report"),
-        renameEntity("template", "caseTemplate"),
-        renameAttribute("caseTemplate", "metricNames", "metrics"),
-        mapEntity("audit")(auditDetailsCleanup),
-        mapEntity("audit")(addAuditRequestId))
-
-    case previousState @ DatabaseState(6) ⇒
-      Seq(
-        mapEntity(_ ⇒ true, e ⇒ {
-          val createdAt = (e \ "startDate")
-            .asOpt[JsString]
-            .getOrElse(Json.toJson(new Date))
-          val createdBy = (e \ "user")
-            .asOpt[JsString]
-            .getOrElse(JsString("init"))
-          e +
-            ("createdAt" → createdAt) +
-            ("createdBy" → createdBy)
-        }))
+        mapAttribute(Seq("case", "case_task", "case_task_log", "case_artifact", "audit", "case_artifact_job"), "startDate")(convertDate),
+        mapAttribute(Seq("case", "case_task", "case_artifact_job"), "endDate")(convertDate),
+        mapAttribute("misp", "date")(convertDate),
+        mapAttribute("misp", "publishDate")(convertDate),
+        mapAttribute(_ ⇒ true, "createdBy", convertDate),
+        mapAttribute(_ ⇒ true, "updatedBy", convertDate))
   }
 
   private val requestCounter = new java.util.concurrent.atomic.AtomicInteger(0)
   def getRequestId = {
     utils.Instance.id + ":mig:" + requestCounter.incrementAndGet()
   }
 
+  def convertDate(json: JsValue): JsValue = {
+    val datePattern = "yyyyMMdd'T'HHmmssZ"
+    val dateReads = Reads.dateReads(datePattern).orElse(Reads.DefaultDateReads)
+    val date = dateReads.reads(json).getOrElse {
+      logger.warn(s"""Invalid date format : "$json" setting now""")
+      new Date
+    }
+    org.elastic4play.JsonFormat.dateWrites.writes(date)
+  }
+
   def removeDot[A <: JsValue](json: A): A = json match {
     case obj: JsObject ⇒
       obj.map {

thehive-backend/app/models/ReportTemplate.scala

@@ -0,0 +1,17 @@
+package models
+
+import javax.inject.{ Inject, Singleton }
+
+import play.api.libs.json.JsObject
+
+import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, EntityDef, ModelDef }
+
+trait ReportTemplateAttributes { _: AttributeDef ⇒
+  val content = attribute("content", F.textFmt, "Content of the template")
+  val reportType = attribute("reportType", F.stringFmt, "Flavor of the report (short or long)")
+  val analyzerId = multiAttribute("analyzerId", F.stringFmt, "Id of analyzers")
+}
+
+@Singleton
+class ReportTemplateModel @Inject() extends ModelDef[ReportTemplateModel, ReportTemplate]("reportTemplate") with ReportTemplateAttributes
+class ReportTemplate(model: ReportTemplateModel, attributes: JsObject) extends EntityDef[ReportTemplateModel, ReportTemplate](model, attributes) with ReportTemplateAttributes

thehive-backend/app/models/Task.scala

@@ -29,6 +29,7 @@ trait TaskAttributes { _: AttributeDef ⇒
   val flag = attribute("flag", F.booleanFmt, "Flag of the task", false)
   val startDate = optionalAttribute("startDate", F.dateFmt, "Timestamp of the comment start")
   val endDate = optionalAttribute("endDate", F.dateFmt, "Timestamp of the comment end")
+  val order = attribute("order", F.numberFmt, "Order of the task", 0L)
 
 }
 @Singleton

thehive-backend/app/models/package.scala

@@ -1,5 +1,5 @@
 
 
 package object models {
-  val version = 7
+  val version = 8
 }

thehive-backend/app/services/CaseMergeSrv.scala

@@ -232,7 +232,6 @@ class CaseMergeSrv @Inject() (
       .set("tlp", JsNumber(cases.map(_.tlp()).max))
       .set("status", JsString(CaseStatus.Open.toString))
       .set("metrics", mergeMetrics(cases))
-      .set("isIncident", JsBoolean(cases.map(_.isIncident()).reduce(_ || _)))
       .set("resolutionStatus", mergeResolutionStatus(cases))
       .set("impactStatus", mergeImpactStatus(cases))
       .set("summary", mergeSummary(cases))

thehive-backend/app/services/CaseSrv.scala

@@ -36,7 +36,11 @@ class CaseSrv @Inject() (
   lazy val log = Logger(getClass)
 
   def create(fields: Fields)(implicit authContext: AuthContext): Future[Case] = {
-    createSrv[CaseModel, Case](caseModel, fields.unset("tasks"))
+    val fieldsWithOwner = fields.get("owner") match {
+      case None    ⇒ fields.set("owner", authContext.userId)
+      case Some(_) ⇒ fields
+    }
+    createSrv[CaseModel, Case](caseModel, fieldsWithOwner.unset("tasks"))
       .flatMap { caze ⇒
         val taskFields = fields.getValues("tasks").collect {
           case task: JsObject ⇒ Fields(task)

ui/app/scripts/directives/updatableDate.js

@@ -27,13 +27,12 @@
                         weekStart: 1,
                         startView: 1,
                         todayBtn: true,
-                        language: 'fr',
                         autoclose: true
                     });
                     scope.dateNow = false;
                     scope.timeUpdater = undefined;
-                    if (angular.isString(scope.value)) {
-                        var m = moment(scope.value, 'YYYYMMDDTHHmmssZZ');
+                    if (angular.isNumber(scope.value)) {
+                        var m = moment(scope.value);
                         if (m.isValid()) {
                             scope.humanDate = m.format('DD-MM-YYYY HH:mm');
                         }
@@ -45,7 +44,7 @@
                         if (angular.isString(scope.humanDate)) {
                             var m = moment(scope.humanDate, 'DD-MM-YYYY HH:mm');
                             if (m.isValid()) {
-                                scope.value = m.format('YYYYMMDDTHHmmssZZ');
+                                scope.value = m.valueOf();
                             }
                         }
                     });

ui/app/scripts/filters/shortDate.js

@@ -2,8 +2,11 @@
     'use strict';
     angular.module('theHiveFilters').filter('shortDate', function() {
         return function(str) {
+            var format = ' MM/DD/YY H:mm';
             if (angular.isString(str) && str.length > 0) {
-                return moment(str, ['YYYYMMDDTHHmmZZ', 'DD-MM-YYYY HH:mm']).format(' MM/DD/YY H:mm');
+                return moment(str, ['YYYYMMDDTHHmmZZ', 'DD-MM-YYYY HH:mm']).format(format);
+            } else if (angular.isNumber(str)) {
+                return moment(str).format(format);
             } else {
                 return '';
             }