#170 Fix MISP synchronization

TheHive-Project · Apr 12, 2017 · 9b7ce39 · 9b7ce39
1 parent 4966c1c
commit 9b7ce39
Show file tree

Hide file tree

Showing 9 changed files with 123 additions and 91 deletions.
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -29,7 +29,7 @@ object Dependencies {
     val reflections = "org.reflections" % "reflections" % "0.9.10"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
     val akkaTest = "com.typesafe.akka" %% "akka-stream-testkit" % "2.4.4"
-    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.1.3"
+    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.1.4-SNAPSHOT"
 
     object Elastic4s {
       private val version = "2.3.0"

diff --git a/thehive-backend/app/controllers/Alert.scala b/thehive-backend/app/controllers/Alert.scala
@@ -1,20 +1,20 @@
 package controllers
 
-import javax.inject.{ Inject, Singleton }
+import javax.inject.{Inject, Singleton}
 
 import akka.stream.Materializer
-import org.elastic4play.controllers.{ Authenticated, FieldsBodyParser, Renderer }
+import org.elastic4play.controllers.{Authenticated, FieldsBodyParser, Renderer}
 import org.elastic4play.models.JsonFormat.baseModelEntityWrites
-import org.elastic4play.services.JsonFormat.{ aggReads, queryReads }
+import org.elastic4play.services.JsonFormat.{aggReads, queryReads}
 import org.elastic4play.services._
-import org.elastic4play.{ BadRequestError, Timed }
+import org.elastic4play.{BadRequestError, Timed}
 import play.api.Logger
 import play.api.http.Status
 import play.api.libs.json.JsArray
 import play.api.mvc.Controller
 import services.AlertSrv
 
-import scala.concurrent.{ ExecutionContext, Future }
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
 
 @Singleton
@@ -88,4 +88,38 @@ class AlertCtrl @Inject() (
       .getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]]
     alertSrv.stats(query, aggs).map(s ⇒ Ok(s))
   }
+
+  @Timed
+  def markAsRead(id: String) = authenticated(Role.write).async { implicit request ⇒
+    for {
+      alert ← alertSrv.get(id)
+      updatedAlert ← alertSrv.markAsRead(alert)
+    } yield renderer.toOutput(OK, updatedAlert)
+  }
+
+  @Timed
+  def markAsUnread(id: String) = authenticated(Role.write).async { implicit request ⇒
+    for {
+      alert ← alertSrv.get(id)
+      updatedAlert ← alertSrv.markAsUnread(alert)
+    } yield renderer.toOutput(OK, updatedAlert)
+  }
+
+  def createCase(id: String) = authenticated(Role.write).async { implicit request ⇒
+    for {
+      alert ← alertSrv.get(id)
+      updatedAlert ← alertSrv.createCase(alert)
+    } yield renderer.toOutput(CREATED, updatedAlert)
+  }
+
+  @Timed
+  def followAlert(id: String) = authenticated(Role.write).async { implicit request ⇒
+    alertSrv.setFollowAlert(id, follow = true)
+      .map { alert ⇒ renderer.toOutput(OK, alert) }
+  }
+
+  def unfollowAlert(id: String) = authenticated(Role.write).async { implicit request ⇒
+    alertSrv.setFollowAlert(id, follow = false)
+      .map { alert ⇒ renderer.toOutput(OK, alert) }
+  }
 }
diff --git a/thehive-backend/app/models/Alert.scala b/thehive-backend/app/models/Alert.scala
@@ -1,12 +1,12 @@
 package models
 
-import javax.inject.{ Inject, Singleton }
+import javax.inject.{Inject, Singleton}
 
 import models.JsonFormat.alertStatusFormat
-import org.elastic4play.models.{ Attribute, AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef }
-import org.elastic4play.models.{ AttributeFormat ⇒ F, AttributeOption ⇒ O }
+import org.elastic4play.models.{Attribute, AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, AttributeFormat => F, AttributeOption => O}
+import org.elastic4play.utils.Hasher
 import play.api.Logger
-import play.api.libs.json.{ JsObject, JsString, Json }
+import play.api.libs.json.{JsObject, JsString, Json}
 import services.AuditedModel
 
 import scala.concurrent.Future
@@ -55,10 +55,12 @@ class AlertModel @Inject() (artifactModel: ArtifactModel)
       if (attrs.keys.contains("_id"))
         attrs
       else {
+        val hasher = Hasher("MD5")
         val tpe = (attrs \ "tpe").asOpt[String].getOrElse("<null>")
         val source = (attrs \ "source").asOpt[String].getOrElse("<null>")
         val sourceRef = (attrs \ "sourceRef").asOpt[String].getOrElse("<null>")
-        attrs + ("_id" → JsString(s"$tpe|$source|$sourceRef"))
+        val _id = hasher.fromString(s"$tpe|$source|$sourceRef").head.toString()
+        attrs + ("_id" → JsString(_id))
       }
     }
   }

diff --git a/thehive-backend/app/models/Migration.scala b/thehive-backend/app/models/Migration.scala
@@ -7,7 +7,7 @@ import akka.stream.Materializer
 import org.elastic4play.models.BaseModelDef
 import org.elastic4play.services._
 import org.elastic4play.utils
-import org.elastic4play.utils.RichJson
+import org.elastic4play.utils.{ Hasher, RichJson }
 import play.api.{ Configuration, Logger }
 import play.api.libs.json.JsValue.jsValueToJsLookup
 import play.api.libs.json._
@@ -87,6 +87,7 @@ class Migration(
         mapAttribute(_ ⇒ true, "updatedAt", convertDate))
     case DatabaseState(8) ⇒
       requireUpdateMispAlertArtifact = true
+      val hasher = Hasher("MD5")
       Seq(
         renameEntity("misp", "alert"),
         mapEntity("alert") { misp ⇒
@@ -103,12 +104,14 @@ class Migration(
               case "tlp:red"   ⇒ 3L
             }
             .getOrElse(2L)
+          val source = (misp \ "serverId").asOpt[String].getOrElse("<null>")
+          val _id = hasher.fromString(s"misp|$source|$eventId").head.toString()
           (misp \ "caze").asOpt[JsString].fold(JsObject(Nil))(c ⇒ Json.obj("caze" → c)) ++
             Json.obj(
               "_type" → "alert",
-              "_id" → ("misp:" + (misp \ "_id").as[String]),
+              "_id" → _id,
               "type" → "misp",
-              "source" → (misp \ "serverId").as[JsString],
+              "source" → source,
               "sourceRef" → eventId,
               "date" → date,
               "lastSyncDate" → (misp \ "publishDate").as[Date],

diff --git a/thehive-backend/app/services/AlertSrv.scala b/thehive-backend/app/services/AlertSrv.scala
@@ -4,50 +4,50 @@ import javax.inject.Inject
 
 import akka.NotUsed
 import akka.stream.Materializer
-import akka.stream.scaladsl.{ Sink, Source }
+import akka.stream.scaladsl.{Sink, Source}
 import connectors.ConnectorRouter
 import models._
 import org.elastic4play.controllers.Fields
 import org.elastic4play.services._
 import play.api.Configuration
-import play.api.libs.json.{ JsNumber, JsObject, Json }
+import play.api.libs.json.{JsNumber, JsObject, Json}
 
-import scala.concurrent.{ ExecutionContext, Future }
-import scala.util.{ Success, Try }
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Success, Try}
 
 trait AlertTransformer {
   def createCase(alert: Alert)(implicit authContext: AuthContext): Future[Case]
 }
 
 class AlertSrv(
-    templates: Map[String, String],
-    alertModel: AlertModel,
-    createSrv: CreateSrv,
-    getSrv: GetSrv,
-    updateSrv: UpdateSrv,
-    deleteSrv: DeleteSrv,
-    findSrv: FindSrv,
-    caseSrv: CaseSrv,
-    artifactSrv: ArtifactSrv,
-    caseTemplateSrv: CaseTemplateSrv,
-    connectors: ConnectorRouter,
-    implicit val ec: ExecutionContext,
-    implicit val mat: Materializer) extends AlertTransformer {
+                templates: Map[String, String],
+                alertModel: AlertModel,
+                createSrv: CreateSrv,
+                getSrv: GetSrv,
+                updateSrv: UpdateSrv,
+                deleteSrv: DeleteSrv,
+                findSrv: FindSrv,
+                caseSrv: CaseSrv,
+                artifactSrv: ArtifactSrv,
+                caseTemplateSrv: CaseTemplateSrv,
+                connectors: ConnectorRouter,
+                implicit val ec: ExecutionContext,
+                implicit val mat: Materializer) extends AlertTransformer {
 
   @Inject() def this(
-    configuration: Configuration,
-    alertModel: AlertModel,
-    createSrv: CreateSrv,
-    getSrv: GetSrv,
-    updateSrv: UpdateSrv,
-    deleteSrv: DeleteSrv,
-    findSrv: FindSrv,
-    caseSrv: CaseSrv,
-    artifactSrv: ArtifactSrv,
-    caseTemplateSrv: CaseTemplateSrv,
-    connectors: ConnectorRouter,
-    ec: ExecutionContext,
-    mat: Materializer) = this(
+                      configuration: Configuration,
+                      alertModel: AlertModel,
+                      createSrv: CreateSrv,
+                      getSrv: GetSrv,
+                      updateSrv: UpdateSrv,
+                      deleteSrv: DeleteSrv,
+                      findSrv: FindSrv,
+                      caseSrv: CaseSrv,
+                      artifactSrv: ArtifactSrv,
+                      caseTemplateSrv: CaseTemplateSrv,
+                      connectors: ConnectorRouter,
+                      ec: ExecutionContext,
+                      mat: Materializer) = this(
     Map.empty[String, String],
     alertModel: AlertModel,
     createSrv,
@@ -94,14 +94,14 @@ class AlertSrv(
   def markAsRead(alert: Alert)(implicit authContext: AuthContext): Future[Alert] = {
     alert.caze() match {
       case Some(_) ⇒ updateSrv[AlertModel, Alert](alertModel, alert.id, Fields.empty.set("status", "Imported"))
-      case None    ⇒ updateSrv[AlertModel, Alert](alertModel, alert.id, Fields.empty.set("status", "Ignore"))
+      case None ⇒ updateSrv[AlertModel, Alert](alertModel, alert.id, Fields.empty.set("status", "Ignore"))
     }
   }
 
   def markAsUnread(alert: Alert)(implicit authContext: AuthContext): Future[Alert] = {
     alert.caze() match {
       case Some(_) ⇒ updateSrv[AlertModel, Alert](alertModel, alert.id, Fields.empty.set("status", "Update"))
-      case None    ⇒ updateSrv[AlertModel, Alert](alertModel, alert.id, Fields.empty.set("status", "New"))
+      case None ⇒ updateSrv[AlertModel, Alert](alertModel, alert.id, Fields.empty.set("status", "New"))
     }
   }
 
@@ -153,10 +153,6 @@ class AlertSrv(
 
   def stats(queryDef: QueryDef, aggs: Seq[Agg]): Future[JsObject] = findSrv(alertModel, queryDef, aggs: _*)
 
-  def ignoreAlert(alertId: String)(implicit authContext: AuthContext): Future[Alert] = {
-    updateSrv[AlertModel, Alert](alertModel, alertId, Fields(Json.obj("status" → AlertStatus.Ignore)))
-  }
-
   def setFollowAlert(alertId: String, follow: Boolean)(implicit authContext: AuthContext): Future[Alert] = {
     updateSrv[AlertModel, Alert](alertModel, alertId, Fields(Json.obj("follow" → follow)))
   }

diff --git a/thehive-backend/conf/routes b/thehive-backend/conf/routes
@@ -58,6 +58,11 @@ POST     /api/alert                               controllers.AlertCtrl.create()
 GET      /api/alert/:alertId                      controllers.AlertCtrl.get(alertId)
 PATCH    /api/alert/:alertId                      controllers.AlertCtrl.update(alertId)
 DELETE   /api/alert/:alertId                      controllers.AlertCtrl.delete(alertId)
+POST     /api/alert/:alertId/markAsRead           controllers.AlertCtrl.markAsRead(alertId)
+POST     /api/alert/:alertId/markAsUnread         controllers.AlertCtrl.markAsUnread(alertId)
+POST     /api/alert/:alertId/createCase           controllers.AlertCtrl.createCase(alertId)
+POST     /api/alert/:alertId/follow               controllers.AlertCtrl.followAlert(alertId)
+POST     /api/alert/:alertId/unfollow             controllers.AlertCtrl.unfollowAlert(alertId)
 
 GET      /api/flow                                controllers.FlowCtrl.flow(rootId: Option[String], count: Option[Int])
 

diff --git a/thehive-misp/app/connectors/misp/JsonFormat.scala b/thehive-misp/app/connectors/misp/JsonFormat.scala
@@ -11,7 +11,6 @@ object JsonFormat {
 
   implicit val mispAlertReads = Reads { json ⇒
     for {
-      uuid ← (json \ "uuid").validate[String]
       org ← (json \ "Orgc" \ "name").validate[String]
       info ← (json \ "info").validate[String]
       eventId ← (json \ "id").validate[String]
@@ -33,7 +32,6 @@ object JsonFormat {
       publishDate = new Date(publishTimestamp.toLong * 1000)
       threatLevel ← (json \ "threat_level_id").validate[String]
     } yield MispAlert(
-      uuid,
       org,
       eventId,
       date,
@@ -52,21 +50,15 @@ object JsonFormat {
     for {
       id ← (json \ "id").validate[String]
       tpe ← (json \ "type").validate[String]
-      category ← (json \ "category").validate[String]
-      uuid ← (json \ "uuid").validate[String]
-      eventId ← (json \ "id").validate[String]
       timestamp ← (json \ "timestamp").validate[String]
       date = new Date(timestamp.toLong * 1000)
-      comment ← (json \ "comment").validate[String]
+      comment = (json \ "comment").asOpt[String].getOrElse("")
       value ← (json \ "value").validate[String]
       category ← (json \ "category").validate[String]
       tags ← JsArray((json \ "EventTag" \\ "name")).validate[Seq[String]]
     } yield MispAttribute(
       id,
       tpe,
-      category,
-      uuid,
-      eventId.toLong,
       date,
       comment,
       value,

diff --git a/thehive-misp/app/connectors/misp/MispModel.scala b/thehive-misp/app/connectors/misp/MispModel.scala
@@ -3,7 +3,6 @@ package connectors.misp
 import java.util.Date
 
 case class MispAlert(
-  eventUuid: String,
   source: String,
   sourceRef: String,
   date: Date,
@@ -18,9 +17,6 @@ case class MispAlert(
 case class MispAttribute(
   id: String,
   tpe: String,
-  category: String,
-  uuid: String,
-  eventId: Long,
   date: Date,
   comment: String,
   value: String,