#52 Support bidirectional synchronization with MISP

thehive-misp/app/connectors/misp/JsonFormat.scala

@@ -6,6 +6,8 @@ import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup
 import play.api.libs.json.JsValue.jsValueToJsLookup
 import play.api.libs.json._
 
+import org.elastic4play.services.JsonFormat.attachmentFormat
+
 object JsonFormat {
 
   implicit val mispAlertReads: Reads[MispAlert] = Reads[MispAlert] { json ⇒
@@ -64,7 +66,7 @@ object JsonFormat {
       date,
       comment,
       value,
-      tags :+ s"MISP:category$category" :+ s"MISP:type=$tpe"))
+      tags))
 
   implicit val exportedAttributeWrites: Writes[ExportedMispAttribute] = Writes[ExportedMispAttribute] { attribute ⇒
     Json.obj(
@@ -73,4 +75,17 @@ object JsonFormat {
       "value" → attribute.value.fold[String](identity, _.name),
       "comment" → attribute.comment)
   }
-}
+
+  implicit val mispArtifactWrites: Writes[MispArtifact] = OWrites[MispArtifact] { artifact ⇒
+    Json.obj(
+      "dataType" → artifact.dataType,
+      "message" → artifact.message,
+      "tlp" → artifact.tlp,
+      "tags" → artifact.tags,
+      "startDate" → artifact.startDate) + (artifact.value match {
+        case SimpleArtifactData(data)                           ⇒ "data" → JsString(data)
+        case RemoteAttachmentArtifact(filename, reference, tpe) ⇒ "remoteAttachment" → Json.obj("filename" → filename, "reference" → reference, "type" → tpe)
+        case AttachmentArtifact(attachment)                     ⇒ "attachment" → Json.toJson(attachment)
+      })
+  }
+}

thehive-misp/app/connectors/misp/MispConverter.scala

@@ -1,37 +1,49 @@
 package connectors.misp
 
-import play.api.libs.json.{ JsObject, JsString, Json }
-
 trait MispConverter {
-  def convertAttribute(mispAttribute: MispAttribute): Seq[JsObject] = {
-    val dataType = toArtifact(mispAttribute.tpe)
+  def convertAttribute(mispAttribute: MispAttribute): Seq[MispArtifact] = {
     val tags = Seq(s"MISP:type=${mispAttribute.tpe}", s"MISP:category=${mispAttribute.category}")
-    val fields = Json.obj(
-      "data" → mispAttribute.value,
-      "dataType" → dataType,
-      "message" → mispAttribute.comment,
-      "startDate" → mispAttribute.date,
-      "tags" → tags)
+    if (mispAttribute.tpe == "attachment" || mispAttribute.tpe == "malware-sample") {
+      Seq(
+        MispArtifact(
+          value     = RemoteAttachmentArtifact(mispAttribute.value.split("\\|").head, mispAttribute.id, mispAttribute.tpe),
+          dataType  = "file",
+          message   = mispAttribute.comment,
+          tlp       = 0,
+          tags      = tags ++ mispAttribute.tags,
+          startDate = mispAttribute.date))
+    }
+    else {
+      val dataType = toArtifact(mispAttribute.tpe)
+      val artifact =
+        MispArtifact(
+          value     = SimpleArtifactData(mispAttribute.value),
+          dataType  = dataType,
+          message   = mispAttribute.comment,
+          tlp       = 0,
+          tags      = tags ++ mispAttribute.tags,
+          startDate = mispAttribute.date)
 
-    val types = mispAttribute.tpe.split('|').toSeq
-    if (types.length > 1) {
-      val values = mispAttribute.value.split('|').toSeq
-      val typesValues = types.zipAll(values, "noType", "noValue")
-      val additionnalMessage = typesValues
-        .map {
-          case (t, v) ⇒ s"$t: $v"
+      val types = mispAttribute.tpe.split('|').toSeq
+      if (types.length > 1) {
+        val values = mispAttribute.value.split('|').toSeq
+        val typesValues = types.zipAll(values, "noType", "noValue")
+        val additionnalMessage = typesValues
+          .map {
+            case (t, v) ⇒ s"$t: $v"
+          }
+          .mkString("\n")
+        typesValues.map {
+          case (tpe, value) ⇒
+            artifact.copy(
+              dataType = toArtifact(tpe),
+              value    = SimpleArtifactData(value),
+              message  = mispAttribute.comment + "\n" + additionnalMessage)
         }
-        .mkString("\n")
-      typesValues.map {
-        case (tpe, value) ⇒
-          fields +
-            ("dataType" → JsString(toArtifact(tpe))) +
-            ("data" → JsString(value)) +
-            ("message" → JsString(mispAttribute.comment + "\n" + additionnalMessage))
       }
-    }
-    else {
-      Seq(fields)
+      else {
+        Seq(artifact)
+      }
     }
   }
 

thehive-misp/app/connectors/misp/MispExport.scala

@@ -18,7 +18,8 @@ import akka.stream.Materializer
 import org.elastic4play.InternalError
 import org.elastic4play.controllers.Fields
 import org.elastic4play.models.JsonFormat.baseModelEntityWrites
-import org.elastic4play.services.{ AttachmentSrv, AuthContext }
+import org.elastic4play.services.{ Attachment, AttachmentSrv, AuthContext }
+import org.elastic4play.services.JsonFormat.attachmentFormat
 import org.elastic4play.utils.RichFuture
 
 @Singleton
@@ -43,23 +44,6 @@ class MispExport @Inject() (
       .map(alertIdSource ⇒ alertIdSource.map(_._1) → alertIdSource.map(_._2))
   }
 
-  def buildAttributeList(caze: Case): Future[Seq[ExportedMispAttribute]] = {
-    import org.elastic4play.services.QueryDSL._
-    artifactSrv
-      .find(parent("case", withId(caze.id)), Some("all"), Nil)
-      ._1
-      .map { artifact ⇒
-        val (category, tpe) = fromArtifact(artifact.dataType(), artifact.data())
-        val value = (artifact.data(), artifact.attachment()) match {
-          case (Some(data), None)       ⇒ Left(data)
-          case (None, Some(attachment)) ⇒ Right(attachment)
-          case _                        ⇒ sys.error("???")
-        }
-        ExportedMispAttribute(artifact, tpe, category, value, artifact.message())
-      }
-      .runWith(Sink.seq)
-  }
-
   def removeDuplicateAttributes(attributes: Seq[ExportedMispAttribute]): Seq[ExportedMispAttribute] = {
     val attrIndex = attributes.zipWithIndex
 
@@ -147,39 +131,64 @@ class MispExport @Inject() (
 
     for {
       (maybeAlertId, maybeEventId) ← relatedMispEvent(mispName, caze.id)
-      attributes ← buildAttributeList(caze)
+      _ = println(s"[0] $maybeAlertId $maybeEventId")
+      attributes ← mispSrv.getAttributesFromCase(caze)
+      _ = println(s"[1] $attributes")
       uniqueAttributes = removeDuplicateAttributes(attributes)
-      simpleAttributes = uniqueAttributes.filter(_.value.isLeft) // FIXME used only if event doesn't exist
+      _ = println(s"[2] $uniqueAttributes")
       (eventId, existingAttributes) ← maybeEventId.fold {
+        val simpleAttributes = uniqueAttributes.filter(_.value.isLeft)
+        println(s"[3] $simpleAttributes")
         // if no event is associated to this case, create a new one
         createEvent(mispConnection, caze.title(), caze.severity(), caze.startDate(), simpleAttributes).map {
-          case (eventId, exportedAttributes) ⇒ eventId → exportedAttributes.map(_.value.left.get)
+          case (eventId, exportedAttributes) ⇒ eventId → exportedAttributes.map(_.value.map(_.name))
         }
       } { eventId ⇒ // if an event already exists, retrieve its attributes in order to export only new one
-        mispSrv.getAttributes(mispConnection, eventId, None).map { attributes ⇒
-          eventId → attributes.map { attribute ⇒
-            (attribute \ "data").asOpt[String].getOrElse((attribute \ "remoteAttachment" \ "filename").as[String])
+        mispSrv.getAttributesFromMisp(mispConnection, eventId, None).map { attributes ⇒
+          eventId → attributes.map {
+            case MispArtifact(SimpleArtifactData(data), _, _, _, _, _)                             ⇒ Left(data)
+            case MispArtifact(RemoteAttachmentArtifact(filename, _, _), _, _, _, _, _)             ⇒ Right(filename)
+            case MispArtifact(AttachmentArtifact(Attachment(filename, _, _, _, _)), _, _, _, _, _) ⇒ Right(filename)
           }
         }
       }
-      newAttributes = uniqueAttributes.filterNot(attr ⇒ existingAttributes.contains(attr.value.fold(identity, _.name)))
+      _ = println(s"[4] $existingAttributes")
+      newAttributes = uniqueAttributes.filterNot(attr ⇒ existingAttributes.contains(attr.value.map(_.name)))
+      _ = println(s"[5] $newAttributes")
       exportedArtifact ← Future.traverse(newAttributes)(attr ⇒ exportAttribute(mispConnection, eventId, attr).toTry)
-      alertFields = Fields(Json.obj(
-        "type" → "misp",
-        "source" → mispName,
-        "sourceRef" → eventId,
-        "date" → caze.startDate(),
-        "lastSyncDate" → new Date(0),
-        "case" → caze.id,
-        "title" → caze.title(),
-        "description" → "Case have been exported to MISP",
-        "severity" → caze.severity(),
-        "tags" → caze.tags(),
-        "tlp" → caze.tlp(),
-        "artifacts" → uniqueAttributes.map(_.artifact),
-        "status" → "Imported",
-        "follow" → false))
-      alert ← maybeAlertId.fold(alertSrv.create(alertFields))(alertId ⇒ alertSrv.update(alertId, alertFields))
+      _ = println(s"[6] $exportedArtifact")
+      alert ← maybeAlertId.fold {
+        alertSrv.create(Fields(Json.obj(
+          "type" → "misp",
+          "source" → mispName,
+          "sourceRef" → eventId,
+          "date" → caze.startDate(),
+          "lastSyncDate" → new Date(0),
+          "case" → caze.id,
+          "title" → caze.title(),
+          "description" → "Case have been exported to MISP",
+          "severity" → caze.severity(),
+          "tags" → caze.tags(),
+          "tlp" → caze.tlp(),
+          "artifacts" → uniqueAttributes.map(_.artifact),
+          "status" → "Imported",
+          "follow" → false)))
+      } { alertId ⇒
+        val artifacts = uniqueAttributes.map { exportedArtifact ⇒
+          Json.obj(
+            "data" → exportedArtifact.artifact.data(),
+            "dataType" → exportedArtifact.artifact.dataType(),
+            "message" → exportedArtifact.artifact.message(),
+            "startDate" → exportedArtifact.artifact.startDate(),
+            "attachment" → exportedArtifact.artifact.attachment(),
+            "tlp" → exportedArtifact.artifact.tlp(),
+            "tags" → exportedArtifact.artifact.tags(),
+            "ioc" → exportedArtifact.artifact.ioc())
+        }
+        alertSrv.update(alertId, Fields(Json.obj(
+          "artifacts" → artifacts,
+          "status" → "Imported")))
+      }
     } yield alert.id → exportedArtifact
   }
 }

thehive-misp/app/connectors/misp/MispModel.scala

@@ -5,6 +5,18 @@ import java.util.Date
 import models.Artifact
 
 import org.elastic4play.services.Attachment
+import org.elastic4play.utils.Hash
+
+sealed trait ArtifactData
+case class SimpleArtifactData(data: String) extends ArtifactData
+case class AttachmentArtifact(attachment: Attachment) extends ArtifactData {
+  def name: String = attachment.name
+  def hashes: Seq[Hash] = attachment.hashes
+  def size: Long = attachment.size
+  def contentType: String = attachment.contentType
+  def id: String = attachment.id
+}
+case class RemoteAttachmentArtifact(filename: String, reference: String, tpe: String) extends ArtifactData
 
 case class MispAlert(
   source: String,
@@ -35,4 +47,13 @@ case class ExportedMispAttribute(
   value: Either[String, Attachment],
   comment: Option[String])
 
-case class MispExportError(message: String, artifact: Artifact) extends Exception(message)
+case class MispArtifact(
+  value: ArtifactData,
+  dataType: String,
+  message: String,
+  tlp: Long,
+  tags: Seq[String],
+  startDate: Date)
+
+case class MispExportError(message: String, artifact: Artifact) extends Exception(message)
+

thehive-misp/app/connectors/misp/MispSrv.scala

@@ -23,8 +23,7 @@ import net.lingala.zip4j.model.FileHeader
 import services._
 
 import org.elastic4play.controllers.{ Fields, FileInputValue }
-import org.elastic4play.services.{ AuthContext, TempSrv }
-import org.elastic4play.utils.RichJson
+import org.elastic4play.services.{ Attachment, AuthContext, TempSrv }
 import org.elastic4play.{ InternalError, NotFoundError }
 
 @Singleton
@@ -74,14 +73,35 @@ class MispSrv @Inject() (
         val eventsSize = events.size
         if (eventJsonSize != eventsSize)
           logger.warn(s"MISP returns $eventJsonSize events but only $eventsSize contain valid data")
-        events.toList
+        events.filter(_.lastSyncDate after fromDate).toList
       }
   }
 
-  def getAttributes(
+  def getAttributesFromCase(caze: Case): Future[Seq[ExportedMispAttribute]] = {
+    import org.elastic4play.services.QueryDSL._
+    val (artifacts, totalArtifacts) = artifactSrv
+      .find(and(withParent(caze), "status" ~= "Ok"), Some("all"), Nil)
+    totalArtifacts.foreach(t ⇒ println(s"Case ${caze.id} has $t artifact(s)"))
+    artifacts
+      .map { artifact ⇒
+        println(s"[-] $artifact")
+        val (category, tpe) = fromArtifact(artifact.dataType(), artifact.data())
+        val value = (artifact.data(), artifact.attachment()) match {
+          case (Some(data), None)       ⇒ Left(data)
+          case (None, Some(attachment)) ⇒ Right(attachment)
+          case _ ⇒
+            logger.error(s"Artifact $artifact has neither data nor attachment")
+            sys.error("???")
+        }
+        ExportedMispAttribute(artifact, tpe, category, value, artifact.message())
+      }
+      .runWith(Sink.seq)
+  }
+
+  def getAttributesFromMisp(
     mispConnection: MispConnection,
     eventId: String,
-    fromDate: Option[Date]): Future[Seq[JsObject]] = {
+    fromDate: Option[Date]): Future[Seq[MispArtifact]] = {
 
     val date = fromDate.fold(0L)(_.getTime / 1000)
 
@@ -94,27 +114,23 @@ class MispSrv @Inject() (
       // add ("deleted" → "only") to see only deleted attributes
       .map { response ⇒
         val refDate = fromDate.getOrElse(new Date(0))
-        val artifactTags = JsString(s"src:${mispConnection.name}") +: JsArray(mispConnection.artifactTags.map(JsString))
+        val artifactTags = s"src:${mispConnection.name}" +: mispConnection.artifactTags
         (Json.parse(response.body) \ "response" \\ "Attribute")
           .flatMap(_.as[Seq[MispAttribute]])
           .filter(_.date after refDate)
-          .flatMap {
-            case a if a.tpe == "attachment" || a.tpe == "malware-sample" ⇒
-              Seq(
-                Json.obj(
-                  "dataType" → "file",
-                  "message" → a.comment,
-                  "tags" → (artifactTags.value ++ a.tags.map(JsString)),
-                  "remoteAttachment" → Json.obj(
-                    "filename" → a.value,
-                    "reference" → a.id,
-                    "type" → a.tpe),
-                  "startDate" → a.date))
-            case a ⇒ convertAttribute(a).map { j ⇒
-              val tags = artifactTags ++ (j \ "tags").asOpt[JsArray].getOrElse(JsArray(Nil))
-              j.setIfAbsent("tlp", 2L) + ("tags" → tags)
-            }
+          .flatMap(convertAttribute)
+          .groupBy {
+            case MispArtifact(SimpleArtifactData(data), dataType, _, _, _, _)                             ⇒ dataType → Right(data)
+            case MispArtifact(RemoteAttachmentArtifact(filename, _, _), dataType, _, _, _, _)             ⇒ dataType → Left(filename)
+            case MispArtifact(AttachmentArtifact(Attachment(filename, _, _, _, _)), dataType, _, _, _, _) ⇒ dataType → Left(filename)
+          }
+          .values
+          .map { mispArtifact ⇒
+            mispArtifact.head.copy(
+              tags = (mispArtifact.head.tags ++ artifactTags).distinct,
+              tlp  = 2L)
           }
+          .toSeq
       }
   }
 
@@ -207,7 +223,7 @@ class MispSrv @Inject() (
       else {
         getInstanceConfig(alert.source())
           .flatMap { mcfg ⇒
-            getAttributes(mcfg, alert.sourceRef(), None)
+            getAttributesFromMisp(mcfg, alert.sourceRef(), None)
           }
           .map(alert → _)
           .recover {
@@ -224,7 +240,7 @@ class MispSrv @Inject() (
       .mapAsyncUnordered(5) {
         case (alert, artifacts) ⇒
           logger.info(s"Updating alert ${alert.id}")
-          alertSrv.update(alert.id, Fields.empty.set("artifacts", JsArray(artifacts)))
+          alertSrv.update(alert.id, Fields.empty.set("artifacts", Json.toJson(artifacts)))
             .recover {
               case t ⇒ logger.error(s"Update alert ${alert.id} fail", t)
             }
@@ -276,10 +292,10 @@ class MispSrv @Inject() (
     }
   }
 
+  private[MispSrv] val fileNameExtractor = """attachment; filename="(.*)"""".r
   def downloadAttachment(
     mispConnection: MispConnection,
     attachmentId: String)(implicit authContext: AuthContext): Future[FileInputValue] = {
-    val fileNameExtractor = """attachment; filename="(.*)"""".r
 
     mispConnection(s"attributes/download/$attachmentId")
       .withMethod("GET")