#1566 Export tags to MISP event

misp/client/src/main/scala/org/thp/misp/client/MispClient.scala

@@ -7,7 +7,7 @@ import akka.stream.alpakka.json.scaladsl.JsonReader
 import akka.stream.scaladsl.{JsonFraming, Source}
 import akka.util.ByteString
 import org.thp.client.{ApplicationError, Authentication, ProxyWS}
-import org.thp.misp.dto.{Attribute, Event, Organisation, User}
+import org.thp.misp.dto.{Attribute, Event, Organisation, Tag, User}
 import org.thp.scalligraph.InternalError
 import play.api.Logger
 import play.api.http.Status
@@ -45,10 +45,11 @@ class MispClient(
         Failure(InternalError(s"MISP server $name is inaccessible", t))
     }
 
-  private def configuredProxy: Option[String] = ws match {
-    case c: ProxyWS => c.proxy.map(p => s"http://${p.host}:${p.port}")
-    case _          => None
-  }
+  private def configuredProxy: Option[String] =
+    ws match {
+      case c: ProxyWS => c.proxy.map(p => s"http://${p.host}:${p.port}")
+      case _          => None
+    }
   logger.info(s"""Add MISP connection $name
                  |  url:              $baseUrl
                  |  proxy:            ${configuredProxy.getOrElse("<not set>")}
@@ -62,40 +63,73 @@ class MispClient(
   private def request(url: String): WSRequest =
     auth(ws.url(s"$strippedUrl/$url").withHttpHeaders("Accept" -> "application/json"))
 
-  private def get(url: String)(implicit ec: ExecutionContext): Future[JsValue] =
+  private def get(url: String)(implicit ec: ExecutionContext): Future[JsValue] = {
+    logger.trace(s"MISP request: GET $url")
     request(url).get().transform {
-      case Success(r) if r.status == Status.OK => Success(r.json)
-      case Success(r)                          => Try(r.json)
-      case Failure(t)                          => throw t
+      case Success(r) if r.status == Status.OK =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}")
+        Success(r.json)
+      case Success(r) =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}")
+        Try(r.json)
+      case Failure(t) =>
+        logger.trace(s"MISP error: $t")
+        throw t
     }
+  }
 
-  private def post(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[JsValue] =
+  private def post(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[JsValue] = {
+    logger.trace(s"MISP request: POST $url\n$body")
     request(url).post(body).transform {
-      case Success(r) if r.status == Status.OK => Success(r.json)
-      case Success(r)                          => Try(r.json)
-      case Failure(t)                          => throw t
+      case Success(r) if r.status == Status.OK =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}")
+        Success(r.json)
+      case Success(r) =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}")
+        Try(r.json)
+      case Failure(t) =>
+        logger.trace(s"MISP error: $t")
+        throw t
     }
+  }
 
-  private def post(url: String, body: Source[ByteString, _])(implicit ec: ExecutionContext): Future[JsValue] =
+  private def post(url: String, body: Source[ByteString, _])(implicit ec: ExecutionContext): Future[JsValue] = {
+    logger.trace(s"MISP request: POST $url (stream body)")
     request(url).post(body).transform {
-      case Success(r) if r.status == Status.OK => Success(r.json)
-      case Success(r)                          => Try(r.json)
-      case Failure(t)                          => throw t
+      case Success(r) if r.status == Status.OK =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}")
+        Success(r.json)
+      case Success(r) =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}")
+        Try(r.json)
+      case Failure(t) =>
+        logger.trace(s"MISP error: $t")
+        throw t
     }
-//
+  }
+
+  //
 //  private def getStream(url: String)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] =
 //    request(url).withMethod("GET").stream().transform {
 //      case Success(r) if r.status == Status.OK => Success(r.bodyAsSource)
 //      case Success(r)                          => Try(r.bodyAsSource)
 //      case Failure(t)                          => throw t
 //    }
 
-  private def postStream(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] =
+  private def postStream(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = {
+    logger.trace(s"MISP request: POST $url\n$body")
     request(url).withMethod("POST").withBody(body).stream().transform {
-      case Success(r) if r.status == Status.OK => Success(r.bodyAsSource)
-      case Success(r)                          => Try(r.bodyAsSource)
-      case Failure(t)                          => throw t
+      case Success(r) if r.status == Status.OK =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText} (stream body)")
+        Success(r.bodyAsSource)
+      case Success(r) =>
+        logger.trace(s"MISP response: ${r.status} ${r.statusText} (stream body)")
+        Try(r.bodyAsSource)
+      case Failure(t) =>
+        logger.trace(s"MISP error: $t")
+        throw t
     }
+  }
 
   def getCurrentUser(implicit ec: ExecutionContext): Future[User] = {
     logger.debug("Get current user")
@@ -177,7 +211,8 @@ class MispClient(
         maybeAttribute.fold(error => { logger.warn(s"Attribute has invalid format: ${data.decodeString("UTF-8")}", error); Nil }, List(_))
       }
       .mapAsyncUnordered(2) {
-        case attribute @ Attribute(id, "malware-sample" | "attachment", _, _, _, _, _, _, _, None, _, _, _, _) => // TODO need to unzip malware samples ?
+        case attribute @ Attribute(id, "malware-sample" | "attachment", _, _, _, _, _, _, _, None, _, _, _, _) =>
+          // TODO need to unzip malware samples ?
           downloadAttachment(id).map {
             case (filename, contentType, src) => attribute.copy(data = Some((filename, contentType, src)))
           }
@@ -204,14 +239,16 @@ class MispClient(
       case Failure(t) => throw t
     }
 
-  def uploadAttachment(eventId: String, comment: String, filename: String, data: Source[ByteString, _])(
-      implicit ec: ExecutionContext
+  def uploadAttachment(eventId: String, comment: String, filename: String, data: Source[ByteString, _])(implicit
+      ec: ExecutionContext
   ): Future[JsValue] = {
     val stream = data
       .via(Base64Flow.encode())
       .intersperse(
         ByteString(
-          s"""{"request":{"category":"Payload delivery","type":"malware-sample","comment":${JsString(comment).toString},"files":[{"filename":${JsString(
+          s"""{"request":{"category":"Payload delivery","type":"malware-sample","comment":${JsString(
+            comment
+          ).toString},"files":[{"filename":${JsString(
             filename
           ).toString},"data":""""
         ),
@@ -229,6 +266,7 @@ class MispClient(
       analysis: Int,
       distribution: Int,
       attributes: Seq[Attribute],
+      tags: Seq[Tag],
       extendsEvent: Option[String] = None
   )(implicit ec: ExecutionContext): Future[String] = {
     logger.debug(s"Create MISP event $info, with ${attributes.size} attributes")
@@ -243,6 +281,7 @@ class MispClient(
         "analysis"        -> analysis.toString,
         "distribution"    -> distribution,
         "Attribute"       -> stringAttributes,
+        "Tag"             -> tags,
         "extends_uuid"    -> extendsEvent
       )
     )

misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala

@@ -55,8 +55,8 @@ object Attribute {
       "category" -> attribute.category,
       "type"     -> attribute.`type`,
       "value"    -> attribute.value,
-      "comment"  -> attribute.comment
-//      "Tag"      -> attribute.tags
+      "comment"  -> attribute.comment,
+      "Tag"      -> attribute.tags
     )
   }
 }

misp/client/src/main/scala/org/thp/misp/dto/Tag.scala

@@ -20,8 +20,5 @@ object Tag {
       } and
       (JsPath \ "exportable").readNullable[Boolean])(Tag.apply _)
 
-  implicit val writes: Writes[Tag] = Writes[Tag] {
-    case Tag(Some(id), name, colour, _) => Json.obj("id" -> id, "name" -> name, "colour" -> colour.map(c => f"#$c%06X"))
-    case Tag(_, name, _, _)             => JsString(name)
-  }
+  implicit val writes: Writes[Tag] = Json.writes[Tag]
 }

misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala

@@ -31,7 +31,12 @@ class MispExportSrv @Inject() (
 
   lazy val logger: Logger = Logger(getClass)
 
-  def observableToAttribute(observable: RichObservable): Option[Attribute] =
+  def observableToAttribute(observable: RichObservable, exportTags: Boolean): Option[Attribute] = {
+    lazy val mispTags =
+      if (exportTags)
+        observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(observable.tlp)
+      else
+        tlpTags.get(observable.tlp).toSeq
     connector
       .attributeConverter(observable.`type`)
       .map {
@@ -50,7 +55,7 @@ class MispExportSrv @Inject() (
             value = observable.data.fold(observable.attachment.get.name)(_.data),
             firstSeen = None,
             lastSeen = None,
-            tags = observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None))
+            tags = mispTags
           )
       }
       .orElse {
@@ -59,6 +64,7 @@ class MispExportSrv @Inject() (
         )
         None
       }
+  }
 
   def getMispClient(mispId: String): Future[TheHiveMispClient] =
     connector
@@ -77,8 +83,8 @@ class MispExportSrv @Inject() (
       .filterByType("misp")
       .headOption
 
-  def getAttributes(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] =
-    caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute)
+  def getAttributes(`case`: Case with Entity, exportTags: Boolean)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] =
+    caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute(_, exportTags))
 
   def removeDuplicateAttributes(attributes: Iterator[Attribute]): Seq[Attribute] = {
     var attrSet = Set.empty[(String, String, String)]
@@ -93,9 +99,21 @@ class MispExportSrv @Inject() (
     builder.result()
   }
 
-  def createEvent(client: TheHiveMispClient, `case`: Case, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit
+  val tlpTags = Map(
+    0 -> MispTag(None, "tlp:white", None, None),
+    1 -> MispTag(None, "tlp:green", None, None),
+    2 -> MispTag(None, "tlp:amber", None, None),
+    3 -> MispTag(None, "tlp:red", None, None)
+  )
+  def createEvent(client: TheHiveMispClient, `case`: Case with Entity, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit
       ec: ExecutionContext
-  ): Future[String] =
+  ): Future[String] = {
+    val mispTags =
+      if (client.exportCaseTags)
+        db.roTransaction { implicit graph =>
+          caseSrv.get(`case`._id).tags.toSeq.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(`case`.tlp)
+        }
+      else tlpTags.get(`case`.tlp).toSeq
     client.createEvent(
       info = `case`.title,
       date = `case`.startDate,
@@ -104,8 +122,10 @@ class MispExportSrv @Inject() (
       analysis = 0,
       distribution = 0,
       attributes = attributes,
+      tags = mispTags,
       extendsEvent = extendsEvent
     )
+  }
 
   def createAlert(client: TheHiveMispClient, `case`: Case with Entity, eventId: String)(implicit
       graph: Graph,
@@ -147,7 +167,7 @@ class MispExportSrv @Inject() (
       orgName <- Future.fromTry(client.currentOrganisationName)
       maybeAlert = db.roTransaction(implicit graph => getAlert(`case`, orgName))
       _          = logger.debug(maybeAlert.fold("Related MISP event doesn't exist")(a => s"Related MISP event found : ${a.sourceRef}"))
-      attributes = db.roTransaction(implicit graph => removeDuplicateAttributes(getAttributes(`case`)))
+      attributes = db.roTransaction(implicit graph => removeDuplicateAttributes(getAttributes(`case`, client.exportObservableTags)))
       eventId <- createEvent(client, `case`, attributes, maybeAlert.map(_.sourceRef))
       _       <- Future.fromTry(db.tryTransaction(implicit graph => createAlert(client, `case`, eventId)))
     } yield eventId

misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala

@@ -29,6 +29,7 @@ case class TheHiveMispClientConfig(
     caseTemplate: Option[String],
     artifactTags: Seq[String] = Nil,
     exportCaseTags: Boolean = false,
+    exportObservableTags: Boolean = false,
     includedTheHiveOrganisations: Seq[String] = Seq("*"),
     excludedTheHiveOrganisations: Seq[String] = Nil
 )
@@ -49,6 +50,7 @@ object TheHiveMispClientConfig {
       caseTemplate                 <- (JsPath \ "caseTemplate").readNullable[String]
       artifactTags                 <- (JsPath \ "tags").readWithDefault[Seq[String]](Nil)
       exportCaseTags               <- (JsPath \ "exportCaseTags").readWithDefault[Boolean](false)
+      exportObservableTags         <- (JsPath \ "exportObservableTags").readWithDefault[Boolean](false)
       includedTheHiveOrganisations <- (JsPath \ "includedTheHiveOrganisations").readWithDefault[Seq[String]](Seq("*"))
       excludedTheHiveOrganisations <- (JsPath \ "excludedTheHiveOrganisations").readWithDefault[Seq[String]](Nil)
     } yield TheHiveMispClientConfig(
@@ -64,6 +66,7 @@ object TheHiveMispClientConfig {
       caseTemplate,
       artifactTags,
       exportCaseTags,
+      exportObservableTags,
       includedTheHiveOrganisations,
       excludedTheHiveOrganisations
     )
@@ -100,7 +103,8 @@ class TheHiveMispClient(
     purpose: MispPurpose.Value,
     val caseTemplate: Option[String],
     artifactTags: Seq[String], // FIXME use artifactTags
-    exportCaseTags: Boolean,   //  FIXME use exportCaseTags
+    val exportCaseTags: Boolean,
+    val exportObservableTags: Boolean,
     includedTheHiveOrganisations: Seq[String],
     excludedTheHiveOrganisations: Seq[String]
 ) extends MispClient(
@@ -128,6 +132,7 @@ class TheHiveMispClient(
       config.caseTemplate,
       config.artifactTags,
       config.exportCaseTags,
+      config.exportObservableTags,
       config.includedTheHiveOrganisations,
       config.excludedTheHiveOrganisations
     )

misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala

@@ -51,20 +51,22 @@ class TestMispClientProvider @Inject() (Action: DefaultActionBuilder, implicit v
     Json.parse(data)
   }
 
-  override def get(): TheHiveMispClient = new TheHiveMispClient(
-    name = "test",
-    baseUrl = baseUrl,
-    auth = NoAuthentication,
-    ws = ws,
-    maxAge = None,
-    excludedOrganisations = Nil,
-    excludedTags = Set.empty,
-    whitelistTags = Set.empty,
-    purpose = MispPurpose.ImportAndExport,
-    caseTemplate = None,
-    artifactTags = Seq("TEST"),
-    exportCaseTags = true,
-    includedTheHiveOrganisations = Seq("*"),
-    excludedTheHiveOrganisations = Nil
-  )
+  override def get(): TheHiveMispClient =
+    new TheHiveMispClient(
+      name = "test",
+      baseUrl = baseUrl,
+      auth = NoAuthentication,
+      ws = ws,
+      maxAge = None,
+      excludedOrganisations = Nil,
+      excludedTags = Set.empty,
+      whitelistTags = Set.empty,
+      purpose = MispPurpose.ImportAndExport,
+      caseTemplate = None,
+      artifactTags = Seq("TEST"),
+      exportCaseTags = true,
+      exportObservableTags = true,
+      includedTheHiveOrganisations = Seq("*"),
+      excludedTheHiveOrganisations = Nil
+    )
 }