diff --git a/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala b/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala index 3a664de478..edae588bc5 100644 --- a/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala +++ b/misp/client/src/main/scala/org/thp/misp/client/MispClient.scala @@ -7,7 +7,7 @@ import akka.stream.alpakka.json.scaladsl.JsonReader import akka.stream.scaladsl.{JsonFraming, Source} import akka.util.ByteString import org.thp.client.{ApplicationError, Authentication, ProxyWS} -import org.thp.misp.dto.{Attribute, Event, Organisation, User} +import org.thp.misp.dto.{Attribute, Event, Organisation, Tag, User} import org.thp.scalligraph.InternalError import play.api.Logger import play.api.http.Status @@ -45,10 +45,11 @@ class MispClient( Failure(InternalError(s"MISP server $name is inaccessible", t)) } - private def configuredProxy: Option[String] = ws match { - case c: ProxyWS => c.proxy.map(p => s"http://${p.host}:${p.port}") - case _ => None - } + private def configuredProxy: Option[String] = + ws match { + case c: ProxyWS => c.proxy.map(p => s"http://${p.host}:${p.port}") + case _ => None + } logger.info(s"""Add MISP connection $name | url: $baseUrl | proxy: ${configuredProxy.getOrElse("")} @@ -62,27 +63,52 @@ class MispClient( private def request(url: String): WSRequest = auth(ws.url(s"$strippedUrl/$url").withHttpHeaders("Accept" -> "application/json")) - private def get(url: String)(implicit ec: ExecutionContext): Future[JsValue] = + private def get(url: String)(implicit ec: ExecutionContext): Future[JsValue] = { + logger.trace(s"MISP request: GET $url") request(url).get().transform { - case Success(r) if r.status == Status.OK => Success(r.json) - case Success(r) => Try(r.json) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Success(r.json) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Try(r.json) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } + } - private def post(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[JsValue] = + private def post(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[JsValue] = { + logger.trace(s"MISP request: POST $url\n$body") request(url).post(body).transform { - case Success(r) if r.status == Status.OK => Success(r.json) - case Success(r) => Try(r.json) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Success(r.json) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Try(r.json) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } + } - private def post(url: String, body: Source[ByteString, _])(implicit ec: ExecutionContext): Future[JsValue] = + private def post(url: String, body: Source[ByteString, _])(implicit ec: ExecutionContext): Future[JsValue] = { + logger.trace(s"MISP request: POST $url (stream body)") request(url).post(body).transform { - case Success(r) if r.status == Status.OK => Success(r.json) - case Success(r) => Try(r.json) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Success(r.json) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText}\n${r.body}") + Try(r.json) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } -// + } + + // // private def getStream(url: String)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = // request(url).withMethod("GET").stream().transform { // case Success(r) if r.status == Status.OK => Success(r.bodyAsSource) @@ -90,12 +116,20 @@ class MispClient( // case Failure(t) => throw t // } - private def postStream(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = + private def postStream(url: String, body: JsValue)(implicit ec: ExecutionContext): Future[Source[ByteString, Any]] = { + logger.trace(s"MISP request: POST $url\n$body") request(url).withMethod("POST").withBody(body).stream().transform { - case Success(r) if r.status == Status.OK => Success(r.bodyAsSource) - case Success(r) => Try(r.bodyAsSource) - case Failure(t) => throw t + case Success(r) if r.status == Status.OK => + logger.trace(s"MISP response: ${r.status} ${r.statusText} (stream body)") + Success(r.bodyAsSource) + case Success(r) => + logger.trace(s"MISP response: ${r.status} ${r.statusText} (stream body)") + Try(r.bodyAsSource) + case Failure(t) => + logger.trace(s"MISP error: $t") + throw t } + } def getCurrentUser(implicit ec: ExecutionContext): Future[User] = { logger.debug("Get current user") @@ -177,7 +211,8 @@ class MispClient( maybeAttribute.fold(error => { logger.warn(s"Attribute has invalid format: ${data.decodeString("UTF-8")}", error); Nil }, List(_)) } .mapAsyncUnordered(2) { - case attribute @ Attribute(id, "malware-sample" | "attachment", _, _, _, _, _, _, _, None, _, _, _, _) => // TODO need to unzip malware samples ? + case attribute @ Attribute(id, "malware-sample" | "attachment", _, _, _, _, _, _, _, None, _, _, _, _) => + // TODO need to unzip malware samples ? downloadAttachment(id).map { case (filename, contentType, src) => attribute.copy(data = Some((filename, contentType, src))) } @@ -204,14 +239,16 @@ class MispClient( case Failure(t) => throw t } - def uploadAttachment(eventId: String, comment: String, filename: String, data: Source[ByteString, _])( - implicit ec: ExecutionContext + def uploadAttachment(eventId: String, comment: String, filename: String, data: Source[ByteString, _])(implicit + ec: ExecutionContext ): Future[JsValue] = { val stream = data .via(Base64Flow.encode()) .intersperse( ByteString( - s"""{"request":{"category":"Payload delivery","type":"malware-sample","comment":${JsString(comment).toString},"files":[{"filename":${JsString( + s"""{"request":{"category":"Payload delivery","type":"malware-sample","comment":${JsString( + comment + ).toString},"files":[{"filename":${JsString( filename ).toString},"data":"""" ), @@ -229,6 +266,7 @@ class MispClient( analysis: Int, distribution: Int, attributes: Seq[Attribute], + tags: Seq[Tag], extendsEvent: Option[String] = None )(implicit ec: ExecutionContext): Future[String] = { logger.debug(s"Create MISP event $info, with ${attributes.size} attributes") @@ -243,6 +281,7 @@ class MispClient( "analysis" -> analysis.toString, "distribution" -> distribution, "Attribute" -> stringAttributes, + "Tag" -> tags, "extends_uuid" -> extendsEvent ) ) diff --git a/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala b/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala index 33c65b1927..1b4dc4fcdb 100644 --- a/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala +++ b/misp/client/src/main/scala/org/thp/misp/dto/Attribute.scala @@ -55,8 +55,8 @@ object Attribute { "category" -> attribute.category, "type" -> attribute.`type`, "value" -> attribute.value, - "comment" -> attribute.comment -// "Tag" -> attribute.tags + "comment" -> attribute.comment, + "Tag" -> attribute.tags ) } } diff --git a/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala b/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala index de4880844f..683b1ee489 100644 --- a/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala +++ b/misp/client/src/main/scala/org/thp/misp/dto/Tag.scala @@ -20,8 +20,5 @@ object Tag { } and (JsPath \ "exportable").readNullable[Boolean])(Tag.apply _) - implicit val writes: Writes[Tag] = Writes[Tag] { - case Tag(Some(id), name, colour, _) => Json.obj("id" -> id, "name" -> name, "colour" -> colour.map(c => f"#$c%06X")) - case Tag(_, name, _, _) => JsString(name) - } + implicit val writes: Writes[Tag] = Json.writes[Tag] } diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala index fdcfabbb56..58c00318cd 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala @@ -31,7 +31,12 @@ class MispExportSrv @Inject() ( lazy val logger: Logger = Logger(getClass) - def observableToAttribute(observable: RichObservable): Option[Attribute] = + def observableToAttribute(observable: RichObservable, exportTags: Boolean): Option[Attribute] = { + lazy val mispTags = + if (exportTags) + observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(observable.tlp) + else + tlpTags.get(observable.tlp).toSeq connector .attributeConverter(observable.`type`) .map { @@ -50,7 +55,7 @@ class MispExportSrv @Inject() ( value = observable.data.fold(observable.attachment.get.name)(_.data), firstSeen = None, lastSeen = None, - tags = observable.tags.map(t => MispTag(None, t.toString, Some(t.colour), None)) + tags = mispTags ) } .orElse { @@ -59,6 +64,7 @@ class MispExportSrv @Inject() ( ) None } + } def getMispClient(mispId: String): Future[TheHiveMispClient] = connector @@ -77,8 +83,8 @@ class MispExportSrv @Inject() ( .filterByType("misp") .headOption - def getAttributes(`case`: Case with Entity)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] = - caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute) + def getAttributes(`case`: Case with Entity, exportTags: Boolean)(implicit graph: Graph, authContext: AuthContext): Iterator[Attribute] = + caseSrv.get(`case`).observables.isIoc.richObservable.toIterator.flatMap(observableToAttribute(_, exportTags)) def removeDuplicateAttributes(attributes: Iterator[Attribute]): Seq[Attribute] = { var attrSet = Set.empty[(String, String, String)] @@ -93,9 +99,21 @@ class MispExportSrv @Inject() ( builder.result() } - def createEvent(client: TheHiveMispClient, `case`: Case, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit + val tlpTags = Map( + 0 -> MispTag(None, "tlp:white", None, None), + 1 -> MispTag(None, "tlp:green", None, None), + 2 -> MispTag(None, "tlp:amber", None, None), + 3 -> MispTag(None, "tlp:red", None, None) + ) + def createEvent(client: TheHiveMispClient, `case`: Case with Entity, attributes: Seq[Attribute], extendsEvent: Option[String])(implicit ec: ExecutionContext - ): Future[String] = + ): Future[String] = { + val mispTags = + if (client.exportCaseTags) + db.roTransaction { implicit graph => + caseSrv.get(`case`._id).tags.toSeq.map(t => MispTag(None, t.toString, Some(t.colour), None)) ++ tlpTags.get(`case`.tlp) + } + else tlpTags.get(`case`.tlp).toSeq client.createEvent( info = `case`.title, date = `case`.startDate, @@ -104,8 +122,10 @@ class MispExportSrv @Inject() ( analysis = 0, distribution = 0, attributes = attributes, + tags = mispTags, extendsEvent = extendsEvent ) + } def createAlert(client: TheHiveMispClient, `case`: Case with Entity, eventId: String)(implicit graph: Graph, @@ -147,7 +167,7 @@ class MispExportSrv @Inject() ( orgName <- Future.fromTry(client.currentOrganisationName) maybeAlert = db.roTransaction(implicit graph => getAlert(`case`, orgName)) _ = logger.debug(maybeAlert.fold("Related MISP event doesn't exist")(a => s"Related MISP event found : ${a.sourceRef}")) - attributes = db.roTransaction(implicit graph => removeDuplicateAttributes(getAttributes(`case`))) + attributes = db.roTransaction(implicit graph => removeDuplicateAttributes(getAttributes(`case`, client.exportObservableTags))) eventId <- createEvent(client, `case`, attributes, maybeAlert.map(_.sourceRef)) _ <- Future.fromTry(db.tryTransaction(implicit graph => createAlert(client, `case`, eventId))) } yield eventId diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala index 4841cda933..50f0bc2dfb 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/TheHiveMispClient.scala @@ -29,6 +29,7 @@ case class TheHiveMispClientConfig( caseTemplate: Option[String], artifactTags: Seq[String] = Nil, exportCaseTags: Boolean = false, + exportObservableTags: Boolean = false, includedTheHiveOrganisations: Seq[String] = Seq("*"), excludedTheHiveOrganisations: Seq[String] = Nil ) @@ -49,6 +50,7 @@ object TheHiveMispClientConfig { caseTemplate <- (JsPath \ "caseTemplate").readNullable[String] artifactTags <- (JsPath \ "tags").readWithDefault[Seq[String]](Nil) exportCaseTags <- (JsPath \ "exportCaseTags").readWithDefault[Boolean](false) + exportObservableTags <- (JsPath \ "exportObservableTags").readWithDefault[Boolean](false) includedTheHiveOrganisations <- (JsPath \ "includedTheHiveOrganisations").readWithDefault[Seq[String]](Seq("*")) excludedTheHiveOrganisations <- (JsPath \ "excludedTheHiveOrganisations").readWithDefault[Seq[String]](Nil) } yield TheHiveMispClientConfig( @@ -64,6 +66,7 @@ object TheHiveMispClientConfig { caseTemplate, artifactTags, exportCaseTags, + exportObservableTags, includedTheHiveOrganisations, excludedTheHiveOrganisations ) @@ -100,7 +103,8 @@ class TheHiveMispClient( purpose: MispPurpose.Value, val caseTemplate: Option[String], artifactTags: Seq[String], // FIXME use artifactTags - exportCaseTags: Boolean, // FIXME use exportCaseTags + val exportCaseTags: Boolean, + val exportObservableTags: Boolean, includedTheHiveOrganisations: Seq[String], excludedTheHiveOrganisations: Seq[String] ) extends MispClient( @@ -128,6 +132,7 @@ class TheHiveMispClient( config.caseTemplate, config.artifactTags, config.exportCaseTags, + config.exportObservableTags, config.includedTheHiveOrganisations, config.excludedTheHiveOrganisations ) diff --git a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala index 9538840aec..a881cc7298 100644 --- a/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala +++ b/misp/connector/src/test/scala/org/thp/thehive/connector/misp/services/TestMispClientProvider.scala @@ -51,20 +51,22 @@ class TestMispClientProvider @Inject() (Action: DefaultActionBuilder, implicit v Json.parse(data) } - override def get(): TheHiveMispClient = new TheHiveMispClient( - name = "test", - baseUrl = baseUrl, - auth = NoAuthentication, - ws = ws, - maxAge = None, - excludedOrganisations = Nil, - excludedTags = Set.empty, - whitelistTags = Set.empty, - purpose = MispPurpose.ImportAndExport, - caseTemplate = None, - artifactTags = Seq("TEST"), - exportCaseTags = true, - includedTheHiveOrganisations = Seq("*"), - excludedTheHiveOrganisations = Nil - ) + override def get(): TheHiveMispClient = + new TheHiveMispClient( + name = "test", + baseUrl = baseUrl, + auth = NoAuthentication, + ws = ws, + maxAge = None, + excludedOrganisations = Nil, + excludedTags = Set.empty, + whitelistTags = Set.empty, + purpose = MispPurpose.ImportAndExport, + caseTemplate = None, + artifactTags = Seq("TEST"), + exportCaseTags = true, + exportObservableTags = true, + includedTheHiveOrganisations = Seq("*"), + excludedTheHiveOrganisations = Nil + ) }