diff --git a/ScalliGraph b/ScalliGraph index 00adeee13a..7f2f8c4fd0 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 00adeee13aa551d53c93f1204ec364b73fcf2b57 +Subproject commit 7f2f8c4fd04f254f76545090350fb80917582d00 diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Attachment.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Attachment.scala index f1fe2ce0b6..b5b7a1377b 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Attachment.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Attachment.scala @@ -1,6 +1,12 @@ package org.thp.thehive.dto.v0 -import play.api.libs.json.{Json, OFormat} +import play.api.libs.json.{Json, OFormat, Writes} + +case class InputAttachment(name: String, contentType: String, id: String) + +object InputAttachment { + implicit val writes: Writes[InputAttachment] = Json.writes[InputAttachment] +} case class OutputAttachment(name: String, hashes: Seq[String], size: Long, contentType: String, id: String) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala index 369ead4aa9..37173a6296 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v0/Observable.scala @@ -1,7 +1,6 @@ package org.thp.thehive.dto.v0 import java.util.Date - import org.scalactic.Accumulation._ import org.scalactic.Good import org.thp.scalligraph.controllers._ @@ -9,11 +8,12 @@ import play.api.libs.json.{JsObject, Json, OFormat, Writes} case class InputObservable( dataType: String, - @WithParser(InputObservable.fp) + @WithParser(InputObservable.dataParser) data: Seq[String] = Nil, message: Option[String] = None, startDate: Option[Date] = None, - attachment: Option[FFile] = None, + @WithParser(InputObservable.fileOrAttachmentParser) + attachment: Seq[Either[FFile, InputAttachment]] = Seq.empty, tlp: Option[Int] = None, tags: Set[String] = Set.empty, ioc: Option[Boolean] = None, @@ -22,14 +22,31 @@ case class InputObservable( ) object InputObservable { + implicit val fileOrAttachmentWrites: Writes[Either[FFile, InputAttachment]] = Writes[Either[FFile, InputAttachment]] { + case Left(file) => Json.toJson(file) + case Right(attachment) => Json.toJson(attachment) + } implicit val writes: Writes[InputObservable] = Json.writes[InputObservable] - val fp: FieldsParser[Seq[String]] = FieldsParser[Seq[String]]("data") { + val dataParser: FieldsParser[Seq[String]] = FieldsParser[Seq[String]]("data") { case (_, FString(s)) => Good(Seq(s)) case (_, FAny(s)) => Good(s) case (_, FSeq(a)) => a.validatedBy(FieldsParser.string(_)) case (_, FUndefined) => Good(Nil) } + + val fileOrAttachmentParser: FieldsParser[Seq[Either[FFile, InputAttachment]]] = + FieldsParser[FFile] + .map("fileOrAttachmentParser")(f => Seq(Left(f))) + .recover( + FieldsParser[InputAttachment] + .map("fileOrAttachmentParser")(a => Seq(Right(a))) + .recover( + FieldsParser[InputAttachment] + .sequence + .map("fileOrAttachmentParser")(as => as.map(Right(_))) + ) + ) } case class OutputObservable( diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Attachment.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Attachment.scala index e0f9d9dfcc..afd3d5719e 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Attachment.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Attachment.scala @@ -1,6 +1,12 @@ package org.thp.thehive.dto.v1 -import play.api.libs.json.{Json, OFormat} +import play.api.libs.json.{Json, OFormat, Writes} + +case class InputAttachment(name: String, contentType: String, id: String) + +object InputAttachment { + implicit val writes: Writes[InputAttachment] = Json.writes[InputAttachment] +} case class OutputAttachment(name: String, hashes: Seq[String], size: Long, contentType: String, id: String) diff --git a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala index 4211b762f9..3562dab2cd 100644 --- a/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala +++ b/dto/src/main/scala/org/thp/thehive/dto/v1/Observable.scala @@ -1,7 +1,6 @@ package org.thp.thehive.dto.v1 import java.util.Date - import org.scalactic.Accumulation._ import org.scalactic.Good import org.thp.scalligraph.controllers._ @@ -9,11 +8,12 @@ import play.api.libs.json.{JsObject, Json, OFormat, Writes} case class InputObservable( dataType: String, - @WithParser(InputObservable.fp) + @WithParser(InputObservable.dataParser) data: Seq[String] = Nil, message: Option[String] = None, startDate: Option[Date] = None, - attachment: Option[FFile] = None, + @WithParser(InputObservable.fileOrAttachmentParser) + attachment: Seq[Either[FFile, InputAttachment]] = Seq.empty, tlp: Option[Int] = None, tags: Set[String] = Set.empty, ioc: Option[Boolean] = None, @@ -22,14 +22,32 @@ case class InputObservable( ) object InputObservable { + implicit val fileOrAttachmentWrites: Writes[Either[FFile, InputAttachment]] = Writes[Either[FFile, InputAttachment]] { + case Left(file) => Json.toJson(file) + case Right(attachment) => Json.toJson(attachment) + } + implicit val writes: Writes[InputObservable] = Json.writes[InputObservable] - val fp: FieldsParser[Seq[String]] = FieldsParser[Seq[String]]("data") { + val dataParser: FieldsParser[Seq[String]] = FieldsParser[Seq[String]]("data") { case (_, FString(s)) => Good(Seq(s)) case (_, FAny(s)) => Good(s) case (_, FSeq(a)) => a.validatedBy(FieldsParser.string(_)) case (_, FUndefined) => Good(Nil) } + + val fileOrAttachmentParser: FieldsParser[Seq[Either[FFile, InputAttachment]]] = + FieldsParser[FFile] + .map("fileOrAttachmentParser")(f => Seq(Left(f))) + .recover( + FieldsParser[InputAttachment] + .map("fileOrAttachmentParser")(a => Seq(Right(a))) + .recover( + FieldsParser[InputAttachment] + .sequence + .map("fileOrAttachmentParser")(as => as.map(Right(_))) + ) + ) } case class OutputObservable( diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index 0196222e9a..1a8ba2b182 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -1,20 +1,16 @@ package org.thp.thehive.controllers.v0 -import java.io.FilterInputStream -import java.nio.file.Files - -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ -import org.thp.scalligraph.models.{Database, UMapping} +import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.thehive.controllers.v0.Conversion._ -import org.thp.thehive.dto.v0.InputObservable +import org.thp.thehive.dto.v0.{InputAttachment, InputObservable} import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ @@ -27,8 +23,11 @@ import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.libs.json.{JsArray, JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} +import java.io.FilterInputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.collection.JavaConverters._ -import scala.util.Success +import scala.util.{Failure, Success} @Singleton class ObservableCtrl @Inject() ( @@ -38,6 +37,7 @@ class ObservableCtrl @Inject() ( observableSrv: ObservableSrv, observableTypeSrv: ObservableTypeSrv, caseSrv: CaseSrv, + attachmentSrv: AttachmentSrv, errorHandler: ErrorHandler, @Named("v0") override val queryExecutor: QueryExecutor, override val publicData: PublicObservable, @@ -68,48 +68,60 @@ class ObservableCtrl @Inject() ( } .map { case (case0, observableType) => - val initialSuccessesAndFailures: (Seq[JsValue], Seq[JsValue]) = - inputAttachObs.foldLeft[(Seq[JsValue], Seq[JsValue])](Nil -> Nil) { - case ((successes, failures), inputObservable) => - inputObservable.attachment.fold((successes, failures)) { attachment => - db - .tryTransaction { implicit graph => - observableSrv - .create(inputObservable.toObservable, observableType, attachment, inputObservable.tags, Nil) - .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) - } - .fold( - e => - successes -> (failures :+ errorHandler.toErrorResult(e)._2 ++ Json - .obj( - "object" -> Json - .obj("data" -> s"file:${attachment.filename}", "attachment" -> Json.obj("name" -> attachment.filename)) - )), - s => (successes :+ s) -> failures - ) - } + val (successes, failures) = inputAttachObs + .flatMap { obs => + obs.attachment.map(createAttachmentObservable(case0, obs, observableType, _)) ++ + obs.data.map(createSimpleObservable(case0, obs, observableType, _)) } - - val (successes, failures) = inputObservable - .data - .foldLeft(initialSuccessesAndFailures) { - case ((successes, failures), data) => - db - .tryTransaction { implicit graph => - observableSrv - .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) - .flatMap(o => caseSrv.addObservable(case0, o).map(_ => o.toJson)) - } - .fold( - failure => (successes, failures :+ errorHandler.toErrorResult(failure)._2 ++ Json.obj("object" -> Json.obj("data" -> data))), - success => (successes :+ success, failures) - ) + .foldLeft[(Seq[JsValue], Seq[JsValue])]((Nil, Nil)) { + case ((s, f), Right(o)) => (s :+ o, f) + case ((s, f), Left(o)) => (s, f :+ o) } if (failures.isEmpty) Results.Created(JsArray(successes)) else Results.MultiStatus(Json.obj("success" -> successes, "failure" -> failures)) } } + def createSimpleObservable( + `case`: Case with Entity, + inputObservable: InputObservable, + observableType: ObservableType with Entity, + data: String + )(implicit authContext: AuthContext): Either[JsValue, JsValue] = + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) + } match { + case Success(o) => Right(o.toJson) + case Failure(error) => Left(errorHandler.toErrorResult(error)._2 ++ Json.obj("object" -> Json.obj("data" -> data))) + } + + def createAttachmentObservable( + `case`: Case with Entity, + inputObservable: InputObservable, + observableType: ObservableType with Entity, + fileOrAttachment: Either[FFile, InputAttachment] + )(implicit authContext: AuthContext): Either[JsValue, JsValue] = + db + .tryTransaction { implicit graph => + val observable = fileOrAttachment match { + case Left(file) => observableSrv.create(inputObservable.toObservable, observableType, file, inputObservable.tags, Nil) + case Right(attachment) => + for { + attach <- attachmentSrv.duplicate(attachment.name, attachment.contentType, attachment.id) + obs <- observableSrv.create(inputObservable.toObservable, observableType, attach, inputObservable.tags, Nil) + } yield obs + } + observable.flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) + } match { + case Success(o) => Right(o.toJson) + case _ => + val filename = fileOrAttachment.fold(_.filename, _.name) + Left(Json.obj("object" -> Json.obj("data" -> s"file:$filename", "attachment" -> Json.obj("name" -> filename)))) + } + def get(observableId: String): Action[AnyContent] = entrypoint("get observable") .authRoTransaction(db) { implicit request => implicit graph => @@ -214,8 +226,8 @@ class ObservableCtrl @Inject() ( } } - private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = - observable.attachment.toSeq.flatMap { attachment => + private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] = + observable.attachment.flatMap(_.swap.toSeq).flatMap { attachment => val zipFile = new ZipFile(attachment.filepath.toFile) val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] @@ -225,7 +237,7 @@ class ObservableCtrl @Inject() ( files .filterNot(_.isDirectory) .flatMap(extractAndCheckSize(zipFile, _)) - .map(ffile => observable.copy(attachment = Some(ffile))) + .map(ffile => observable.copy(attachment = Seq(Left(ffile)))) } } diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index f6902e11ec..e41f8822f7 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -1,20 +1,16 @@ package org.thp.thehive.controllers.v1 -import java.io.FilterInputStream -import java.nio.file.Files - -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ -import org.thp.scalligraph.models.Database +import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} import org.thp.thehive.controllers.v1.Conversion._ -import org.thp.thehive.dto.v1.InputObservable +import org.thp.thehive.dto.v1.{InputAttachment, InputObservable} import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ @@ -22,10 +18,15 @@ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services._ import play.api.libs.Files.DefaultTemporaryFileCreator +import play.api.libs.json.{JsArray, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} import play.api.{Configuration, Logger} +import java.io.FilterInputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.collection.JavaConverters._ +import scala.util.{Failure, Success} @Singleton class ObservableCtrl @Inject() ( @@ -36,6 +37,8 @@ class ObservableCtrl @Inject() ( observableTypeSrv: ObservableTypeSrv, caseSrv: CaseSrv, organisationSrv: OrganisationSrv, + attachmentSrv: AttachmentSrv, + errorHandler: ErrorHandler, temporaryFileCreator: DefaultTemporaryFileCreator, configuration: Configuration ) extends QueryableCtrl @@ -84,35 +87,79 @@ class ObservableCtrl @Inject() ( .extract("artifact", FieldsParser[InputObservable]) .extract("isZip", FieldsParser.boolean.optional.on("isZip")) .extract("zipPassword", FieldsParser.string.optional.on("zipPassword")) - .authTransaction(db) { implicit request => implicit graph => + .auth { implicit request => + val inputObservable: InputObservable = request.body("artifact") val isZip: Option[Boolean] = request.body("isZip") val zipPassword: Option[String] = request.body("zipPassword") - val inputObservable: InputObservable = request.body("artifact") val inputAttachObs = if (isZip.contains(true)) getZipFiles(inputObservable, zipPassword) else Seq(inputObservable) - for { - case0 <- - caseSrv - .get(EntityIdOrName(caseId)) - .can(Permissions.manageObservable) - .getOrFail("Case") - observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) - observablesWithData <- - inputObservable - .data - .toTry(d => observableSrv.create(inputObservable.toObservable, observableType, d, inputObservable.tags, Nil)) - observableWithAttachment <- inputAttachObs.toTry( - _.attachment - .map(a => observableSrv.create(inputObservable.toObservable, observableType, a, inputObservable.tags, Nil)) - .flip - ) - createdObservables <- (observablesWithData ++ observableWithAttachment.flatten).toTry { richObservables => - caseSrv - .addObservable(case0, richObservables) - .map(_ => richObservables) + + db + .roTransaction { implicit graph => + for { + case0 <- + caseSrv + .get(EntityIdOrName(caseId)) + .can(Permissions.manageObservable) + .orFail(AuthorizationError("Operation not permitted")) + observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) + } yield (case0, observableType) + } + .map { + case (case0, observableType) => + val (successes, failures) = inputAttachObs + .flatMap { obs => + obs.attachment.map(createAttachmentObservable(case0, obs, observableType, _)) ++ + obs.data.map(createSimpleObservable(case0, obs, observableType, _)) + } + .foldLeft[(Seq[JsValue], Seq[JsValue])]((Nil, Nil)) { + case ((s, f), Right(o)) => (s :+ o, f) + case ((s, f), Left(o)) => (s, f :+ o) + } + if (failures.isEmpty) Results.Created(JsArray(successes)) + else Results.MultiStatus(Json.obj("success" -> successes, "failure" -> failures)) } - } yield Results.Created(createdObservables.toJson) } + def createSimpleObservable( + `case`: Case with Entity, + inputObservable: InputObservable, + observableType: ObservableType with Entity, + data: String + )(implicit authContext: AuthContext): Either[JsValue, JsValue] = + db + .tryTransaction { implicit graph => + observableSrv + .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) + } match { + case Success(o) => Right(o.toJson) + case Failure(error) => Left(errorHandler.toErrorResult(error)._2 ++ Json.obj("object" -> Json.obj("data" -> data))) + } + + def createAttachmentObservable( + `case`: Case with Entity, + inputObservable: InputObservable, + observableType: ObservableType with Entity, + fileOrAttachment: Either[FFile, InputAttachment] + )(implicit authContext: AuthContext): Either[JsValue, JsValue] = + db + .tryTransaction { implicit graph => + val observable = fileOrAttachment match { + case Left(file) => observableSrv.create(inputObservable.toObservable, observableType, file, inputObservable.tags, Nil) + case Right(attachment) => + for { + attach <- attachmentSrv.duplicate(attachment.name, attachment.contentType, attachment.id) + obs <- observableSrv.create(inputObservable.toObservable, observableType, attach, inputObservable.tags, Nil) + } yield obs + } + observable.flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) + } match { + case Success(o) => Right(o.toJson) + case _ => + val filename = fileOrAttachment.fold(_.filename, _.name) + Left(Json.obj("object" -> Json.obj("data" -> s"file:$filename", "attachment" -> Json.obj("name" -> filename)))) + } + def get(observableId: String): Action[AnyContent] = entryPoint("get observable") .authRoTransaction(db) { _ => implicit graph => @@ -197,8 +244,8 @@ class ObservableCtrl @Inject() ( } } - private def getZipFiles(observable: InputObservable, zipPassword: Option[String])(implicit authContext: AuthContext): Seq[InputObservable] = - observable.attachment.toSeq.flatMap { attachment => + private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] = + observable.attachment.flatMap(_.swap.toSeq).flatMap { attachment => val zipFile = new ZipFile(attachment.filepath.toFile) val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]] @@ -208,6 +255,6 @@ class ObservableCtrl @Inject() ( files .filterNot(_.isDirectory) .flatMap(extractAndCheckSize(zipFile, _)) - .map(ffile => observable.copy(attachment = Some(ffile))) + .map(ffile => observable.copy(attachment = Seq(Left(ffile)))) } } diff --git a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala index cc3165c5a3..538a47f4cf 100644 --- a/thehive/app/org/thp/thehive/services/AttachmentSrv.scala +++ b/thehive/app/org/thp/thehive/services/AttachmentSrv.scala @@ -1,14 +1,11 @@ package org.thp.thehive.services -import java.io.InputStream -import java.nio.file.Files - import akka.NotUsed import akka.stream.scaladsl.{Source, StreamConverters} import akka.stream.{IOResult, Materializer} import akka.util.ByteString -import javax.inject.{Inject, Named, Singleton} import org.apache.tinkerpop.gremlin.structure.Graph +import org.thp.scalligraph.NotFoundError import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models.{Database, Entity} @@ -20,6 +17,9 @@ import org.thp.thehive.models.Attachment import org.thp.thehive.services.AttachmentOps._ import play.api.Configuration +import java.io.InputStream +import java.nio.file.Files +import javax.inject.{Inject, Named, Singleton} import scala.concurrent.Future import scala.util.Try @@ -61,6 +61,20 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage storageSrv.saveBinary("attachment", id, data).flatMap(_ => createEntity(Attachment(filename, size, contentType, hs, id))) } + def duplicate(filename: String, contentType: String, attachmentId: String)(implicit + graph: Graph, + authContext: AuthContext + ): Try[Attachment with Entity] = { + val (size, hashes) = getByName(attachmentId).headOption match { + case Some(a) => (a.size, a.hashes) + case None => + val s = storageSrv.getSize("attachment", attachmentId).getOrElse(throw NotFoundError(s"Attachment $attachmentId not found")) + val hs = hashers.fromInputStream(storageSrv.loadBinary("attachment", attachmentId)) + (s, hs) + } + createEntity(Attachment(filename, size, contentType, hashes, attachmentId)) + } + override def getByName(name: String)(implicit graph: Graph): Traversal.V[Attachment] = startTraversal.getByAttachmentId(name)