diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala index 77e89b7a08..e3ba2fb5d4 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala @@ -94,10 +94,19 @@ class ActionOperationSrv @Inject() ( case AddArtifactToCase(_, dataType, dataMessage) => for { - c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_)) - obsType <- observableTypeSrv.getOrFail(EntityIdOrName(dataType)) + c <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_)) + obsType <- observableTypeSrv.getOrFail(EntityIdOrName(dataType)) + organisation <- organisationSrv.getOrFail(authContext.organisation) richObservable <- observableSrv.create( - Observable(Some(dataMessage), 2, ioc = false, sighted = false, ignoreSimilarity = None), + Observable( + Some(dataMessage), + 2, + ioc = false, + sighted = false, + ignoreSimilarity = None, + organisationIds = Seq(organisation._id), + relatedId = c._id + ), obsType, dataMessage, Set.empty[String], diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala index 5f1256fd91..8fb45cdd78 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Conversion.scala @@ -2,6 +2,7 @@ package org.thp.thehive.connector.cortex.services import io.scalaland.chimney.dsl._ import org.thp.cortex.dto.v0.{OutputArtifact, OutputMinireport, JobStatus => CortexJobStatus} +import org.thp.scalligraph.EntityId import org.thp.thehive.connector.cortex.models.JobStatus import org.thp.thehive.models.{Observable, ReportTag, ReportTagLevel} @@ -21,7 +22,7 @@ object Conversion { implicit class CortexOutputArtifactOps(artifact: OutputArtifact) { - def toObservable: Observable = + def toObservable(relatedId: EntityId, organisationIds: EntityId*): Observable = artifact .into[Observable] .withFieldComputed(_.message, _.message) @@ -29,6 +30,8 @@ object Conversion { .withFieldConst(_.ioc, false) .withFieldConst(_.sighted, false) .withFieldConst(_.ignoreSimilarity, None) + .withFieldConst(_.organisationIds, organisationIds) + .withFieldConst(_.relatedId, relatedId) .transform } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala index fd552203ab..c7a9c9cee0 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala @@ -212,11 +212,17 @@ class JobSrv @Inject() ( Future .fromTry { db.tryTransaction { implicit graph => - observableSrv - .create(artifact.toObservable, dataType, artifact.data.get, artifact.tags, Nil) - .flatMap { richObservable => - addObservable(job, richObservable.observable) - } + for { + origObs <- get(job).observable.getOrFail("Observable") + obs <- observableSrv.create( + artifact.toObservable(job._id, origObs.organisationIds: _*), + dataType, + artifact.data.get, + artifact.tags, + Nil + ) + _ <- addObservable(job, obs.observable) + } yield () } } case Failure(e) => Future.failed(e) @@ -258,9 +264,12 @@ class JobSrv @Inject() ( savedAttachment <- Future.fromTry { db.tryTransaction { implicit graph => for { + origObs <- get(job).observable.getOrFail("Observable") createdAttachment <- attachmentSrv.create(fFile) - richObservable <- observableSrv.create(artifact.toObservable, attachmentType, createdAttachment, artifact.tags, Nil) - _ <- reportObservableSrv.create(ReportObservable(), job, richObservable.observable) + richObservable <- + observableSrv + .create(artifact.toObservable(job._id, origObs.organisationIds: _*), attachmentType, createdAttachment, artifact.tags, Nil) + _ <- reportObservableSrv.create(ReportObservable(), job, richObservable.observable) } yield createdAttachment } } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index 2b6ec2eaa4..c43c2dc9d9 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -109,7 +109,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp, ioc, sighted, None), + Observable(message, tlp, ioc, sighted, None, Nil, EntityId("")), // organisation and related Ids are filled by output Seq(mainOrganisation), dataType, tags, @@ -231,7 +231,15 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp.getOrElse(2), ioc.getOrElse(false), sighted = false, ignoreSimilarity = None), + Observable( + message, + tlp.getOrElse(2), + ioc.getOrElse(false), + sighted = false, + ignoreSimilarity = None, + organisationIds = Nil, + relatedId = EntityId("") + ), Nil, dataType, tags, @@ -450,7 +458,7 @@ trait Conversion { ) } yield InputObservable( metaData, - Observable(message, tlp, ioc, sighted, ignoreSimilarity = None), + Observable(message, tlp, ioc, sighted, ignoreSimilarity = None, organisationIds = Nil, relatedId = EntityId("")), Seq(mainOrganisation), dataType, tags, diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index e1efedba2a..ebd6a0986e 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -23,6 +23,7 @@ import org.thp.thehive.migration.IdMapping import org.thp.thehive.migration.dto._ import org.thp.thehive.models._ import org.thp.thehive.services._ +import org.thp.thehive.connector.cortex.services.JobOps._ import play.api.cache.SyncCacheApi import play.api.cache.ehcache.EhCacheModule import play.api.inject.guice.GuiceInjector @@ -600,23 +601,30 @@ class Output @Inject() ( for { observableType <- getObservableType(inputObservable.`type`) tags <- inputObservable.tags.filterNot(_.isEmpty).toTry(getTag) + orgs <- inputObservable.organisations.toTry(getOrganisation) richObservable <- inputObservable .dataOrAttachment .fold( dataValue => dataSrv.createEntity(Data(dataValue)).flatMap { data => - observableSrv.create(inputObservable.observable, observableType, data, tags, Nil) + observableSrv + .create(inputObservable.observable.copy(organisationIds = orgs.map(_._id), relatedId = caseId), observableType, data, tags, Nil) }, inputAttachment => attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { attachment => - observableSrv.create(inputObservable.observable, observableType, attachment, tags, Nil) + observableSrv.create( + inputObservable.observable.copy(organisationIds = orgs.map(_._id), relatedId = caseId), + observableType, + attachment, + tags, + Nil + ) } ) _ = updateMetaData(richObservable.observable, inputObservable.metaData) case0 <- getCase(caseId) - orgs <- inputObservable.organisations.toTry(getOrganisation) _ <- orgs.toTry(o => shareSrv.shareObservable(richObservable, case0, o)) } yield IdMapping(inputObservable.metaData.id, richObservable._id) } @@ -636,6 +644,7 @@ class Output @Inject() ( logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in job $jobId") for { job <- jobSrv.getOrFail(jobId) + jobObs <- jobSrv.get(job).observable.getOrFail("Observable") observableType <- getObservableType(inputObservable.`type`) tags = inputObservable.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq richObservable <- @@ -644,12 +653,25 @@ class Output @Inject() ( .fold( dataValue => dataSrv.createEntity(Data(dataValue)).flatMap { data => - observableSrv.create(inputObservable.observable, observableType, data, tags, Nil) + observableSrv.create( + inputObservable.observable.copy(organisationIds = jobObs.organisationIds, relatedId = jobId), + observableType, + data, + tags, + Nil + ) }, inputAttachment => attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { attachment => - observableSrv.create(inputObservable.observable, observableType, attachment, tags, Nil) + observableSrv + .create( + inputObservable.observable.copy(organisationIds = jobObs.organisationIds, relatedId = jobId), + observableType, + attachment, + tags, + Nil + ) } ) _ = updateMetaData(richObservable.observable, inputObservable.metaData) @@ -678,7 +700,7 @@ class Output @Inject() ( ) tags = inputAlert.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq // alert <- alertSrv.create(inputAlert.alert, organisation, tags, inputAlert.customFields, caseTemplate) // FIXME don't check duplicate - alert <- alertSrv.createEntity(inputAlert.alert) + alert <- alertSrv.createEntity(inputAlert.alert.copy(organisationId = organisation._id)) _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), alert, organisation) _ <- caseTemplate.map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), alert, ct)).flip _ <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t)) @@ -691,26 +713,40 @@ class Output @Inject() ( override def createAlertObservable(alertId: EntityId, inputObservable: InputObservable): Try[IdMapping] = authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in alert $alertId") + val tags = inputObservable.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq for { observableType <- getObservableType(inputObservable.`type`) - tags = inputObservable.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq + alert <- alertSrv.getOrFail(alertId) richObservable <- inputObservable .dataOrAttachment .fold( dataValue => dataSrv.createEntity(Data(dataValue)).flatMap { data => - observableSrv.create(inputObservable.observable, observableType, data, tags, Nil) + observableSrv.create( + inputObservable.observable.copy(organisationIds = Seq(alert.organisationId), relatedId = alertId), + observableType, + data, + tags, + Nil + ) }, inputAttachment => attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { attachment => - observableSrv.create(inputObservable.observable, observableType, attachment, tags, Nil) + observableSrv + .create( + inputObservable.observable.copy(organisationIds = Seq(alert.organisationId), relatedId = alertId), + observableType, + attachment, + tags, + Nil + ) } ) _ = updateMetaData(richObservable.observable, inputObservable.metaData) - alert <- alertSrv.getOrFail(alertId) - _ <- alertSrv.alertObservableSrv.create(AlertObservable(), alert, richObservable.observable) + + _ <- alertSrv.alertObservableSrv.create(AlertObservable(), alert, richObservable.observable) } yield IdMapping(inputObservable.metaData.id, richObservable._id) } diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala index a708f7feff..c70819781b 100644 --- a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala +++ b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala @@ -23,7 +23,7 @@ import play.api.libs.json._ import java.nio.file.Files import java.util.Date -import javax.inject.{Inject, Named, Singleton} +import javax.inject.{Inject, Singleton} import scala.concurrent.duration.DurationInt import scala.concurrent.{Await, ExecutionContext} import scala.util.{Failure, Success, Try} @@ -91,9 +91,9 @@ class MispImportSrv @Inject() ( .fold(observableTypeSrv.getOrFail(EntityName("other")).map(_ -> Seq.empty[String]))(Success(_)) } - def attributeToObservable( - attribute: Attribute - )(implicit graph: Graph): List[(Observable, ObservableType with Entity, Set[String], Either[String, (String, String, Source[ByteString, _])])] = + def attributeToObservable(alert: Alert with Entity, attribute: Attribute)(implicit + graph: Graph + ): List[(Observable, ObservableType with Entity, Set[String], Either[String, (String, String, Source[ByteString, _])])] = attribute .`type` .split('|') @@ -114,7 +114,15 @@ class MispImportSrv @Inject() ( ) List( ( - Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), + Observable( + attribute.comment, + 0, + ioc = false, + sighted = false, + ignoreSimilarity = None, + organisationIds = Seq(alert.organisationId), + relatedId = alert._id + ), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Right(attribute.data.get) @@ -126,7 +134,15 @@ class MispImportSrv @Inject() ( ) List( ( - Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), + Observable( + attribute.comment, + 0, + ioc = false, + sighted = false, + ignoreSimilarity = None, + organisationIds = Seq(alert.organisationId), + relatedId = alert._id + ), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Left(attribute.value) @@ -144,7 +160,15 @@ class MispImportSrv @Inject() ( s"attribute ${attribute.category}:${attribute.`type`} (${attribute.tags}) is converted to observable $observableType with tags $additionalTags" ) ( - Observable(attribute.comment, 0, ioc = false, sighted = false, ignoreSimilarity = None), + Observable( + attribute.comment, + 0, + ioc = false, + sighted = false, + ignoreSimilarity = None, + organisationIds = Seq(alert.organisationId), + relatedId = alert._id + ), observableType, attribute.tags.map(_.name).toSet ++ additionalTags, Left(value) @@ -278,7 +302,7 @@ class MispImportSrv @Inject() ( val queue = client .searchAttributes(event.id, lastSynchro) - .mapConcat(attributeToObservable) + .mapConcat(attributeToObservable(alert, _)) .fold( Map.empty[ (String, String), diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index e2a8f0c40a..2770eb6136 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -6,7 +6,7 @@ import io.scalaland.chimney.dsl._ import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers._ -import org.thp.scalligraph.models.{Database, UMapping} +import org.thp.scalligraph.models.{Database, Entity, UMapping} import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Graph, IdentityConverter, IteratorOutput, Traversal} @@ -301,7 +301,7 @@ class AlertCtrl @Inject() ( } yield Results.Ok(alertWithObservables.toJson) } - private def createObservable(observable: InputObservable)(implicit + private def createObservable(organisation: Organisation with Entity, observable: InputObservable)(implicit graph: Graph, authContext: AuthContext ): Try[Seq[RichObservable]] = @@ -314,15 +314,20 @@ class AlertCtrl @Inject() ( val data = Base64.getDecoder.decode(value) attachmentSrv .create(filename, contentType, data) - .flatMap(attachment => observableSrv.create(observable.toObservable, attachmentType, attachment, observable.tags, Nil)) + .flatMap(attachment => + observableSrv.create(observable.toObservable(organisation._id), attachmentType, attachment, observable.tags, Nil) + ) case Array(filename, contentType) => attachmentSrv .create(filename, contentType, Array.emptyByteArray) - .flatMap(attachment => observableSrv.create(observable.toObservable, attachmentType, attachment, observable.tags, Nil)) + .flatMap(attachment => + observableSrv.create(observable.toObservable(organisation._id), attachmentType, attachment, observable.tags, Nil) + ) case data => Failure(InvalidFormatAttributeError("artifacts.data", "filename;contentType;base64value", Set.empty, FString(data.mkString(";")))) } - case dataType => observable.data.toTry(d => observableSrv.create(observable.toObservable, dataType, d, observable.tags, Nil)) + case dataType => + observable.data.toTry(d => observableSrv.create(observable.toObservable(organisation._id), dataType, d, observable.tags, Nil)) } } diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index eaed19dfc2..8142aae1e3 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -2,6 +2,7 @@ package org.thp.thehive.controllers.v0 import org.apache.tinkerpop.gremlin.process.traversal.P +import java.lang.{Long => JLong} import javax.inject.{Inject, Named, Singleton} import org.thp.scalligraph.controllers.{Entrypoint, FPathElem, FPathEmpty, FieldsParser} import org.thp.scalligraph.models.{Database, UMapping} @@ -16,6 +17,7 @@ import org.thp.thehive.models._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CaseTemplateOps._ import org.thp.thehive.services.CustomFieldOps._ +import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TagOps._ import org.thp.thehive.services.UserOps._ @@ -187,6 +189,7 @@ class CaseCtrl @Inject() ( class PublicCase @Inject() ( caseSrv: CaseSrv, organisationSrv: OrganisationSrv, + observableSrv: ObservableSrv, userSrv: UserSrv, customFieldSrv: CustomFieldSrv, implicit val db: Database @@ -216,7 +219,11 @@ class PublicCase @Inject() ( ) override val outputQuery: Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => caseSteps.richCase(authContext)) override val extraQueries: Seq[ParamQuery[_]] = Seq( - Query[Traversal.V[Case], Traversal.V[Observable]]("observables", (caseSteps, authContext) => caseSteps.observables(authContext)), + Query[Traversal.V[Case], Traversal.V[Observable]]( + "observables", + (caseSteps, authContext) => + observableSrv.startTraversal(caseSteps.graph).has(_.relatedId, P.within(caseSteps._id.toSeq: _*)).visible(authContext) + ), Query[Traversal.V[Case], Traversal.V[Task]]("tasks", (caseSteps, authContext) => caseSteps.tasks(authContext)) ) override val publicProperties: PublicProperties = diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index 0ca4baed6a..7576ab5a8b 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -335,12 +335,14 @@ object Conversion { implicit class InputObservableOps(inputObservable: InputObservable) { - def toObservable: Observable = + def toObservable(relatedId: EntityId, organisationIds: EntityId*): Observable = inputObservable .into[Observable] .withFieldComputed(_.tlp, _.tlp.getOrElse(2)) .withFieldComputed(_.ioc, _.ioc.getOrElse(false)) .withFieldComputed(_.sighted, _.sighted.getOrElse(false)) + .withFieldConst(_.organisationIds, organisationIds) + .withFieldConst(_.relatedId, relatedId) .transform } diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala index c6330b4481..43bad38ea3 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala @@ -36,6 +36,7 @@ class ObservableCtrl @Inject() ( override val db: Database, observableSrv: ObservableSrv, observableTypeSrv: ObservableTypeSrv, + organisationSrv: OrganisationSrv, caseSrv: CaseSrv, attachmentSrv: AttachmentSrv, errorHandler: ErrorHandler, @@ -64,15 +65,19 @@ class ObservableCtrl @Inject() ( .can(Permissions.manageObservable) .orFail(AuthorizationError("Operation not permitted")) observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) - } yield (case0, observableType) + organisation <- organisationSrv.current.getOrFail("Organisation") + } yield (case0, observableType, organisation) } .map { - case (case0, observableType) => - val (successes, failures) = inputAttachObs - .flatMap { obs => - obs.attachment.map(createAttachmentObservable(case0, obs, observableType, _)) ++ - obs.data.map(createSimpleObservable(case0, obs, observableType, _)) - } + case (case0, observableType, organisation) => + val successesAndFailures = + if (observableType.isAttachment) + inputAttachObs + .flatMap(obs => obs.attachment.map(createAttachmentObservable(organisation, case0, obs, observableType, _))) + else + inputAttachObs + .flatMap(obs => obs.data.map(createSimpleObservable(organisation, case0, obs, observableType, _))) + val (successes, failures) = successesAndFailures .foldLeft[(Seq[JsValue], Seq[JsValue])]((Nil, Nil)) { case ((s, f), Right(o)) => (s :+ o, f) case ((s, f), Left(o)) => (s, f :+ o) @@ -83,6 +88,7 @@ class ObservableCtrl @Inject() ( } def createSimpleObservable( + organisation: Organisation with Entity, `case`: Case with Entity, inputObservable: InputObservable, observableType: ObservableType with Entity, @@ -91,7 +97,7 @@ class ObservableCtrl @Inject() ( db .tryTransaction { implicit graph => observableSrv - .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .create(inputObservable.toObservable(organisation._id), observableType, data, inputObservable.tags, Nil) .flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) } match { case Success(o) => Right(o.toJson) @@ -99,6 +105,7 @@ class ObservableCtrl @Inject() ( } def createAttachmentObservable( + organisation: Organisation with Entity, `case`: Case with Entity, inputObservable: InputObservable, observableType: ObservableType with Entity, @@ -107,11 +114,11 @@ class ObservableCtrl @Inject() ( db .tryTransaction { implicit graph => val observable = fileOrAttachment match { - case Left(file) => observableSrv.create(inputObservable.toObservable, observableType, file, inputObservable.tags, Nil) + case Left(file) => observableSrv.create(inputObservable.toObservable(organisation._id), observableType, file, inputObservable.tags, Nil) case Right(attachment) => for { attach <- attachmentSrv.duplicate(attachment.name, attachment.contentType, attachment.id) - obs <- observableSrv.create(inputObservable.toObservable, observableType, attach, inputObservable.tags, Nil) + obs <- observableSrv.create(inputObservable.toObservable(organisation._id), observableType, attach, inputObservable.tags, Nil) } yield obs } observable.flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) @@ -332,5 +339,6 @@ class PublicObservable @Inject() ( .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly) .property("attachment.contentType", UMapping.string.optional)(_.select(_.attachments.value(_.contentType)).readonly) .property("attachment.id", UMapping.string.optional)(_.select(_.attachments.value(_.attachmentId)).readonly) + .property("relatedId", UMapping.entityId)(_.field.readonly) .build } diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index f3afbd6ca0..a4f81dbb79 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -1,6 +1,5 @@ package org.thp.thehive.controllers.v0 -import javax.inject.{Inject, Named, Provider, Singleton} import org.scalactic.Good import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{FObject, Field, FieldsParser} @@ -9,15 +8,16 @@ import org.thp.scalligraph.query._ import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.utils.RichType -import org.thp.scalligraph.{BadRequestError, EntityIdOrName, GlobalQueryExecutor} -import org.thp.thehive.models.{Alert, Case, CaseTemplate, Log, Observable, Task} -import org.thp.thehive.services.CaseOps._ -import org.thp.thehive.services.LogOps._ +import org.thp.scalligraph.{BadRequestError, EntityId, EntityIdOrName, GlobalQueryExecutor} +import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ +import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CaseTemplateOps._ +import org.thp.thehive.services.LogOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.TaskOps._ +import javax.inject.{Inject, Provider, Singleton} import scala.reflect.runtime.{universe => ru} case class OutputParam(from: Long, to: Long, withStats: Boolean, withParents: Int) @@ -134,16 +134,21 @@ class ParentIdInputFilter(parentType: String, parentId: String) extends InputQue .asInstanceOf[Traversal.V[Task]] .filter(_.`case`.get(EntityIdOrName(parentId))) .asInstanceOf[Traversal.Unk] - case t if t <:< ru.typeOf[Observable] && parentType == "alert" => - traversal - .asInstanceOf[Traversal.V[Observable]] - .filter(_.alert.get(EntityIdOrName(parentId))) - .asInstanceOf[Traversal.Unk] case t if t <:< ru.typeOf[Observable] => traversal .asInstanceOf[Traversal.V[Observable]] - .filter(_.`case`.get(EntityIdOrName(parentId))) + .has(_.relatedId, EntityId(parentId)) .asInstanceOf[Traversal.Unk] +// && parentType == "alert" => +// traversal +// .asInstanceOf[Traversal.V[Observable]] +// .filter(_.alert.get(EntityIdOrName(parentId))) +// .asInstanceOf[Traversal.Unk] +// case t if t <:< ru.typeOf[Observable] => +// traversal +// .asInstanceOf[Traversal.V[Observable]] +// .filter(_.`case`.get(EntityIdOrName(parentId))) +// .asInstanceOf[Traversal.Unk] case t if t <:< ru.typeOf[Log] => traversal .asInstanceOf[Traversal.V[Log]] diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index a9f4b94b08..b9c9b57529 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -1,6 +1,6 @@ package org.thp.thehive.controllers.v1 -import javax.inject.{Inject, Named, Singleton} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} @@ -13,11 +13,13 @@ import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.CaseTemplateOps._ +import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services._ import play.api.mvc.{Action, AnyContent, Results} +import javax.inject.{Inject, Singleton} import scala.util.{Success, Try} @Singleton @@ -26,6 +28,7 @@ class CaseCtrl @Inject() ( properties: Properties, caseSrv: CaseSrv, caseTemplateSrv: CaseTemplateSrv, + observableSrv: ObservableSrv, userSrv: UserSrv, tagSrv: TagSrv, organisationSrv: OrganisationSrv, @@ -58,7 +61,11 @@ class CaseCtrl @Inject() ( override val outputQuery: Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => caseSteps.richCase(authContext)) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query[Traversal.V[Case], Traversal.V[Task]]("tasks", (caseSteps, authContext) => caseSteps.tasks(authContext)), - Query[Traversal.V[Case], Traversal.V[Observable]]("observables", (caseSteps, authContext) => caseSteps.observables(authContext)), + Query[Traversal.V[Case], Traversal.V[Observable]]( + "observables", + (caseSteps, authContext) => + observableSrv.startTraversal(caseSteps.graph).has(_.relatedId, P.within(caseSteps._id.toSeq: _*)).visible(authContext) + ), Query[Traversal.V[Case], Traversal.V[User]]("assignableUsers", (caseSteps, authContext) => caseSteps.assignableUsers(authContext)), Query[Traversal.V[Case], Traversal.V[Organisation]]("organisations", (caseSteps, authContext) => caseSteps.organisations.visible(authContext)), Query[Traversal.V[Case], Traversal.V[Alert]]("alerts", (caseSteps, authContext) => caseSteps.alert.visible(authContext)) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala index 533e9d41dd..7f8164a233 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Conversion.scala @@ -330,12 +330,14 @@ object Conversion { ) implicit class InputObservableOps(inputObservable: InputObservable) { - def toObservable: Observable = + def toObservable(relatedId: EntityId, organisationIds: EntityId*): Observable = inputObservable .into[Observable] .withFieldComputed(_.ioc, _.ioc.getOrElse(false)) .withFieldComputed(_.sighted, _.sighted.getOrElse(false)) .withFieldComputed(_.tlp, _.tlp.getOrElse(2)) + .withFieldConst(_.organisationIds, organisationIds) + .withFieldConst(_.relatedId, relatedId) .transform } implicit val observableOutput: Renderer.Aux[RichObservable, OutputObservable] = Renderer.toJson[RichObservable, OutputObservable](richObservable => diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index d5581c8e17..ac353cda3b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -1,8 +1,5 @@ package org.thp.thehive.controllers.v1 -import java.io.FilterInputStream -import java.nio.file.Files -import javax.inject.{Inject, Named, Singleton} import net.lingala.zip4j.ZipFile import net.lingala.zip4j.model.FileHeader import org.thp.scalligraph._ @@ -27,7 +24,7 @@ import play.api.{Configuration, Logger} import java.io.FilterInputStream import java.nio.file.Files -import javax.inject.{Inject, Named, Singleton} +import javax.inject.{Inject, Singleton} import scala.collection.JavaConverters._ import scala.util.{Failure, Success} @@ -105,15 +102,19 @@ class ObservableCtrl @Inject() ( .can(Permissions.manageObservable) .orFail(AuthorizationError("Operation not permitted")) observableType <- observableTypeSrv.getOrFail(EntityName(inputObservable.dataType)) - } yield (case0, observableType) + organisation <- organisationSrv.current.getOrFail("Organisation") + } yield (case0, observableType, organisation) } .map { - case (case0, observableType) => - val (successes, failures) = inputAttachObs - .flatMap { obs => - obs.attachment.map(createAttachmentObservable(case0, obs, observableType, _)) ++ - obs.data.map(createSimpleObservable(case0, obs, observableType, _)) - } + case (case0, observableType, organisation) => + val successesAndFailures = + if (observableType.isAttachment) + inputAttachObs + .flatMap(obs => obs.attachment.map(createAttachmentObservable(organisation, case0, obs, observableType, _))) + else + inputAttachObs + .flatMap(obs => obs.data.map(createSimpleObservable(organisation, case0, obs, observableType, _))) + val (successes, failures) = successesAndFailures .foldLeft[(Seq[JsValue], Seq[JsValue])]((Nil, Nil)) { case ((s, f), Right(o)) => (s :+ o, f) case ((s, f), Left(o)) => (s, f :+ o) @@ -124,6 +125,7 @@ class ObservableCtrl @Inject() ( } def createSimpleObservable( + organisation: Organisation with Entity, `case`: Case with Entity, inputObservable: InputObservable, observableType: ObservableType with Entity, @@ -132,7 +134,7 @@ class ObservableCtrl @Inject() ( db .tryTransaction { implicit graph => observableSrv - .create(inputObservable.toObservable, observableType, data, inputObservable.tags, Nil) + .create(inputObservable.toObservable(organisation._id), observableType, data, inputObservable.tags, Nil) .flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) } match { case Success(o) => Right(o.toJson) @@ -140,6 +142,7 @@ class ObservableCtrl @Inject() ( } def createAttachmentObservable( + organisation: Organisation with Entity, `case`: Case with Entity, inputObservable: InputObservable, observableType: ObservableType with Entity, @@ -148,11 +151,11 @@ class ObservableCtrl @Inject() ( db .tryTransaction { implicit graph => val observable = fileOrAttachment match { - case Left(file) => observableSrv.create(inputObservable.toObservable, observableType, file, inputObservable.tags, Nil) + case Left(file) => observableSrv.create(inputObservable.toObservable(organisation._id), observableType, file, inputObservable.tags, Nil) case Right(attachment) => for { attach <- attachmentSrv.duplicate(attachment.name, attachment.contentType, attachment.id) - obs <- observableSrv.create(inputObservable.toObservable, observableType, attach, inputObservable.tags, Nil) + obs <- observableSrv.create(inputObservable.toObservable(organisation._id), observableType, attach, inputObservable.tags, Nil) } yield obs } observable.flatMap(o => caseSrv.addObservable(`case`, o).map(_ => o)) diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index ae4d2715ce..c0e2d39377 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -17,8 +17,19 @@ case class ObservableData() @BuildEdgeEntity[Observable, Tag] case class ObservableTag() +@DefineIndex(IndexType.standard, "organisationIds", "relatedId", "tlp", "ioc", "sighted", "ignoreSimilarity") +@DefineIndex(IndexType.fulltext, "message") @BuildVertexEntity -case class Observable(message: Option[String], tlp: Int, ioc: Boolean, sighted: Boolean, ignoreSimilarity: Option[Boolean]) +// TODO Add data and dataType +case class Observable( + message: Option[String], + tlp: Int, + ioc: Boolean, + sighted: Boolean, + ignoreSimilarity: Option[Boolean], + organisationIds: Seq[EntityId], + relatedId: EntityId +) case class RichObservable( observable: Observable with Entity, diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index a7982718d7..ca1301d1a4 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -182,6 +182,7 @@ class ShareSrv @Inject() (implicit for { share <- get(`case`, organisation._id).getOrFail("Case") _ <- shareObservableSrv.create(ShareObservable(), share, richObservable.observable) + _ <- observableSrv.get(richObservable.observable).addValue(_.organisationIds, organisation._id).getOrFail("Observable") _ <- auditSrv.observable.create(richObservable.observable, richObservable.toJson) } yield () @@ -193,12 +194,23 @@ class ShareSrv @Inject() (implicit def shareCaseObservables( share: Share with Entity )(implicit graph: Graph, authContext: AuthContext): Try[Seq[ShareObservable with Entity]] = - get(share) - .`case` - .observables - .filter(_.shares.has(T.id, P.neq(share._id))) - .toIterator - .toTry(shareObservableSrv.create(ShareObservable(), share, _)) + for { + organisation <- get(share).organisation.getOrFail("Share") + shareObservables <- + get(share) + .`case` + .observables // list observables related to authContext + .filter(_.shares.has(T.id, P.neq(share._id))) + .toIterator + .toTry { obs => + observableSrv + .get(obs) + .addValue(_.organisationIds, organisation._id) + .getOrFail("Observable") + .flatMap(_ => shareObservableSrv.create(ShareObservable(), share, obs)) + + } + } yield shareObservables /** * Does a full rebuild of the share status of a task, @@ -273,6 +285,7 @@ class ShareSrv @Inject() (implicit case0 <- observableSrv.get(observable).`case`.getOrFail("Observable") share <- caseSrv.get(case0).share(organisation._id).getOrFail("Case") _ <- shareObservableSrv.create(ShareObservable(), share, observable) + _ <- observableSrv.get(observable).addValue(_.organisationIds, organisation._id).getOrFail("Observable") _ <- auditSrv.share.shareObservable(observable, case0, organisation) } yield () } @@ -299,11 +312,11 @@ class ShareSrv @Inject() (implicit case ((toAdd, toRemove), o) if toAdd.contains(o) => (toAdd - o, toRemove) case ((toAdd, toRemove), o) => (toAdd, toRemove + o) } - orgsToRemove.foreach(o => observableSrv.get(observable).share(o._id).remove()) + orgsToRemove.foreach(o => observableSrv.get(observable).removeValue(_.organisationIds, o._id).share(o._id).remove()) orgsToAdd .toTry { organisation => for { - case0 <- observableSrv.get(observable).`case`.getOrFail("Observable") + case0 <- observableSrv.get(observable).addValue(_.organisationIds, organisation._id).`case`.getOrFail("Observable") share <- caseSrv.get(case0).share(organisation._id).getOrFail("Case") _ <- shareObservableSrv.create(ShareObservable(), share, observable) _ <- auditSrv.share.shareObservable(observable, case0, organisation) diff --git a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala index fd54854aea..2948d16ccb 100644 --- a/thehive/test/org/thp/thehive/services/AlertSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/AlertSrvTest.scala @@ -1,8 +1,7 @@ package org.thp.thehive.services import java.util.Date - -import org.thp.scalligraph.{EntityIdOrName, EntityName} +import org.thp.scalligraph.{EntityId, EntityIdOrName, EntityName} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -102,30 +101,40 @@ class AlertSrvTest extends PlaySpecification with TestAppBuilder { } "add an observable if not existing" in testApp { app => - val similarObs = app[Database].tryTransaction { implicit graph => - for { - observableType <- app[ObservableTypeSrv].getOrFail(EntityName("domain")) - observable <- app[ObservableSrv].create( - observable = Observable(Some("if you are lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None), - `type` = observableType, - dataValue = "perdu.com", - tagNames = Set("tag10"), - extensions = Nil - ) - } yield observable - }.get + def similarObs(alertId: EntityId) = + app[Database].tryTransaction { implicit graph => + for { + organisation <- app[OrganisationSrv].getOrFail(EntityName("cert")) + observableType <- app[ObservableTypeSrv].getOrFail(EntityName("domain")) + observable <- app[ObservableSrv].create( + observable = Observable( + Some("if you are lost"), + 1, + ioc = false, + sighted = true, + ignoreSimilarity = None, + organisationIds = Seq(organisation._id), + relatedId = alertId + ), + `type` = observableType, + dataValue = "perdu.com", + tagNames = Set("tag10"), + extensions = Nil + ) + } yield observable + }.get app[Database].tryTransaction { implicit graph => for { alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref4")) - _ <- app[AlertSrv].addObservable(alert, similarObs) + _ <- app[AlertSrv].addObservable(alert, similarObs(alert._id)) } yield () } must beASuccessfulTry app[Database].tryTransaction { implicit graph => for { alert <- app[AlertSrv].getOrFail(EntityName("testType;testSource;ref1")) - _ <- app[AlertSrv].addObservable(alert, similarObs) + _ <- app[AlertSrv].addObservable(alert, similarObs(alert._id)) } yield () } must beASuccessfulTry diff --git a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala index a8019d9ae3..c7b5a44149 100644 --- a/thehive/test/org/thp/thehive/services/CaseSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseSrvTest.scala @@ -281,8 +281,17 @@ class CaseSrvTest extends PlaySpecification with TestAppBuilder { }.get must throwA[CreateError] val newObs = app[Database].tryTransaction { implicit graph => + val organisation = app[OrganisationSrv].current.getOrFail("Organisation").get app[ObservableSrv].create( - Observable(Some("if you feel lost"), 1, ioc = false, sighted = true, ignoreSimilarity = None), + Observable( + Some("if you feel lost"), + 1, + ioc = false, + sighted = true, + ignoreSimilarity = None, + organisationIds = Seq(organisation._id), + c1._id + ), app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Case").get, "lost.com", Set[String](), diff --git a/thehive/test/org/thp/thehive/services/DataSrvTest.scala b/thehive/test/org/thp/thehive/services/DataSrvTest.scala index 922926dd91..73cedc90d7 100644 --- a/thehive/test/org/thp/thehive/services/DataSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/DataSrvTest.scala @@ -1,6 +1,6 @@ package org.thp.thehive.services -import org.thp.scalligraph.EntityName +import org.thp.scalligraph.{EntityId, EntityName} import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -21,8 +21,9 @@ class DataSrvTest extends PlaySpecification with TestAppBuilder { "get related observables" in testApp { app => app[Database].tryTransaction { implicit graph => + val organisation = app[OrganisationSrv].current.getOrFail("Organisation").get app[ObservableSrv].create( - Observable(Some("love"), 1, ioc = false, sighted = true, ignoreSimilarity = None), + Observable(Some("love"), 1, ioc = false, sighted = true, ignoreSimilarity = None, organisationIds = Seq(organisation._id), EntityId("")), app[ObservableTypeSrv].get(EntityName("domain")).getOrFail("Observable").get, "love.com", Set("tagX"),