diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala index b6fdd9b9a9..aea10bd44f 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala @@ -47,6 +47,7 @@ object Conversion { .withFieldComputed(_.id, _._id.toString) .withFieldConst(_._type, "case_artifact_job") .withFieldConst(_.case_artifact, None) + .withFieldComputed(_.operations, a => JsArray(a.operations).toString) .enableMethodAccessors .transform ) @@ -80,6 +81,7 @@ object Conversion { Some(observableWithExtraOutput.toValue((richObservable, JsObject.empty, Some(Left(richCase))))) } ) + .withFieldComputed(_.operations, a => JsArray(a.operations).toString) .enableMethodAccessors .transform } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala index 60ca038fa7..b18a45cf86 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala @@ -29,7 +29,8 @@ case class Job( endDate: Date, // end date of the job or if it is not finished date of the last check report: Option[JsObject], cortexId: String, - cortexJobId: String + cortexJobId: String, + operations: Seq[JsObject] ) case class RichJob( @@ -50,5 +51,5 @@ case class RichJob( def report: Option[JsObject] = job.report def cortexId: String = job.cortexId def cortexJobId: String = job.cortexJobId - + def operations: Seq[JsObject] = job.operations } diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala index 320fc9e1a3..a702511b6f 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala @@ -96,7 +96,7 @@ class ActionSrv @Inject() ( job.report.flatMap(_.full), client.name, job.id, - job.report.fold[Seq[JsObject]](Nil)(_.operations) + Nil ) createdAction <- Future.fromTry { db.tryTransaction { implicit graph => diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala index 519644bce9..3a21524d16 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala @@ -42,6 +42,7 @@ class JobSrv @Inject() ( observableTypeSrv: ObservableTypeSrv, attachmentSrv: AttachmentSrv, reportTagSrv: ReportTagSrv, + actionOperationSrv: ActionOperationSrv, serviceHelper: ServiceHelper, auditSrv: CortexAuditSrv, organisationSrv: OrganisationSrv, @@ -130,6 +131,7 @@ class JobSrv @Inject() ( .withFieldConst(_.report, None) .withFieldConst(_.cortexId, "tbd") .withFieldComputed(_.cortexJobId, _.id) + .withFieldConst(_.operations, Nil) .transform /** @@ -167,11 +169,31 @@ class JobSrv @Inject() ( .availableCortexClients(connector.clients, authContext.organisation) .find(_.name == cortexId) .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful) - job <- Future.fromTry(updateJobStatus(jobId, cortexJob)) - _ <- importCortexArtifacts(job, cortexJob, cortexClient) - _ <- Future.fromTry(importAnalyzerTags(job, cortexJob)) + operations <- Future.fromTry(executeOperations(jobId, cortexJob)) + job <- Future.fromTry(updateJobStatus(jobId, cortexJob, operations)) + _ <- importCortexArtifacts(job, cortexJob, cortexClient) + _ <- Future.fromTry(importAnalyzerTags(job, cortexJob)) } yield job + def executeOperations(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Seq[ActionOperationStatus]] = + db.tryTransaction { implicit graph => + get(jobId) + .observable + .project(_.by.by(_.`case`.option)) + .getOrFail("Observable") + .map { + case (relatedObservable, relatedCase) => + cortexJob + .report + .fold[Seq[ActionOperation]](Nil)(_.operations.map(_.as[ActionOperation])) + .map { operation => + actionOperationSrv + .execute(relatedObservable, operation, relatedCase, None) + .fold(t => ActionOperationStatus(operation, success = false, t.getMessage), identity) + } + } + } + /** * Update job status, set the endDate and remove artifacts from report * @@ -180,7 +202,9 @@ class JobSrv @Inject() ( * @param authContext the authentication context * @return the updated job */ - private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Job with Entity] = + private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob, operations: Seq[ActionOperationStatus])(implicit + authContext: AuthContext + ): Try[Job with Entity] = db.tryTransaction { implicit graph => getOrFail(jobId).flatMap { job => val report = cortexJob.report.flatMap(r => r.full orElse r.errorMessage.map(m => Json.obj("errorMessage" -> m))) @@ -193,6 +217,7 @@ class JobSrv @Inject() ( .update(_.endDate, endDate) .update(_._updatedAt, Some(new Date)) .update(_._updatedBy, Some(authContext.userId)) + .update(_.operations, operations.map(o => Json.toJsObject(o))) .getOrFail("Job") observable <- get(job).observable.getOrFail("Observable") _ <- diff --git a/cortex/connector/src/test/resources/cortex-jobs.json b/cortex/connector/src/test/resources/cortex-jobs.json index 120d64cad7..99fce56568 100644 --- a/cortex/connector/src/test/resources/cortex-jobs.json +++ b/cortex/connector/src/test/resources/cortex-jobs.json @@ -87,12 +87,21 @@ { "data": "192.168.1.1", "message": "myIp", - "tags": [], + "tags": ["tag-test"], "tlp": 2, "dataType": "ip" } ], "operations": [ + { + "type": "AddArtifactToCase", + "data": "myData", + "dataType": "other", + "message": "test-operation", + "tlp": 3, + "ignoreSimilarity": false, + "tags": ["tag1", "tag2"] + } ] }, "tlp": 2, diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala index 1e059b90aa..87a1efdbb2 100644 --- a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala +++ b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala @@ -7,9 +7,10 @@ import org.thp.scalligraph.models.{Database, DummyUserSrv, Schema} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{AppBuilder, EntityName} import org.thp.thehive.TestAppBuilder -import org.thp.thehive.connector.cortex.models.{Job, JobStatus, TheHiveCortexSchemaProvider} +import org.thp.thehive.connector.cortex.models.{ActionOperationStatus, Job, JobStatus, TheHiveCortexSchemaProvider} import org.thp.thehive.connector.cortex.services.JobOps._ import org.thp.thehive.models.Permissions +import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services._ @@ -23,7 +24,8 @@ import scala.concurrent.duration.DurationInt import scala.io.Source class JobSrvTest extends PlaySpecification with TestAppBuilder { - implicit val authContext: AuthContext = DummyUserSrv(userId = "admin@thehive.local", permissions = Permissions.all).authContext + implicit val authContext: AuthContext = + DummyUserSrv(userId = "certuser@thehive.local", organisation = "cert", permissions = Permissions.all).authContext override def appConfigure: AppBuilder = super .appConfigure @@ -33,53 +35,76 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder { .`override`(_.bindToProvider[Schema, TheHiveCortexSchemaProvider]) "job service" should { + val cortexOutputJob = { + val dataSource = Source.fromResource("cortex-jobs.json") + val data = dataSource.mkString + dataSource.close() + Json.parse(data).as[List[OutputJob]].find(_.id == "ZWu85Q1OCVNx03hXK4df").get + } + "handle creation and then finished job" in testApp { app => -// val job = Job( -// workerId = "anaTest2", -// workerName = "anaTest2", -// workerDefinition = "test2", -// status = JobStatus.Waiting, -// startDate = new Date(1561625908856L), -// endDate = new Date(1561625908856L), -// report = None, -// cortexId = "test", -// cortexJobId = "LVyYKFstq3Rtrdc9DFmL" -// ) -// -// val cortexOutputJob = { -// val dataSource = Source.fromResource("cortex-jobs.json") -// val data = dataSource.mkString -// dataSource.close() -// Json.parse(data).as[List[OutputJob]].find(_.id == "ZWu85Q1OCVNx03hXK4df").get -// } -// -// val createdJobTry = app[Database].tryTransaction { implicit graph => -// for { -// observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable") -// createdJob <- app[JobSrv].create(job, observable) -// } yield createdJob -// } -// createdJobTry.map { createdJob => -// Await.result(app[JobSrv].finished(app[CortexClient].name, createdJob._id, cortexOutputJob), 20.seconds) -// } must beASuccessfulTry.which { updatedJob => -// updatedJob.status shouldEqual JobStatus.Success -// updatedJob.report must beSome -// (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg" -// -// app[Database].roTransaction { implicit graph => -// app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue -// app[JobSrv].get(updatedJob).reportObservables.toList.length must equalTo(2).updateMessage { s => -// s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}" -// } -// -// for { -// audit <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit") -// organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation") -// user <- app[UserSrv].startTraversal.getByName("certuser@thehive.local").getOrFail("User") -// } yield new JobFinished().filter(audit, Some(updatedJob), organisation, Some(user)) -// } must beASuccessfulTry(true) -// } - pending("flaky test") + val job = Job( + workerId = "anaTest2", + workerName = "anaTest2", + workerDefinition = "test2", + status = JobStatus.Waiting, + startDate = new Date(1561625908856L), + endDate = new Date(1561625908856L), + report = None, + cortexId = "test", + cortexJobId = "LVyYKFstq3Rtrdc9DFmL", + operations = Nil + ) + + val createdJobTry = app[Database].tryTransaction { implicit graph => + for { + observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable") + createdJob <- app[JobSrv].create(job, observable) + } yield createdJob + } + val finishedJobTry = createdJobTry.map { createdJob => + Await.result(app[JobSrv].finished(app[CortexClient].name, createdJob._id, cortexOutputJob), 20.seconds) + } + finishedJobTry must beASuccessfulTry + val updatedJob = finishedJobTry.get + updatedJob.status shouldEqual JobStatus.Success + updatedJob.report must beSome + (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg" + updatedJob.operations must haveSize(1) + updatedJob.operations.map(o => (o \ "status").as[String]) must contain(beEqualTo("Success")) + .forall + .updateMessage(s => s"$s\nOperation has failed: ${updatedJob.operations.map("\n -" + _).mkString}") + + app[Database].roTransaction { implicit graph => + app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue + val reportObservables = app[JobSrv].get(updatedJob).reportObservables.toSeq + reportObservables.length must equalTo(2).updateMessage { s => + s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}" + } + val ipObservable = reportObservables.find(_.dataType == "ip").get + ipObservable.data must beSome("192.168.1.1") + ipObservable.message must beSome("myIp") + ipObservable.tags must contain(exactly("tag-test")) + ipObservable.tlp must beEqualTo(2) + + val operationObservableMaybe = app[JobSrv] + .get(updatedJob) + .observable + .`case` + .observables + .has(_.message, "test-operation") + .headOption + operationObservableMaybe must beSome.which { operationObservable => + operationObservable.data must beSome("myData") + operationObservable.tlp must beEqualTo(3) + operationObservable.tags must contain(exactly("tag1", "tag2")) + } + for { + audit <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit") + organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation") + user <- app[UserSrv].startTraversal.getByName("certadmin@thehive.local").getOrFail("User") + } yield JobFinished.filter(audit, Some(updatedJob), organisation, Some(user)) + } must beASuccessfulTry(true).setMessage("The audit doesn't match the expected criteria") } "submit a job" in testApp { app => diff --git a/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala b/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala index d3e04c6403..1f02266efe 100644 --- a/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala +++ b/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala @@ -17,7 +17,8 @@ case class OutputJob( cortexId: String, cortexJobId: String, id: String, - case_artifact: Option[OutputObservable] + case_artifact: Option[OutputObservable], + operations: String ) object OutputJob { diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index 34bfc3da95..17b7ca36d0 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -479,7 +479,8 @@ trait Conversion { endDate, report, cortexId, - cortexJobId + cortexJobId, + Nil ) ) }