Merge branch 'feature/traversal-stream' into develop-th4

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala

@@ -61,7 +61,7 @@ class ActionCtrl @Inject() (
     FieldsParser[OutputParam],
     (range, actionSteps, _) => actionSteps.richPage(range.from, range.to, withTotal = true)(_.richAction)
   )
-  override val outputQuery: Query = Query.output[RichAction]()
+  override val outputQuery: Query = Query.output[RichAction, ActionSteps](_.richAction)
 
   def create: Action[AnyContent] =
     entrypoint("create action")

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala

@@ -2,12 +2,6 @@ package org.thp.thehive.connector.cortex.controllers.v0
 
 import java.util.zip.ZipFile
 
-import scala.util.{Failure, Success}
-
-import play.api.Logger
-import play.api.libs.json.{JsFalse, JsObject, JsTrue}
-import play.api.mvc.{Action, AnyContent, Results}
-
 import javax.inject.{Inject, Singleton}
 import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser}
 import org.thp.scalligraph.models.{Database, Entity}
@@ -21,6 +15,11 @@ import org.thp.thehive.connector.cortex.services.{AnalyzerTemplateSrv, AnalyzerT
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.controllers.v0.{IdOrName, OutputParam, QueryableCtrl}
 import org.thp.thehive.models.Permissions
+import play.api.Logger
+import play.api.libs.json.{JsFalse, JsObject, JsTrue}
+import play.api.mvc.{Action, AnyContent, Results}
+
+import scala.util.{Failure, Success}
 
 @Singleton
 class AnalyzerTemplateCtrl @Inject() (
@@ -45,7 +44,7 @@ class AnalyzerTemplateCtrl @Inject() (
     FieldsParser[OutputParam],
     (range, AnalyzerTemplateSteps, _) => AnalyzerTemplateSteps.page(range.from, range.to, withTotal = true)
   )
-  override val outputQuery: Query = Query.output[AnalyzerTemplate with Entity]()
+  override val outputQuery: Query = Query.output[AnalyzerTemplate with Entity]
 
   def get(id: String): Action[AnyContent] =
     entrypoint("get content")

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala

@@ -2,7 +2,7 @@ package org.thp.thehive.connector.cortex.controllers.v0
 
 import io.scalaland.chimney.dsl._
 import org.thp.cortex.dto.v0.{OutputWorker => CortexWorker}
-import org.thp.scalligraph.controllers.Outputer
+import org.thp.scalligraph.controllers.Renderer
 import org.thp.scalligraph.models.Entity
 import org.thp.thehive.connector.cortex.dto.v0.{InputAnalyzerTemplate, OutputAction, OutputAnalyzerTemplate, OutputJob, OutputWorker}
 import org.thp.thehive.connector.cortex.models._
@@ -11,7 +11,7 @@ import play.api.libs.json.{JsArray, JsFalse, Json}
 object Conversion {
   import org.thp.thehive.controllers.v0.Conversion._
 
-  implicit val actionOutput: Outputer.Aux[RichAction, OutputAction] = Outputer[RichAction, OutputAction](
+  implicit val actionOutput: Renderer.Aux[RichAction, OutputAction] = Renderer.json[RichAction, OutputAction](
     _.into[OutputAction]
       .withFieldRenamed(_.workerId, _.responderId)
       .withFieldRenamed(_.workerName, _.responderName)
@@ -24,7 +24,7 @@ object Conversion {
       .transform
   )
 
-  implicit val jobOutput: Outputer.Aux[RichJob, OutputJob] = Outputer[RichJob, OutputJob](job =>
+  implicit val jobOutput: Renderer.Aux[RichJob, OutputJob] = Renderer.json[RichJob, OutputJob](job =>
     job
       .into[OutputJob]
       .withFieldComputed(_.analyzerId, _.workerId)
@@ -46,8 +46,8 @@ object Conversion {
       .transform
   )
 
-  implicit val analyzerTemplateOutput: Outputer.Aux[AnalyzerTemplate with Entity, OutputAnalyzerTemplate] =
-    Outputer[AnalyzerTemplate with Entity, OutputAnalyzerTemplate](at =>
+  implicit val analyzerTemplateOutput: Renderer.Aux[AnalyzerTemplate with Entity, OutputAnalyzerTemplate] =
+    Renderer.json[AnalyzerTemplate with Entity, OutputAnalyzerTemplate](at =>
       at.asInstanceOf[AnalyzerTemplate]
         .into[OutputAnalyzerTemplate]
         .withFieldComputed(_.analyzerId, _.workerId)
@@ -65,8 +65,8 @@ object Conversion {
         .transform
   }
 
-  implicit val workerOutput: Outputer.Aux[(CortexWorker, Seq[String]), OutputWorker] =
-    Outputer[(CortexWorker, Seq[String]), OutputWorker](worker =>
+  implicit val workerOutput: Renderer.Aux[(CortexWorker, Seq[String]), OutputWorker] =
+    Renderer.json[(CortexWorker, Seq[String]), OutputWorker](worker =>
       worker
         ._1
         .into[OutputWorker]

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala

@@ -44,7 +44,7 @@ class JobCtrl @Inject() (
     FieldsParser[OutputParam],
     (range, jobSteps, authContext) => jobSteps.richPage(range.from, range.to, withTotal = true)(_.richJob(authContext))
   )
-  override val outputQuery: Query = Query.output[RichJob]()
+  override val outputQuery: Query = Query.outputWithContext[RichJob, JobSteps]((jobSteps, authContext) => jobSteps.richJob(authContext))
 
   def get(jobId: String): Action[AnyContent] =
     entrypoint("get job")

cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala

@@ -2,24 +2,23 @@ package org.thp.thehive.connector.cortex.services
 
 import java.util.Date
 
-import scala.concurrent.Await
-import scala.concurrent.duration.DurationInt
-import scala.io.Source
-
-import play.api.libs.json.Json
-import play.api.test.PlaySpecification
-
 import org.thp.cortex.client.{CortexClient, TestCortexClientProvider}
 import org.thp.cortex.dto.v0.OutputJob
 import org.thp.scalligraph.AppBuilder
 import org.thp.scalligraph.auth.AuthContext
-import org.thp.scalligraph.models.{Database, DummyUserSrv}
+import org.thp.scalligraph.models.{Database, DummyUserSrv, Schema}
 import org.thp.scalligraph.steps.StepsOps._
 import org.thp.thehive.TestAppBuilder
-import org.thp.thehive.connector.cortex.models.{Job, JobStatus}
+import org.thp.thehive.connector.cortex.models.{Job, JobStatus, TheHiveCortexSchemaProvider}
 import org.thp.thehive.models.Permissions
 import org.thp.thehive.services._
 import org.thp.thehive.services.notification.triggers.JobFinished
+import play.api.libs.json.Json
+import play.api.test.PlaySpecification
+
+import scala.concurrent.Await
+import scala.concurrent.duration.DurationInt
+import scala.io.Source
 
 class JobSrvTest extends PlaySpecification with TestAppBuilder {
   implicit val authContext: AuthContext = DummyUserSrv(userId = "[email protected]", permissions = Permissions.all).authContext
@@ -29,8 +28,8 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder {
       .bindActor[CortexActor]("cortex-actor")
       .bindToProvider[CortexClient, TestCortexClientProvider]
       .bind[Connector, TestConnector]
+      .`override`(_.bindToProvider[Schema, TheHiveCortexSchemaProvider])
 
-  //  def shutdownActorSystem(app: AppBuilder): Future[Terminated] = app.app.actorSystem.terminate()
   "job service" should {
     "handle creation and then finished job" in testApp { app =>
       val job = Job(
@@ -67,7 +66,9 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder {
 
         app[Database].roTransaction { implicit graph =>
           app[JobSrv].get(updatedJob).observable.has("message", "hello world").exists() must beTrue
-          app[JobSrv].get(updatedJob).reportObservables.toList.length must equalTo(2)
+          app[JobSrv].get(updatedJob).reportObservables.toList.length must equalTo(2).updateMessage { s =>
+            s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}"
+          }
 
           for {
             audit        <- app[AuditSrv].initSteps.has("objectId", updatedJob._id).getOrFail()

frontend/app/scripts/services/api/OrganisationSrv.js

@@ -53,16 +53,13 @@
                     },
                     {
                         '_name': 'users'
-                    },
-                    {
-                        '_name': 'toList'
                     }
                 ], {
                     headers: {
                         'X-Organisation': orgId
                     }
                 }).then(function(response) {
-                    return $q.resolve(response.data.result);
+                    return $q.resolve(response.data);
                 });
             };
 
@@ -73,12 +70,9 @@
                     },
                     {
                         '_name': 'caseTemplates'
-                    },
-                    {
-                        '_name': 'toList'
                     }
                 ]).then(function(response) {
-                    return $q.resolve(response.data.result);
+                    return $q.resolve(response.data);
                 });
             };
         });

frontend/app/scripts/services/api/UserSrv.js

@@ -203,13 +203,9 @@
                     operations.push(query);
                 }
 
-                operations.push({
-                    '_name': 'toList'
-                });
-
                 return QuerySrv.query('v1', operations)
                     .then(function(response) {
-                        return $q.resolve(response.data.result);
+                        return $q.resolve(response.data);
                     });
             };
 

project/Dependencies.scala

@@ -9,7 +9,7 @@ object Dependencies {
   lazy val playLogback             = "com.typesafe.play"        %% "play-logback"                       % play.core.PlayVersion.current
   lazy val playGuice               = "com.typesafe.play"        %% "play-guice"                         % play.core.PlayVersion.current
   lazy val playFilters             = "com.typesafe.play"        %% "filters-helpers"                    % play.core.PlayVersion.current
-  lazy val playMockws              = "de.leanovate.play-mockws" %% "play-mockws"                        % "2.7.1" // FIXME play.core.PlayVersion.current
+  lazy val playMockws              = "de.leanovate.play-mockws" %% "play-mockws"                        % "2.8.0" // FIXME play.core.PlayVersion.current
   lazy val akkaCluster             = "com.typesafe.akka"        %% "akka-cluster"                       % akkaVersion
   lazy val akkaClusterTools        = "com.typesafe.akka"        %% "akka-cluster-tools"                 % akkaVersion
   lazy val akkaClusterTyped        = "com.typesafe.akka"        %% "akka-cluster-typed"                 % akkaVersion

thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala

@@ -2,13 +2,6 @@ package org.thp.thehive.controllers.v0
 
 import java.util.Base64
 
-import scala.collection.JavaConverters._
-import scala.util.{Failure, Success, Try}
-
-import play.api.Logger
-import play.api.libs.json.{JsArray, JsObject, Json}
-import play.api.mvc.{Action, AnyContent, Results}
-
 import gremlin.scala.{__, By, Graph, Key, StepLabel, Vertex}
 import io.scalaland.chimney.dsl._
 import javax.inject.{Inject, Singleton}
@@ -24,6 +17,12 @@ import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.{InputAlert, InputObservable, OutputSimilarCase}
 import org.thp.thehive.models._
 import org.thp.thehive.services._
+import play.api.Logger
+import play.api.libs.json.{JsArray, JsObject, Json}
+import play.api.mvc.{Action, AnyContent, Results}
+
+import scala.collection.JavaConverters._
+import scala.util.{Failure, Success, Try}
 
 @Singleton
 class AlertCtrl @Inject() (
@@ -60,22 +59,20 @@ class AlertCtrl @Inject() (
           richAlert -> alertSrv.get(richAlert.alert)(alertSteps.graph).observables.richObservable.toList
         }
   )
-  override val outputQuery: Query = Query.output[(RichAlert, Seq[RichObservable])]()
+  override val outputQuery: Query = Query.output[RichAlert, AlertSteps](_.richAlert)
   override val extraQueries: Seq[ParamQuery[_]] = Seq(
     Query[AlertSteps, CaseSteps]("cases", (alertSteps, _) => alertSteps.`case`),
     Query[AlertSteps, ObservableSteps]("observables", (alertSteps, _) => alertSteps.observables),
-    Query[AlertSteps, List[RichAlert]]("toList", (alertSteps, _) => alertSteps.richAlert.toList),
-    Query[AlertSteps, List[(RichAlert, Seq[RichObservable])]](
+    Query[AlertSteps, Traversal[(RichAlert, Seq[RichObservable]), (RichAlert, Seq[RichObservable])]](
       "withObservables",
       (alertSteps, _) =>
         alertSteps
           .richAlert
           .map { richAlert =>
             richAlert -> alertSrv.get(richAlert.alert)(alertSteps.graph).observables.richObservable.toList
           }
-          .toList
     ),
-    Query.output[RichAlert]()
+    Query.output[(RichAlert, Seq[RichObservable])]
   )
 
   def create: Action[AnyContent] =

thehive/app/org/thp/thehive/controllers/v0/AuditCtrl.scala

@@ -2,11 +2,6 @@ package org.thp.thehive.controllers.v0
 
 import java.util.Date
 
-import scala.util.Success
-
-import play.api.libs.json.{JsArray, JsObject, Json}
-import play.api.mvc.{Action, AnyContent, Results}
-
 import gremlin.scala.{By, Key}
 import javax.inject.{Inject, Singleton}
 import org.apache.tinkerpop.gremlin.process.traversal.Order
@@ -18,6 +13,10 @@ import org.thp.scalligraph.steps.StepsOps._
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.models.RichAudit
 import org.thp.thehive.services._
+import play.api.libs.json.{JsArray, JsObject, Json}
+import play.api.mvc.{Action, AnyContent, Results}
+
+import scala.util.Success
 
 @Singleton
 class AuditCtrl @Inject() (
@@ -37,22 +36,19 @@ class AuditCtrl @Inject() (
     (param, graph, authContext) => auditSrv.get(param.idOrName)(graph).visible(authContext)
   )
 
-  override val extraQueries: Seq[ParamQuery[_]] = Seq(
-    Query[AuditSteps, List[RichAudit]]("toList", (auditSteps, _) => auditSteps.richAudit.toList)
-  )
-  val entityName: String = "audit"
+  override val entityName: String = "audit"
 
-  val initialQuery: org.thp.scalligraph.query.Query =
+  override val initialQuery: Query =
     Query.init[AuditSteps]("listAudit", (graph, authContext) => auditSrv.initSteps(graph).visible(authContext))
-  val publicProperties: List[org.thp.scalligraph.query.PublicProperty[_, _]] = properties.audit ::: metaProperties[LogSteps]
+  override val publicProperties: List[org.thp.scalligraph.query.PublicProperty[_, _]] = properties.audit ::: metaProperties[LogSteps]
 
-  val pageQuery: org.thp.scalligraph.query.ParamQuery[org.thp.thehive.controllers.v0.OutputParam] =
+  override val pageQuery: ParamQuery[org.thp.thehive.controllers.v0.OutputParam] =
     Query.withParam[OutputParam, AuditSteps, PagedResult[RichAudit]](
       "page",
       FieldsParser[OutputParam],
       (range, auditSteps, _) => auditSteps.richPage(range.from, range.to, withTotal = true)(_.richAudit)
     )
-  val outputQuery: org.thp.scalligraph.query.Query = Query.output[RichAudit]()
+  override val outputQuery: Query = Query.output[RichAudit, AuditSteps](_.richAudit)
 
   def flow(caseId: Option[String], count: Option[Int]): Action[AnyContent] =
     entryPoint("audit flow")

thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala

@@ -1,11 +1,5 @@
 package org.thp.thehive.controllers.v0
 
-import scala.util.Success
-
-import play.api.Logger
-import play.api.libs.json.{JsArray, JsNumber, JsObject}
-import play.api.mvc.{Action, AnyContent, Results}
-
 import javax.inject.{Inject, Singleton}
 import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser}
 import org.thp.scalligraph.models.Database
@@ -17,6 +11,11 @@ import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.{InputCase, InputTask}
 import org.thp.thehive.models._
 import org.thp.thehive.services._
+import play.api.Logger
+import play.api.libs.json.{JsArray, JsNumber, JsObject}
+import play.api.mvc.{Action, AnyContent, Results}
+
+import scala.util.Success
 
 @Singleton
 class CaseCtrl @Inject() (
@@ -54,13 +53,10 @@ class CaseCtrl @Inject() (
           }
     }
   )
-  override val outputQuery: Query = Query.output[(RichCase, JsObject)]()
+  override val outputQuery: Query = Query.outputWithContext[RichCase, CaseSteps]((caseSteps, authContext) => caseSteps.richCase(authContext))
   override val extraQueries: Seq[ParamQuery[_]] = Seq(
-    Query[CaseSteps, List[RichCase]]("toList", (caseSteps, authContext) => caseSteps.richCase(authContext).toList),
-    Query.withParam[IdOrName, CaseSteps, CaseSteps]("getCase", FieldsParser[IdOrName], (idOrName, caseSteps, _) => caseSteps.get(idOrName.idOrName)),
     Query[CaseSteps, ObservableSteps]("observables", (caseSteps, authContext) => caseSteps.observables(authContext)),
-    Query[CaseSteps, TaskSteps]("tasks", (caseSteps, authContext) => caseSteps.tasks(authContext)),
-    Query.output[RichCase]()
+    Query[CaseSteps, TaskSteps]("tasks", (caseSteps, authContext) => caseSteps.tasks(authContext))
   )
 
   def create: Action[AnyContent] =

thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala

@@ -1,19 +1,18 @@
 package org.thp.thehive.controllers.v0
 
-import play.api.Logger
-import play.api.mvc.{Action, AnyContent, Results}
-
 import javax.inject.{Inject, Singleton}
+import org.thp.scalligraph.RichSeq
 import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser}
 import org.thp.scalligraph.models.Database
 import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperty, Query}
 import org.thp.scalligraph.steps.PagedResult
 import org.thp.scalligraph.steps.StepsOps._
-import org.thp.scalligraph.RichSeq
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.InputCaseTemplate
 import org.thp.thehive.models.{Permissions, RichCaseTemplate}
 import org.thp.thehive.services._
+import play.api.Logger
+import play.api.mvc.{Action, AnyContent, Results}
 
 @Singleton
 class CaseTemplateCtrl @Inject() (
@@ -41,10 +40,7 @@ class CaseTemplateCtrl @Inject() (
     FieldsParser[OutputParam],
     (range, caseTemplateSteps, _) => caseTemplateSteps.richPage(range.from, range.to, withTotal = true)(_.richCaseTemplate)
   )
-  override val outputQuery: Query = Query.output[RichCaseTemplate]()
-  override val extraQueries: Seq[ParamQuery[_]] = Seq(
-    Query[CaseTemplateSteps, List[RichCaseTemplate]]("toList", (caseTemplateSteps, _) => caseTemplateSteps.richCaseTemplate.toList)
-  )
+  override val outputQuery: Query = Query.output[RichCaseTemplate, CaseTemplateSteps](_.richCaseTemplate)
 
   def create: Action[AnyContent] =
     entrypoint("create case template")