Merge branch 'develop-th4' into issue-1264

# Conflicts:
#	thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala
#	thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala
#	thehive/app/org/thp/thehive/services/CaseSrv.scala
#	thehive/test/org/thp/thehive/controllers/v0/CaseCtrlTest.scala
#	thehive/test/org/thp/thehive/controllers/v1/TaxonomyCtrlTest.scala
#	thehive/test/org/thp/thehive/services/CaseSrvTest.scala
#	thehive/test/resources/data/Alert.json
#	thehive/test/resources/data/CaseTag.json
#	thehive/test/resources/data/Data.json
#	thehive/test/resources/data/Observable.json
#	thehive/test/resources/data/ObservableData.json
#	thehive/test/resources/data/ObservableObservableType.json
#	thehive/test/resources/data/Task.json

build.sbt

@@ -2,8 +2,8 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.0.5-1"
-val scala212               = "2.12.12"
+val thehiveVersion         = "4.1.0-1-SNAPSHOT"
+val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
 
@@ -62,7 +62,8 @@ libraryDependencies in ThisBuild ++= {
 }
 dependencyOverrides in ThisBuild ++= Seq(
 //  "org.locationtech.spatial4j" % "spatial4j"                 % "0.6",
-  "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2"
+//  "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2"
+  akkaActor
 )
 PlayKeys.includeDocumentationInBinary := false
 milestoneFilter := ((milestone: Milestone) => milestone.title.startsWith("4"))
@@ -322,6 +323,7 @@ lazy val mispClient = (project in file("misp/client"))
     libraryDependencies ++= Seq(
       ws,
       alpakka,
+      akkaHttp,
       specs      % Test,
       playMockws % Test
     )
@@ -340,13 +342,12 @@ lazy val thehiveMigration = (project in file("migration"))
     libraryDependencies ++= Seq(
       elastic4sCore,
       elastic4sHttpStreams,
-      elastic4sHttp,
+      elastic4sClient,
 //      jts,
       ehcache,
       scopt,
       specs % Test
     ),
-    fork := true,
     normalizedName := "migrate"
   )
 

conf/logback.xml

@@ -36,6 +36,8 @@
 
     <logger name="org.thp.scalligraph.models" level="TRACE"/>
     <logger name="org.thp.scalligraph.traversal" level="TRACE"/>
+    <logger name="org.thp.thehive.services.StreamSrv" level="INFO"/>
+    <logger name="org.thp.thehive.services.StreamActor" level="INFO"/>
     <!--
     <logger name="org.janusgraph.graphdb" level="INFO" />
     <logger name="org.thp.thehive.client" level="DEBUG" />

cortex/client/src/test/scala/org/thp/cortex/client/CortexClientTest.scala

@@ -1,12 +1,11 @@
 package org.thp.cortex.client
 
-import java.util.Date
-
 import org.thp.cortex.dto.v0._
 import org.thp.scalligraph.AppBuilder
 import play.api.libs.json.{JsObject, JsString, Json}
 import play.api.test.PlaySpecification
 
+import java.util.Date
 import scala.concurrent.duration._
 
 class CortexClientTest extends PlaySpecification {
@@ -64,55 +63,61 @@ class CortexClientTest extends PlaySpecification {
         OutputReport(
           summary = Seq(OutputMinireport("info", "test", "data", JsString("test"))),
           success = true,
-          full = Some(Json.parse("""{
-                                    "data": "imageedit_2_3904987689.jpg",
-                                    "input": {
-                                      "file": "attachment7619802021796183482",
-                                      "filename": "imageedit_2_3904987689.jpg",
-                                      "dataType": "file",
-                                      "tlp": 2,
-                                      "message": "179e85c4-4170-45fe-9d2d-3173539554a6",
-                                      "contentType": "image/jpeg",
-                                      "parameters": {
-                                      },
-                                      "config": {
-                                        "proxy_https": null,
-                                        "cacerts": null,
-                                        "max_pap": 2,
-                                        "jobTimeout": 30,
-                                        "check_tlp": true,
-                                        "proxy_http": null,
-                                        "max_tlp": 2,
-                                        "auto_extract_artifacts": false,
-                                        "jobCache": 10,
-                                        "check_pap": true
-                                      },
-                                      "pap": 2
-                                    }
-                                  }""").as[JsObject]),
-          artifacts = Json.parse("""[
-                                         {
-                                           "attachment": {
-                                             "contentType": "application/octet-stream",
-                                             "id": "e64871cf4652cb6e1babc06a376e7c79256dd6b967ca845ae06708cbeb686663",
-                                             "name": "passwd",
-                                             "size": 2644
-                                           },
-                                           "dataType": "file",
-                                           "message": null,
-                                           "tags": ["file", "virus"],
-                                           "tlp": 3
-                                         },
-                                         {
-                                           "data": "127.0.0.1",
-                                           "dataType": "ip",
-                                           "message": null,
-                                           "tags": [
-                                             "localhost"
-                                           ],
-                                           "tlp": 2
-                                         }
-                                       ]""").as[List[OutputArtifact]],
+          full = Some(
+            Json
+              .parse("""{
+                "data": "imageedit_2_3904987689.jpg",
+                "input": {
+                  "file": "attachment7619802021796183482",
+                  "filename": "imageedit_2_3904987689.jpg",
+                  "dataType": "file",
+                  "tlp": 2,
+                  "message": "179e85c4-4170-45fe-9d2d-3173539554a6",
+                  "contentType": "image/jpeg",
+                  "parameters": {
+                  },
+                  "config": {
+                    "proxy_https": null,
+                    "cacerts": null,
+                    "max_pap": 2,
+                    "jobTimeout": 30,
+                    "check_tlp": true,
+                    "proxy_http": null,
+                    "max_tlp": 2,
+                    "auto_extract_artifacts": false,
+                    "jobCache": 10,
+                    "check_pap": true
+                  },
+                  "pap": 2
+                }
+              }""")
+              .as[JsObject]
+          ),
+          artifacts = Json
+            .parse("""[
+              {
+                "attachment": {
+                  "contentType": "application/octet-stream",
+                  "id": "e64871cf4652cb6e1babc06a376e7c79256dd6b967ca845ae06708cbeb686663",
+                  "name": "passwd",
+                  "size": 2644
+                },
+                "dataType": "file",
+                "message": null,
+                "tags": ["file", "virus"],
+                "tlp": 3
+              },
+              {
+                "data": "127.0.0.1",
+                "dataType": "ip",
+                "message": null,
+                "tags": [
+                  "localhost"
+                ],
+                "tlp": 2
+              }
+            ]""")
+            .as[List[OutputArtifact]],
           operations = Nil,
           errorMessage = None,
           input = None

cortex/client/src/test/scala/org/thp/cortex/client/TestCortexClientProvider.scala

@@ -1,10 +1,6 @@
 package org.thp.cortex.client
 
-import java.net.URLEncoder
-import java.nio.file.{Path, Paths}
-
 import akka.stream.scaladsl._
-import javax.inject.{Inject, Provider}
 import mockws.MockWS
 import org.thp.client.NoAuthentication
 import org.thp.cortex.dto.v0.{OutputJob, OutputWorker}
@@ -13,6 +9,9 @@ import play.api.libs.json.{JsValue, Json}
 import play.api.mvc._
 import play.api.test.Helpers._
 
+import java.net.URLEncoder
+import java.nio.file.{Path, Paths}
+import javax.inject.{Inject, Provider}
 import scala.concurrent.ExecutionContext
 import scala.io.Source
 import scala.util.matching.Regex
@@ -42,7 +41,8 @@ class TestCortexClientProvider @Inject() (Action: DefaultActionBuilder, implicit
       val filename = URLEncoder.encode(s"$id.test.txt", "utf-8")
       Action(
         Result(
-          header = ResponseHeader(200, Map("Content-Disposition" -> s"""attachment; filename="$filename"""", "Content-Transfer-Encoding" -> "binary")),
+          header =
+            ResponseHeader(200, Map("Content-Disposition" -> s"""attachment; filename="$filename"""", "Content-Transfer-Encoding" -> "binary")),
           body = HttpEntity.Streamed(FileIO.fromPath(fileResource(id)), None, None)
         )
       )

cortex/connector/src/main/resources/play/reference-overrides.conf

@@ -1,12 +1,12 @@
 akka {
   actor {
     serializers {
-      cortex-schema-updater = "org.thp.thehive.connector.cortex.models.SchemaUpdaterSerializer"
+      //cortex-schema-updater = "org.thp.thehive.connector.cortex.models.SchemaUpdaterSerializer"
       cortex-jobs = "org.thp.thehive.connector.cortex.services.CortexSerializer"
     }
 
     serialization-bindings {
-      "org.thp.thehive.connector.cortex.models.SchemaUpdaterMessage" = cortex-schema-updater
+      //"org.thp.thehive.connector.cortex.models.SchemaUpdaterMessage" = cortex-schema-updater
       "org.thp.thehive.connector.cortex.services.CortexActorMessage" = cortex-jobs
     }
   }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/CortexModule.scala

@@ -2,10 +2,10 @@ package org.thp.thehive.connector.cortex
 
 import com.google.inject.AbstractModule
 import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder}
-import org.thp.scalligraph.models.{Database, Schema}
+import org.thp.scalligraph.models.UpdatableSchema
 import org.thp.scalligraph.query.QueryExecutor
 import org.thp.thehive.connector.cortex.controllers.v0.{CortexQueryExecutor => CortexQueryExecutorV0}
-import org.thp.thehive.connector.cortex.models.{CortexSchemaDefinition, DatabaseProvider}
+import org.thp.thehive.connector.cortex.models.CortexSchemaDefinition
 import org.thp.thehive.connector.cortex.services.notification.notifiers.{RunAnalyzerProvider, RunResponderProvider}
 import org.thp.thehive.connector.cortex.services.{Connector, CortexActor}
 import org.thp.thehive.services.notification.notifiers.NotifierProvider
@@ -24,14 +24,13 @@ class CortexModule(environment: Environment, configuration: Configuration) exten
     queryExecutorBindings.addBinding.to[CortexQueryExecutorV0]
     val connectorBindings = ScalaMultibinder.newSetBinder[TheHiveConnector](binder)
     connectorBindings.addBinding.to[Connector]
-    val schemaBindings = ScalaMultibinder.newSetBinder[Schema](binder)
+    val schemaBindings = ScalaMultibinder.newSetBinder[UpdatableSchema](binder)
     schemaBindings.addBinding.to[CortexSchemaDefinition]
 
     val notifierBindings = ScalaMultibinder.newSetBinder[NotifierProvider](binder)
     notifierBindings.addBinding.to[RunResponderProvider]
     notifierBindings.addBinding.to[RunAnalyzerProvider]
 
-    bind[Database].annotatedWithName("with-thehive-cortex-schema").toProvider[DatabaseProvider]
     bindActor[CortexActor]("cortex-actor")
     ()
   }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/CortexRouter.scala

@@ -1,10 +1,11 @@
 package org.thp.thehive.connector.cortex
 
-import javax.inject.{Inject, Provider, Singleton}
 import org.thp.thehive.connector.cortex.controllers.v0
 import play.api.Logger
 import play.api.routing.Router
 
+import javax.inject.{Inject, Provider, Singleton}
+
 @Singleton
 class CortexRouter @Inject() (routerV0: v0.Router) extends Provider[Router] {
 

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala

@@ -1,6 +1,5 @@
 package org.thp.thehive.connector.cortex.controllers.v0
 
-import javax.inject.{Inject, Named, Singleton}
 import org.thp.scalligraph.EntityIdOrName
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser}
@@ -20,13 +19,14 @@ import org.thp.thehive.services._
 import play.api.libs.json.{JsObject, Json, OWrites}
 import play.api.mvc.{AnyContent, Results, Action => PlayAction}
 
+import javax.inject.{Inject, Named, Singleton}
 import scala.concurrent.{ExecutionContext, Future}
 import scala.reflect.runtime.{universe => ru}
 
 @Singleton
 class ActionCtrl @Inject() (
     override val entrypoint: Entrypoint,
-    @Named("with-thehive-schema") override val db: Database,
+    override val db: Database,
     actionSrv: ActionSrv,
     entityHelper: EntityHelper,
     caseSrv: CaseSrv,
@@ -76,19 +76,17 @@ class ActionCtrl @Inject() (
 }
 
 @Singleton
-class PublicAction @Inject() (actionSrv: ActionSrv, @Named("with-thehive-schema") db: Database) extends PublicData {
+class PublicAction @Inject() (actionSrv: ActionSrv, organisationSrv: OrganisationSrv, db: Database) extends PublicData {
 
   override val entityName: String = "action"
   override val initialQuery: Query =
-    Query.init[Traversal.V[Action]]("listAction", (graph, authContext) => actionSrv.startTraversal(graph).visible(authContext))
+    Query.init[Traversal.V[Action]]("listAction", (graph, authContext) => actionSrv.startTraversal(graph).visible(organisationSrv)(authContext))
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Action]](
     "getAction",
-    FieldsParser[EntityIdOrName],
-    (idOrName, graph, authContext) => actionSrv.get(idOrName)(graph).visible(authContext)
+    (idOrName, graph, authContext) => actionSrv.get(idOrName)(graph).visible(organisationSrv)(authContext)
   )
   override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Action], IteratorOutput](
     "page",
-    FieldsParser[OutputParam],
     (range, actionSteps, _) => actionSteps.richPage(range.from, range.to, withTotal = true)(_.richAction)
   )
   override val outputQuery: Query = Query.output[RichAction, Traversal.V[Action]](_.richAction)

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerCtrl.scala

@@ -1,13 +1,13 @@
 package org.thp.thehive.connector.cortex.controllers.v0
 
 import akka.actor.ActorSystem
-import javax.inject.{Inject, Singleton}
 import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser}
 import org.thp.thehive.connector.cortex.controllers.v0.Conversion._
 import org.thp.thehive.connector.cortex.services.AnalyzerSrv
 import org.thp.thehive.controllers.v0.Conversion._
 import play.api.mvc.{Action, AnyContent, Results}
 
+import javax.inject.{Inject, Singleton}
 import scala.concurrent.ExecutionContext
 
 @Singleton

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala

@@ -1,9 +1,6 @@
 package org.thp.thehive.connector.cortex.controllers.v0
 
-import java.util.zip.ZipFile
-
 import com.google.inject.name.Named
-import javax.inject.{Inject, Singleton}
 import org.thp.scalligraph.EntityIdOrName
 import org.thp.scalligraph.controllers.{Entrypoint, FFile, FieldsParser}
 import org.thp.scalligraph.models.{Database, Entity, UMapping}
@@ -21,12 +18,14 @@ import org.thp.thehive.models.Permissions
 import play.api.libs.json.{JsFalse, JsObject, JsTrue}
 import play.api.mvc.{Action, AnyContent, Results}
 
+import java.util.zip.ZipFile
+import javax.inject.{Inject, Singleton}
 import scala.util.{Failure, Success}
 
 @Singleton
 class AnalyzerTemplateCtrl @Inject() (
     override val entrypoint: Entrypoint,
-    @Named("with-thehive-cortex-schema") override val db: Database,
+    override val db: Database,
     analyzerTemplateSrv: AnalyzerTemplateSrv,
     @Named("v0") override val queryExecutor: QueryExecutor,
     override val publicData: PublicAnalyzerTemplate
@@ -100,13 +99,11 @@ class PublicAnalyzerTemplate @Inject() (analyzerTemplateSrv: AnalyzerTemplateSrv
     Query.init[Traversal.V[AnalyzerTemplate]]("listAnalyzerTemplate", (graph, _) => analyzerTemplateSrv.startTraversal(graph))
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[AnalyzerTemplate]](
     "getReportTemplate",
-    FieldsParser[EntityIdOrName],
     (idOrName, graph, _) => analyzerTemplateSrv.get(idOrName)(graph)
   )
   override val pageQuery: ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[AnalyzerTemplate], IteratorOutput](
       "page",
-      FieldsParser[OutputParam],
       (range, analyzerTemplateTraversal, _) => analyzerTemplateTraversal.page(range.from, range.to, withTotal = true)
     )
   override val outputQuery: Query = Query.output[AnalyzerTemplate with Entity]

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala

@@ -22,6 +22,7 @@ object Conversion {
       .withFieldComputed(_.objectType, _.context._label)
       .withFieldComputed(_.operations, a => JsArray(a.operations).toString)
       .withFieldComputed(_.report, _.report.map(_.toString).getOrElse("{}"))
+      .enableMethodAccessors
       .transform
   )
 
@@ -46,6 +47,7 @@ object Conversion {
       .withFieldComputed(_.id, _._id.toString)
       .withFieldConst(_._type, "case_artifact_job")
       .withFieldConst(_.case_artifact, None)
+      .enableMethodAccessors
       .transform
   )
 
@@ -78,6 +80,7 @@ object Conversion {
               Some(observableWithExtraOutput.toValue((richObservable, JsObject.empty, Some(richCase))))
           }
         )
+        .enableMethodAccessors
         .transform
     }