#1377 Add support of ES7

TheHive-Project · May 28, 2020 · 2b91a5a · 2b91a5a
1 parent c24a397
commit 2b91a5a
Show file tree

Hide file tree

Showing 50 changed files with 238 additions and 316 deletions.
diff --git a/build.sbt b/build.sbt
@@ -17,6 +17,7 @@ lazy val thehiveBackend = (project in file("thehive-backend"))
       Library.zip4j,
       Library.reflections,
       Library.akkaCluster,
+      Library.akkaClusterTyped,
       Library.akkaClusterTools
     ),
     play.sbt.routes.RoutesKeys.routesImport -= "controllers.Assets.Asset"

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -1,27 +1,27 @@
 import sbt._
 
 object Dependencies {
-  val scalaVersion = "2.12.6"
+  val scalaVersion = "2.12.11"
 
   object Library {
 
     object Play {
-      val version = play.core.PlayVersion.current
-      val ws = "com.typesafe.play" %% "play-ws" % version
-      val ahc = "com.typesafe.play" %% "play-ahc-ws" % version
-      val cache = "com.typesafe.play" %% "play-ehcache" % version
-      val test = "com.typesafe.play" %% "play-test" % version
-      val specs2 = "com.typesafe.play" %% "play-specs2" % version
-      val filters = "com.typesafe.play" %% "filters-helpers" % version
-      val guice = "com.typesafe.play" %% "play-guice" % version
+      val ws = "com.typesafe.play" %% "play-ws" % play.core.PlayVersion.current
+      val ahc = "com.typesafe.play" %% "play-ahc-ws" % play.core.PlayVersion.current
+      val cache = "com.typesafe.play" %% "play-ehcache" % play.core.PlayVersion.current
+      val test = "com.typesafe.play" %% "play-test" % play.core.PlayVersion.current
+      val specs2 = "com.typesafe.play" %% "play-specs2" % play.core.PlayVersion.current
+      val filters = "com.typesafe.play" %% "filters-helpers" % play.core.PlayVersion.current
+      val guice = "com.typesafe.play" %% "play-guice" % play.core.PlayVersion.current
     }
 
-    val scalaGuice = "net.codingwell" %% "scala-guice" % "4.2.3"
+    val scalaGuice = "net.codingwell" %% "scala-guice" % "4.2.6"
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
-    val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.5"
-    val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.21"
-    val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.21"
+    val zip4j = "net.lingala.zip4j" % "zip4j" % "2.6.0"
+    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.12.0-SNAPSHOT"
+    val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % play.core.PlayVersion.akkaVersion
+    val akkaClusterTyped = "com.typesafe.akka" %% "akka-cluster-typed" % play.core.PlayVersion.akkaVersion
+    val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % play.core.PlayVersion.akkaVersion
   }
 }
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,7 +1,7 @@
 // Comment to get more information during initialization
 logLevel := Level.Info
 
-addSbtPlugin("com.typesafe.play"   % "sbt-plugin"           % "2.6.23")
+addSbtPlugin("com.typesafe.play"   % "sbt-plugin"           % "2.8.2")
 addSbtPlugin("org.foundweekends"   % "sbt-bintray"          % "0.5.1")
-addSbtPlugin("org.scalameta"       % "sbt-scalafmt"         % "2.0.0")
+addSbtPlugin("org.scalameta"       % "sbt-scalafmt"         % "2.4.0")
 addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.3.0")
diff --git a/thehive-backend/app/connectors/Connectors.scala b/thehive-backend/app/connectors/Connectors.scala
@@ -1,13 +1,12 @@
 package connectors
 
 import scala.collection.immutable
-
 import play.api.libs.json.{JsObject, Json}
 import play.api.mvc._
 import play.api.routing.sird.UrlContext
 import play.api.routing.{Router, SimpleRouter}
-
 import com.google.inject.AbstractModule
+import scala.reflect.runtime.{universe => ru}
 import javax.inject.{Inject, Singleton}
 import models.HealthStatus
 import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder}
@@ -35,7 +34,7 @@ class ConnectorRouter @Inject()(connectors: immutable.Set[Connector], actionBuil
 
 abstract class ConnectorModule extends AbstractModule with ScalaModule {
 
-  def registerController[C <: Connector](implicit evidence: Manifest[C]): Unit = {
+  def registerController[C <: Connector: ru.TypeTag]: Unit = {
     val connectorBindings = ScalaMultibinder.newSetBinder[Connector](binder)
     connectorBindings.addBinding.to[C]
     ()

diff --git a/thehive-backend/app/controllers/ArtifactCtrl.scala b/thehive-backend/app/controllers/ArtifactCtrl.scala
@@ -3,25 +3,24 @@ package controllers
 import java.io.FilterInputStream
 import java.nio.file.{Files, Path}
 
-import scala.collection.JavaConverters._
-import scala.concurrent.{ExecutionContext, Future}
-
+import javax.inject.{Inject, Singleton}
+import models.Roles
+import net.lingala.zip4j.ZipFile
 import play.api.http.Status
 import play.api.libs.json.{JsArray, JsValue}
 import play.api.mvc._
 import play.api.{Configuration, Logger}
 
-import javax.inject.{Inject, Singleton}
-import models.Roles
-import net.lingala.zip4j.core.ZipFile
+import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, Future}
+//import net.lingala.zip4j.core.ZipFile
 import net.lingala.zip4j.model.FileHeader
-import services.ArtifactSrv
-
 import org.elastic4play.controllers._
 import org.elastic4play.models.JsonFormat.baseModelEntityWrites
 import org.elastic4play.services.JsonFormat.{aggReads, queryReads}
 import org.elastic4play.services._
 import org.elastic4play.{BadRequestError, InternalError, InvalidFormatAttributeError, Timed}
+import services.ArtifactSrv
 
 @Singleton
 class ArtifactCtrl @Inject()(
@@ -70,7 +69,7 @@ class ArtifactCtrl @Inject()(
     }
   }
 
-  private def getFieldsFromZipFile(caseId: String, fields: Fields, filepath: Path)(implicit authContext: AuthContext): Seq[Fields] = {
+  private def getFieldsFromZipFile(fields: Fields, filepath: Path)(implicit authContext: AuthContext): Seq[Fields] = {
     val zipFile                = new ZipFile(filepath.toFile)
     val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]]
 
@@ -79,7 +78,7 @@ class ArtifactCtrl @Inject()(
         .getString("zipPassword")
         .filterNot(_.isEmpty)
         .getOrElse(configuration.get[String]("datastore.attachment.password"))
-      zipFile.setPassword(pw)
+      zipFile.setPassword(pw.toCharArray)
     }
 
     /*val multiFields = */
@@ -125,7 +124,7 @@ class ArtifactCtrl @Inject()(
         .get("attachment")
         .map {
           case FileInputValue(_, filepath, _) if fields.getBoolean("isZip").getOrElse(false) ⇒
-            Future.successful(getFieldsFromZipFile(caseId, fields, filepath))
+            Future.successful(getFieldsFromZipFile(fields, filepath))
           case _: FileInputValue ⇒ Future.successful(Seq(fields))
           case JsonInputValue(JsArray(attachments)) ⇒
             Future.traverse(attachments)(attachment ⇒ getFieldsFromAttachment(fields, attachment)).map(_.flatten)
@@ -155,7 +154,7 @@ class ArtifactCtrl @Inject()(
   }
 
   @Timed
-  def get(id: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒
+  def get(id: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { _ ⇒
     artifactSrv
       .get(id)
       .map(artifact ⇒ renderer.toOutput(OK, artifact))

diff --git a/thehive-backend/app/controllers/AttachmentCtrl.scala b/thehive-backend/app/controllers/AttachmentCtrl.scala
@@ -1,23 +1,21 @@
 package controllers
 
 import java.nio.file.Files
-import javax.inject.{Inject, Singleton}
-
-import play.api.http.HttpEntity
-import play.api.libs.Files.DefaultTemporaryFileCreator
-import play.api.mvc._
-import play.api.{mvc, Configuration}
 
 import akka.stream.scaladsl.FileIO
-import net.lingala.zip4j.core.ZipFile
-import net.lingala.zip4j.model.ZipParameters
-import net.lingala.zip4j.util.Zip4jConstants
+import javax.inject.{Inject, Singleton}
 import models.Roles
-
+import net.lingala.zip4j.ZipFile
+import net.lingala.zip4j.model.ZipParameters
+import net.lingala.zip4j.model.enums.{CompressionLevel, EncryptionMethod}
 import org.elastic4play.Timed
-import org.elastic4play.controllers.{Authenticated, Renderer}
+import org.elastic4play.controllers.Authenticated
 import org.elastic4play.models.AttachmentAttributeFormat
 import org.elastic4play.services.AttachmentSrv
+import play.api.http.HttpEntity
+import play.api.libs.Files.DefaultTemporaryFileCreator
+import play.api.mvc._
+import play.api.{Configuration, mvc}
 
 /**
   * Controller used to access stored attachments (plain or zipped)
@@ -29,7 +27,6 @@ class AttachmentCtrl(
     attachmentSrv: AttachmentSrv,
     authenticated: Authenticated,
     components: ControllerComponents,
-    renderer: Renderer
 ) extends AbstractController(components) {
 
   @Inject() def this(
@@ -38,17 +35,16 @@ class AttachmentCtrl(
       attachmentSrv: AttachmentSrv,
       authenticated: Authenticated,
       components: ControllerComponents,
-      renderer: Renderer
   ) =
-    this(configuration.get[String]("datastore.attachment.password"), tempFileCreator, attachmentSrv, authenticated, components, renderer)
+    this(configuration.get[String]("datastore.attachment.password"), tempFileCreator, attachmentSrv, authenticated, components)
 
   /**
     * Download an attachment, identified by its hash, in plain format
     * File name can be specified. This method is not protected : browser will
     * open the document directly. It must be used only for safe file
     */
   @Timed("controllers.AttachmentCtrl.download")
-  def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     if (hash.startsWith("{{")) // angularjs hack
       NoContent
     else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
@@ -69,20 +65,21 @@ class AttachmentCtrl(
     * File name can be specified (zip extension is append)
     */
   @Timed("controllers.AttachmentCtrl.downloadZip")
-  def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
       BadRequest("File name is invalid")
     else {
       val f = tempFileCreator.create("zip", hash).path
       Files.delete(f)
       val zipFile   = new ZipFile(f.toFile)
+      zipFile.setPassword(password.toCharArray)
       val zipParams = new ZipParameters
-      zipParams.setCompressionLevel(Zip4jConstants.DEFLATE_LEVEL_FASTEST)
+      zipParams.setCompressionLevel(CompressionLevel.FASTEST)
       zipParams.setEncryptFiles(true)
-      zipParams.setEncryptionMethod(Zip4jConstants.ENC_METHOD_STANDARD)
-      zipParams.setPassword(password)
+      zipParams.setEncryptionMethod(EncryptionMethod.ZIP_STANDARD)
+//      zipParams.setsetPassword(password.toCharArray)
       zipParams.setFileNameInZip(name.getOrElse(hash))
-      zipParams.setSourceExternalStream(true)
+//      zipParams.setSourceExternalStream(true)
       zipFile.addStream(attachmentSrv.stream(hash), zipParams)
 
       Result(

diff --git a/thehive-backend/app/controllers/AuditCtrl.scala b/thehive-backend/app/controllers/AuditCtrl.scala
@@ -30,7 +30,7 @@ class AuditCtrl @Inject()(
     * Return audit logs. For each item, include ancestor entities
     */
   @Timed
-  def flow(rootId: Option[String], count: Option[Int]): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def flow(rootId: Option[String], count: Option[Int]): Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     val (audits, total) = auditSrv(rootId.filterNot(_ == "any"), count.getOrElse(10))
     renderer.toOutput(OK, audits, total)
   }

diff --git a/thehive-backend/app/controllers/AuthenticationCtrl.scala b/thehive-backend/app/controllers/AuthenticationCtrl.scala
@@ -1,9 +1,8 @@
 package controllers
 
 import javax.inject.{Inject, Singleton}
-
 import models.UserStatus
-import org.elastic4play.controllers.{Authenticated, Fields, FieldsBodyParser, Renderer}
+import org.elastic4play.controllers.{Authenticated, Fields, FieldsBodyParser}
 import org.elastic4play.database.DBIndex
 import org.elastic4play.services.AuthSrv
 import org.elastic4play.{AuthorizationError, OAuth2Redirect, Timed}
@@ -18,7 +17,6 @@ class AuthenticationCtrl @Inject()(
     userSrv: UserSrv,
     authenticated: Authenticated,
     dbIndex: DBIndex,
-    renderer: Renderer,
     components: ControllerComponents,
     fieldsBodyParser: FieldsBodyParser,
     implicit val ec: ExecutionContext
@@ -63,7 +61,7 @@ class AuthenticationCtrl @Inject()(
   }
 
   @Timed
-  def logout = Action {
+  def logout: Action[AnyContent] = Action {
     Ok.withNewSession
   }
 }
diff --git a/thehive-backend/app/controllers/CaseCtrl.scala b/thehive-backend/app/controllers/CaseCtrl.scala
@@ -126,7 +126,7 @@ class CaseCtrl @Inject()(
   }
 
   @Timed
-  def linkedCases(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def linkedCases(id: String): Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     caseSrv
       .linkedCases(id)
       .runWith(Sink.seq)

diff --git a/thehive-backend/app/controllers/CaseTemplateCtrl.scala b/thehive-backend/app/controllers/CaseTemplateCtrl.scala
@@ -36,7 +36,7 @@ class CaseTemplateCtrl @Inject()(
   }
 
   @Timed
-  def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     caseTemplateSrv
       .get(id)
       .map(caze ⇒ renderer.toOutput(OK, caze))

diff --git a/thehive-backend/app/controllers/CustomFieldsCtrl.scala b/thehive-backend/app/controllers/CustomFieldsCtrl.scala
@@ -1,17 +1,15 @@
 package controllers
 
 import scala.concurrent.{ExecutionContext, Future}
-
 import play.api.http.Status
 import play.api.libs.json.{JsNumber, JsObject, Json}
 import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents}
-
 import akka.stream.Materializer
 import akka.stream.scaladsl.Sink
-import com.sksamuel.elastic4s.http.ElasticDsl.{search, termsAggregation}
+import com.sksamuel.elastic4s.ElasticDsl.{search, termsAgg}
+import com.sksamuel.elastic4s.requests.searches.aggs.responses.bucket.Terms
 import javax.inject.{Inject, Singleton}
 import models.Roles
-
 import org.elastic4play.NotFoundError
 import org.elastic4play.controllers.Authenticated
 import org.elastic4play.database.DBFind
@@ -43,9 +41,9 @@ class CustomFieldsCtrl @Inject()(
           .flatMap { customFieldType ⇒
             val filter = and("relations" in ("case", "alert", "caseTemplate"), contains(s"customFields.$customField.$customFieldType"))
             dbfind(
-              indexName ⇒ search(indexName).query(filter.query).aggregations(termsAggregation("t").field("relations"))
+              indexName ⇒ search(indexName).query(filter.query).aggregations(termsAgg("t","relations"))
             ).map { searchResponse ⇒
-                val buckets = searchResponse.aggregations.terms("t").buckets
+                val buckets = searchResponse.aggregations.result[Terms]("t").buckets
                 val total   = buckets.map(_.docCount).sum
                 val result  = buckets.map(b ⇒ b.key → JsNumber(b.docCount)) :+ ("total" → JsNumber(total))
                 Ok(JsObject(result))

diff --git a/thehive-backend/app/controllers/DBListCtrl.scala b/thehive-backend/app/controllers/DBListCtrl.scala
@@ -23,14 +23,14 @@ class DBListCtrl @Inject()(
 ) extends AbstractController(components) {
 
   @Timed("controllers.DBListCtrl.list")
-  def list: Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def list: Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     dblists.listAll.map { listNames ⇒
       renderer.toOutput(OK, listNames)
     }
   }
 
   @Timed("controllers.DBListCtrl.listItems")
-  def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     val (src, _) = dblists(listName).getItems[JsValue]
     val items = src
       .map { case (id, value) ⇒ s""""$id":$value""" }

diff --git a/thehive-backend/app/controllers/DashboardCtrl.scala b/thehive-backend/app/controllers/DashboardCtrl.scala
@@ -21,7 +21,6 @@ import org.elastic4play.{AuthorizationError, BadRequestError, Timed}
 @Singleton
 class DashboardCtrl @Inject()(
     dashboardSrv: DashboardSrv,
-    auxSrv: AuxSrv,
     authenticated: Authenticated,
     renderer: Renderer,
     components: ControllerComponents,
@@ -41,7 +40,7 @@ class DashboardCtrl @Inject()(
   }
 
   @Timed
-  def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     dashboardSrv.get(id).map { dashboard ⇒
       renderer.toOutput(OK, dashboard)
     }

diff --git a/thehive-backend/app/controllers/DescribeCtrl.scala b/thehive-backend/app/controllers/DescribeCtrl.scala
@@ -33,7 +33,7 @@ class DescribeCtrl @Inject()(
     Json.obj("label" → model.label, "path" → model.path, "attributes" → attributeDefinitions)
   }
 
-  def describe(modelName: String): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def describe(modelName: String): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     modelSrv(modelName)
       .map { model ⇒
         renderer.toOutput(OK, modelToJson(model))
@@ -43,7 +43,7 @@ class DescribeCtrl @Inject()(
 
   private val allModels: Seq[String] = Seq("case", "case_artifact", "case_task", "case_task_log", "alert", "case_artifact_job", "audit", "action")
 
-  def describeAll: Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def describeAll: Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     val entityDefinitions = modelSrv
       .list
       .collect {

diff --git a/thehive-backend/app/controllers/LogCtrl.scala b/thehive-backend/app/controllers/LogCtrl.scala
@@ -34,7 +34,7 @@ class LogCtrl @Inject()(
   }
 
   @Timed
-  def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     logSrv
       .get(id)
       .map(log ⇒ renderer.toOutput(OK, log))