#191 Add support of ElasticSearch 6

TheHive-Project · May 15, 2019 · 0d68069 · 0d68069
1 parent b0cd833
commit 0d68069
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 34 deletions.
diff --git a/app/org/thp/cortex/controllers/AttachmentCtrl.scala b/app/org/thp/cortex/controllers/AttachmentCtrl.scala
@@ -53,7 +53,7 @@ class AttachmentCtrl(
     * open the document directly. It must be used only for safe file
     */
   @Timed("controllers.AttachmentCtrl.download")
-  def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     if (hash.startsWith("{{")) // angularjs hack
       NoContent
     else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
@@ -74,7 +74,7 @@ class AttachmentCtrl(
     * File name can be specified (zip extension is append)
     */
   @Timed("controllers.AttachmentCtrl.downloadZip")
-  def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
       BadRequest("File name is invalid")
     else {

diff --git a/app/org/thp/cortex/controllers/DBListCtrl.scala b/app/org/thp/cortex/controllers/DBListCtrl.scala
@@ -22,13 +22,13 @@ class DBListCtrl @Inject() (
     fieldsBodyParser: FieldsBodyParser,
     implicit val ec: ExecutionContext) extends AbstractController(components) {
 
-  def list: Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒
+  def list: Action[AnyContent] = authenticated(Roles.read).async { _ ⇒
     dblists.listAll.map { listNames ⇒
       renderer.toOutput(OK, listNames)
     }
   }
 
-  def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
+  def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
     val (src, _) = dblists(listName).getItems[JsValue]
     val items = src.map { case (id, value) ⇒ s""""$id":$value""" }
       .intersperse("{", ",", "}")

diff --git a/app/org/thp/cortex/controllers/StatusCtrl.scala b/app/org/thp/cortex/controllers/StatusCtrl.scala
@@ -9,7 +9,7 @@ import play.api.libs.json.{ JsBoolean, JsString, Json }
 import play.api.libs.json.Json.toJsFieldJsValueWrapper
 import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
 
-import com.sksamuel.elastic4s.ElasticDsl
+import com.sksamuel.elastic4s.http.ElasticDsl
 import org.thp.cortex.models.Worker
 
 import org.elastic4play.database.DBIndex
@@ -26,16 +26,14 @@ class StatusCtrl @Inject() (
 
   private[controllers] def getVersion(c: Class[_]) = Option(c.getPackage.getImplementationVersion).getOrElse("SNAPSHOT")
 
-  def get: Action[AnyContent] = Action.async { _ ⇒
-    dbIndex.clusterVersions.map { versions ⇒
+  def get: Action[AnyContent] = Action {
       Ok(Json.obj(
         "versions" → Json.obj(
           "Cortex" → getVersion(classOf[Worker]),
           "Elastic4Play" → getVersion(classOf[AuthSrv]),
           "Play" → getVersion(classOf[AbstractController]),
           "Elastic4s" → getVersion(classOf[ElasticDsl]),
-          "ElasticSearch client" → getVersion(classOf[org.elasticsearch.Build]),
-          "ElasticSearch cluster" → versions.mkString(", ")),
+          "ElasticSearch client" → getVersion(classOf[org.elasticsearch.client.Node])),
         "config" → Json.obj(
           "authType" → (authSrv match {
             case multiAuthSrv: MultiAuthSrv ⇒ multiAuthSrv.authProviders.map { a ⇒ JsString(a.name) }
@@ -44,7 +42,6 @@ class StatusCtrl @Inject() (
           "capabilities" → authSrv.capabilities.map(c ⇒ JsString(c.toString)),
           "ssoAutoLogin" → JsBoolean(configuration.getOptional[Boolean]("auth.sso.autologin").getOrElse(false)))))
     }
-  }
 
   def health: Action[AnyContent] = TODO
 }
diff --git a/app/org/thp/cortex/models/Migration.scala b/app/org/thp/cortex/models/Migration.scala
@@ -5,13 +5,13 @@ import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.Success
 
 import play.api.Logger
-import play.api.libs.json.{ JsNull, JsString, JsValue, Json }
+import play.api.libs.json.{ JsNull, JsNumber, JsString, JsValue, Json }
 
 import org.thp.cortex.services.{ OrganizationSrv, UserSrv, WorkerSrv }
 
 import org.elastic4play.controllers.Fields
 import org.elastic4play.services.Operation._
-import org.elastic4play.services.{ DatabaseState, MigrationOperations, Operation }
+import org.elastic4play.services.{ DatabaseState, IndexType, MigrationOperations, Operation }
 import org.elastic4play.utils.Hasher
 
 @Singleton
@@ -34,6 +34,8 @@ class Migration @Inject() (
     }
   }
 
+  override def indexType(version: Int): IndexType.Value = if (version > 3) IndexType.indexWithoutMappingTypes else IndexType.indexWithMappingTypes
+
   val operations: PartialFunction[DatabaseState, Seq[Operation]] = {
     case DatabaseState(1) ⇒
       val hasher = Hasher("MD5")
@@ -83,5 +85,15 @@ class Migration @Inject() (
               ("baseConfig" -> definition.baseConfiguration.fold[JsValue](JsNull)(JsString.apply))
           }
       })
+
+    case DatabaseState(3) ⇒ Seq(
+        mapEntity("sequence") { seq =>
+          val oldId = (seq \ "_id").as[String]
+          val counter = (seq \ "counter").as[JsNumber]
+          seq - "counter" - "_routing" +
+            ("_id" -> JsString("sequence_" + oldId)) +
+            ("sequenceCounter" -> counter)
+        }
+      )
   }
 }
diff --git a/app/org/thp/cortex/models/Roles.scala b/app/org/thp/cortex/models/Roles.scala
@@ -2,8 +2,8 @@ package org.thp.cortex.models
 
 import play.api.libs.json.{ JsString, JsValue }
 
-import com.sksamuel.elastic4s.ElasticDsl.keywordField
-import com.sksamuel.elastic4s.mappings.KeywordFieldDefinition
+import com.sksamuel.elastic4s.http.ElasticDsl.keywordField
+import com.sksamuel.elastic4s.mappings.KeywordField
 import org.scalactic.{ Every, Good, One, Or }
 
 import org.elastic4play.{ AttributeError, InvalidFormatAttributeError }
@@ -47,5 +47,5 @@ object RoleAttributeFormat extends AttributeFormat[Role]("role") {
 
   }
 
-  override def elasticType(attributeName: String): KeywordFieldDefinition = keywordField(attributeName)
+  override def elasticType(attributeName: String): KeywordField = keywordField(attributeName)
 }
diff --git a/app/org/thp/cortex/models/package.scala b/app/org/thp/cortex/models/package.scala
@@ -1,5 +1,5 @@
 package org.thp.cortex
 
 package object models {
-  val modelVersion = 3
+  val modelVersion = 4
 }
diff --git a/app/org/thp/cortex/services/ErrorHandler.scala b/app/org/thp/cortex/services/ErrorHandler.scala
@@ -1,20 +1,21 @@
 package org.thp.cortex.services
 
-import org.thp.cortex.models.{ WorkerNotFoundError, JobNotFoundError, RateLimitExceeded }
-import play.api.Logger
-import play.api.http.{ HttpErrorHandler, Status, Writeable }
-import play.api.mvc.{ RequestHeader, ResponseHeader, Result, Results }
+import java.net.ConnectException
+
 import scala.concurrent.Future
 
+import org.thp.cortex.models.{ JobNotFoundError, RateLimitExceeded, WorkerNotFoundError }
+import play.api.Logger
+import play.api.http.{ HttpErrorHandler, Status, Writeable }
 import play.api.libs.json.{ JsNull, JsValue, Json }
+import play.api.mvc.{ RequestHeader, ResponseHeader, Result, Results }
 
-import org.elasticsearch.client.transport.NoNodeAvailableException
-
-import org.elastic4play._
-import org.elastic4play.JsonFormat._
-import org.elasticsearch.index.IndexNotFoundException
-import org.elasticsearch.index.query.QueryShardException
+import org.elastic4play.{ AttributeCheckingError, AuthenticationError, AuthorizationError, BadRequestError, CreateError, ErrorWithObject, GetError, IndexNotFoundException, InternalError, MultiError, NotFoundError, SearchError, UpdateError }
+import org.elastic4play.JsonFormat.attributeCheckingExceptionWrites
 
+/**
+  * This class handles errors. It traverses all causes of exception to find known error and shows the appropriate message
+  */
 class ErrorHandler extends HttpErrorHandler {
   private[ErrorHandler] lazy val logger = Logger(getClass)
   def onClientError(request: RequestHeader, statusCode: Int, message: String): Future[Result] = Future.successful {
@@ -31,12 +32,12 @@ class ErrorHandler extends HttpErrorHandler {
       case nfe: NumberFormatException          ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "NumberFormatException", "message" → ("Invalid format " + nfe.getMessage)))
       case NotFoundError(message)              ⇒ Some(Status.NOT_FOUND → Json.obj("type" → "NotFoundError", "message" → message))
       case BadRequestError(message)            ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "BadRequest", "message" → message))
-      case SearchError(message, cause)         ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "SearchError", "message" → s"$message (${cause.getMessage})"))
+      case SearchError(message)                ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "SearchError", "message" → s"$message"))
       case ace: AttributeCheckingError         ⇒ Some(Status.BAD_REQUEST → Json.toJson(ace))
       case iae: IllegalArgumentException       ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "IllegalArgument", "message" → iae.getMessage))
-      case _: NoNodeAvailableException         ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "NoNodeAvailable", "message" → "ElasticSearch cluster is unreachable"))
+      case _: ConnectException                 ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "NoNodeAvailable", "message" → "ElasticSearch cluster is unreachable"))
       case CreateError(_, message, attributes) ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "CreateError", "message" → message, "object" → attributes))
-      case ErrorWithObject(tpe, message, obj)  ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → tpe, "message" → message, "object" → obj))
+      case ErrorWithObject(tpe, message, obj)  ⇒ Some(Status.BAD_REQUEST           → Json.obj("type" → tpe, "message"        → message, "object" → obj))
       case GetError(message)                   ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "GetError", "message" → message))
       case MultiError(message, exceptions) ⇒
         val suberrors = exceptions.map(e ⇒ toErrorResult(e)).collect {
@@ -45,16 +46,17 @@ class ErrorHandler extends HttpErrorHandler {
         Some(Status.MULTI_STATUS → Json.obj("type" → "MultiError", "error" → message, "suberrors" → suberrors))
       case JobNotFoundError(jobId)         ⇒ Some(Status.NOT_FOUND → Json.obj("type" → "JobNotFoundError", "message" → s"Job $jobId not found"))
       case WorkerNotFoundError(analyzerId) ⇒ Some(Status.NOT_FOUND → Json.obj("type" → "AnalyzerNotFoundError", "message" → s"analyzer $analyzerId not found"))
-      case _: IndexNotFoundException       ⇒ Some(520 → JsNull)
-      case qse: QueryShardException        ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "Invalid search query", "message" → qse.getMessage))
-      case t: Throwable                    ⇒ Option(t.getCause).flatMap(toErrorResult)
+      case IndexNotFoundException ⇒ Some(520 → JsNull)
+      case t: Throwable           ⇒ Option(t.getCause).flatMap(toErrorResult)
     }
   }
 
   def toResult[C](status: Int, c: C)(implicit writeable: Writeable[C]) = Result(header = ResponseHeader(status), body = writeable.toEntity(c))
 
   def onServerError(request: RequestHeader, exception: Throwable): Future[Result] = {
-    val (status, body) = toErrorResult(exception).getOrElse(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → exception.getClass.getName, "message" → exception.getMessage))
+    val (status, body) = toErrorResult(exception).getOrElse(
+      Status.INTERNAL_SERVER_ERROR → Json.obj("type" → exception.getClass.getName, "message" → exception.getMessage)
+    )
     logger.info(s"${request.method} ${request.uri} returned $status", exception)
     Future.successful(toResult(status, body))
   }

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -18,7 +18,7 @@ object Dependencies {
 
   val reflections = "org.reflections" % "reflections" % "0.9.11"
   val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-  val elastic4play = "org.thehive-project" %% "elastic4play" % "1.10.0"
+  val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.0"
   val dockerClient = "com.spotify" % "docker-client" % "8.14.4"
 }