diff --git a/app/org/thp/cortex/controllers/AttachmentCtrl.scala b/app/org/thp/cortex/controllers/AttachmentCtrl.scala index 7a8bb43c8..5fb21beaf 100644 --- a/app/org/thp/cortex/controllers/AttachmentCtrl.scala +++ b/app/org/thp/cortex/controllers/AttachmentCtrl.scala @@ -53,7 +53,7 @@ class AttachmentCtrl( * open the document directly. It must be used only for safe file */ @Timed("controllers.AttachmentCtrl.download") - def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒ + def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒ if (hash.startsWith("{{")) // angularjs hack NoContent else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty) @@ -74,7 +74,7 @@ class AttachmentCtrl( * File name can be specified (zip extension is append) */ @Timed("controllers.AttachmentCtrl.downloadZip") - def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒ + def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒ if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty) BadRequest("File name is invalid") else { diff --git a/app/org/thp/cortex/controllers/DBListCtrl.scala b/app/org/thp/cortex/controllers/DBListCtrl.scala index f812475b4..aa47fa631 100644 --- a/app/org/thp/cortex/controllers/DBListCtrl.scala +++ b/app/org/thp/cortex/controllers/DBListCtrl.scala @@ -22,13 +22,13 @@ class DBListCtrl @Inject() ( fieldsBodyParser: FieldsBodyParser, implicit val ec: ExecutionContext) extends AbstractController(components) { - def list: Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ + def list: Action[AnyContent] = authenticated(Roles.read).async { _ ⇒ dblists.listAll.map { listNames ⇒ renderer.toOutput(OK, listNames) } } - def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒ + def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { _ ⇒ val (src, _) = dblists(listName).getItems[JsValue] val items = src.map { case (id, value) ⇒ s""""$id":$value""" } .intersperse("{", ",", "}") diff --git a/app/org/thp/cortex/controllers/StatusCtrl.scala b/app/org/thp/cortex/controllers/StatusCtrl.scala index 165a1b492..859ae2d5f 100644 --- a/app/org/thp/cortex/controllers/StatusCtrl.scala +++ b/app/org/thp/cortex/controllers/StatusCtrl.scala @@ -9,7 +9,7 @@ import play.api.libs.json.{ JsBoolean, JsString, Json } import play.api.libs.json.Json.toJsFieldJsValueWrapper import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } -import com.sksamuel.elastic4s.ElasticDsl +import com.sksamuel.elastic4s.http.ElasticDsl import org.thp.cortex.models.Worker import org.elastic4play.database.DBIndex @@ -26,16 +26,14 @@ class StatusCtrl @Inject() ( private[controllers] def getVersion(c: Class[_]) = Option(c.getPackage.getImplementationVersion).getOrElse("SNAPSHOT") - def get: Action[AnyContent] = Action.async { _ ⇒ - dbIndex.clusterVersions.map { versions ⇒ + def get: Action[AnyContent] = Action { Ok(Json.obj( "versions" → Json.obj( "Cortex" → getVersion(classOf[Worker]), "Elastic4Play" → getVersion(classOf[AuthSrv]), "Play" → getVersion(classOf[AbstractController]), "Elastic4s" → getVersion(classOf[ElasticDsl]), - "ElasticSearch client" → getVersion(classOf[org.elasticsearch.Build]), - "ElasticSearch cluster" → versions.mkString(", ")), + "ElasticSearch client" → getVersion(classOf[org.elasticsearch.client.Node])), "config" → Json.obj( "authType" → (authSrv match { case multiAuthSrv: MultiAuthSrv ⇒ multiAuthSrv.authProviders.map { a ⇒ JsString(a.name) } @@ -44,7 +42,6 @@ class StatusCtrl @Inject() ( "capabilities" → authSrv.capabilities.map(c ⇒ JsString(c.toString)), "ssoAutoLogin" → JsBoolean(configuration.getOptional[Boolean]("auth.sso.autologin").getOrElse(false))))) } - } def health: Action[AnyContent] = TODO } diff --git a/app/org/thp/cortex/models/Migration.scala b/app/org/thp/cortex/models/Migration.scala index 17f16d408..2203a55d0 100644 --- a/app/org/thp/cortex/models/Migration.scala +++ b/app/org/thp/cortex/models/Migration.scala @@ -5,13 +5,13 @@ import scala.concurrent.{ ExecutionContext, Future } import scala.util.Success import play.api.Logger -import play.api.libs.json.{ JsNull, JsString, JsValue, Json } +import play.api.libs.json.{ JsNull, JsNumber, JsString, JsValue, Json } import org.thp.cortex.services.{ OrganizationSrv, UserSrv, WorkerSrv } import org.elastic4play.controllers.Fields import org.elastic4play.services.Operation._ -import org.elastic4play.services.{ DatabaseState, MigrationOperations, Operation } +import org.elastic4play.services.{ DatabaseState, IndexType, MigrationOperations, Operation } import org.elastic4play.utils.Hasher @Singleton @@ -34,6 +34,8 @@ class Migration @Inject() ( } } + override def indexType(version: Int): IndexType.Value = if (version > 3) IndexType.indexWithoutMappingTypes else IndexType.indexWithMappingTypes + val operations: PartialFunction[DatabaseState, Seq[Operation]] = { case DatabaseState(1) ⇒ val hasher = Hasher("MD5") @@ -83,5 +85,15 @@ class Migration @Inject() ( ("baseConfig" -> definition.baseConfiguration.fold[JsValue](JsNull)(JsString.apply)) } }) + + case DatabaseState(3) ⇒ Seq( + mapEntity("sequence") { seq => + val oldId = (seq \ "_id").as[String] + val counter = (seq \ "counter").as[JsNumber] + seq - "counter" - "_routing" + + ("_id" -> JsString("sequence_" + oldId)) + + ("sequenceCounter" -> counter) + } + ) } } diff --git a/app/org/thp/cortex/models/Roles.scala b/app/org/thp/cortex/models/Roles.scala index f268adfa1..1e73e522a 100644 --- a/app/org/thp/cortex/models/Roles.scala +++ b/app/org/thp/cortex/models/Roles.scala @@ -2,8 +2,8 @@ package org.thp.cortex.models import play.api.libs.json.{ JsString, JsValue } -import com.sksamuel.elastic4s.ElasticDsl.keywordField -import com.sksamuel.elastic4s.mappings.KeywordFieldDefinition +import com.sksamuel.elastic4s.http.ElasticDsl.keywordField +import com.sksamuel.elastic4s.mappings.KeywordField import org.scalactic.{ Every, Good, One, Or } import org.elastic4play.{ AttributeError, InvalidFormatAttributeError } @@ -47,5 +47,5 @@ object RoleAttributeFormat extends AttributeFormat[Role]("role") { } - override def elasticType(attributeName: String): KeywordFieldDefinition = keywordField(attributeName) + override def elasticType(attributeName: String): KeywordField = keywordField(attributeName) } \ No newline at end of file diff --git a/app/org/thp/cortex/models/package.scala b/app/org/thp/cortex/models/package.scala index bafbee1ba..0bbb2c3af 100644 --- a/app/org/thp/cortex/models/package.scala +++ b/app/org/thp/cortex/models/package.scala @@ -1,5 +1,5 @@ package org.thp.cortex package object models { - val modelVersion = 3 + val modelVersion = 4 } diff --git a/app/org/thp/cortex/services/ErrorHandler.scala b/app/org/thp/cortex/services/ErrorHandler.scala index 217a4d7a3..0757378b5 100644 --- a/app/org/thp/cortex/services/ErrorHandler.scala +++ b/app/org/thp/cortex/services/ErrorHandler.scala @@ -1,20 +1,21 @@ package org.thp.cortex.services -import org.thp.cortex.models.{ WorkerNotFoundError, JobNotFoundError, RateLimitExceeded } -import play.api.Logger -import play.api.http.{ HttpErrorHandler, Status, Writeable } -import play.api.mvc.{ RequestHeader, ResponseHeader, Result, Results } +import java.net.ConnectException + import scala.concurrent.Future +import org.thp.cortex.models.{ JobNotFoundError, RateLimitExceeded, WorkerNotFoundError } +import play.api.Logger +import play.api.http.{ HttpErrorHandler, Status, Writeable } import play.api.libs.json.{ JsNull, JsValue, Json } +import play.api.mvc.{ RequestHeader, ResponseHeader, Result, Results } -import org.elasticsearch.client.transport.NoNodeAvailableException - -import org.elastic4play._ -import org.elastic4play.JsonFormat._ -import org.elasticsearch.index.IndexNotFoundException -import org.elasticsearch.index.query.QueryShardException +import org.elastic4play.{ AttributeCheckingError, AuthenticationError, AuthorizationError, BadRequestError, CreateError, ErrorWithObject, GetError, IndexNotFoundException, InternalError, MultiError, NotFoundError, SearchError, UpdateError } +import org.elastic4play.JsonFormat.attributeCheckingExceptionWrites +/** + * This class handles errors. It traverses all causes of exception to find known error and shows the appropriate message + */ class ErrorHandler extends HttpErrorHandler { private[ErrorHandler] lazy val logger = Logger(getClass) def onClientError(request: RequestHeader, statusCode: Int, message: String): Future[Result] = Future.successful { @@ -31,12 +32,12 @@ class ErrorHandler extends HttpErrorHandler { case nfe: NumberFormatException ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "NumberFormatException", "message" → ("Invalid format " + nfe.getMessage))) case NotFoundError(message) ⇒ Some(Status.NOT_FOUND → Json.obj("type" → "NotFoundError", "message" → message)) case BadRequestError(message) ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "BadRequest", "message" → message)) - case SearchError(message, cause) ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "SearchError", "message" → s"$message (${cause.getMessage})")) + case SearchError(message) ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "SearchError", "message" → s"$message")) case ace: AttributeCheckingError ⇒ Some(Status.BAD_REQUEST → Json.toJson(ace)) case iae: IllegalArgumentException ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "IllegalArgument", "message" → iae.getMessage)) - case _: NoNodeAvailableException ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "NoNodeAvailable", "message" → "ElasticSearch cluster is unreachable")) + case _: ConnectException ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "NoNodeAvailable", "message" → "ElasticSearch cluster is unreachable")) case CreateError(_, message, attributes) ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "CreateError", "message" → message, "object" → attributes)) - case ErrorWithObject(tpe, message, obj) ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → tpe, "message" → message, "object" → obj)) + case ErrorWithObject(tpe, message, obj) ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → tpe, "message" → message, "object" → obj)) case GetError(message) ⇒ Some(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → "GetError", "message" → message)) case MultiError(message, exceptions) ⇒ val suberrors = exceptions.map(e ⇒ toErrorResult(e)).collect { @@ -45,16 +46,17 @@ class ErrorHandler extends HttpErrorHandler { Some(Status.MULTI_STATUS → Json.obj("type" → "MultiError", "error" → message, "suberrors" → suberrors)) case JobNotFoundError(jobId) ⇒ Some(Status.NOT_FOUND → Json.obj("type" → "JobNotFoundError", "message" → s"Job $jobId not found")) case WorkerNotFoundError(analyzerId) ⇒ Some(Status.NOT_FOUND → Json.obj("type" → "AnalyzerNotFoundError", "message" → s"analyzer $analyzerId not found")) - case _: IndexNotFoundException ⇒ Some(520 → JsNull) - case qse: QueryShardException ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "Invalid search query", "message" → qse.getMessage)) - case t: Throwable ⇒ Option(t.getCause).flatMap(toErrorResult) + case IndexNotFoundException ⇒ Some(520 → JsNull) + case t: Throwable ⇒ Option(t.getCause).flatMap(toErrorResult) } } def toResult[C](status: Int, c: C)(implicit writeable: Writeable[C]) = Result(header = ResponseHeader(status), body = writeable.toEntity(c)) def onServerError(request: RequestHeader, exception: Throwable): Future[Result] = { - val (status, body) = toErrorResult(exception).getOrElse(Status.INTERNAL_SERVER_ERROR → Json.obj("type" → exception.getClass.getName, "message" → exception.getMessage)) + val (status, body) = toErrorResult(exception).getOrElse( + Status.INTERNAL_SERVER_ERROR → Json.obj("type" → exception.getClass.getName, "message" → exception.getMessage) + ) logger.info(s"${request.method} ${request.uri} returned $status", exception) Future.successful(toResult(status, body)) } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 364d0ce38..d08cccb79 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -18,7 +18,7 @@ object Dependencies { val reflections = "org.reflections" % "reflections" % "0.9.11" val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2" - val elastic4play = "org.thehive-project" %% "elastic4play" % "1.10.0" + val elastic4play = "org.thehive-project" %% "elastic4play" % "1.11.0" val dockerClient = "com.spotify" % "docker-client" % "8.14.4" }