Merge branch 'release/3.0.0-RC1'

CHANGELOG.md

@@ -1,7 +1,28 @@
 # Change Log
 
-## [2.1.3](https://github.com/TheHive-Project/Cortex/tree/2.1.3)
+## [3.0.0-RC1](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC1) (2019-04-05)
 
+[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.3...3.0.0-RC1)
+
+**Implemented enhancements:**
+
+- Remove size limitations [\#178](https://github.com/TheHive-Project/Cortex/issues/178)
+- Collapse job error messages by default in job history [\#171](https://github.com/TheHive-Project/Cortex/issues/171)
+- Update Copyright with year 2019 [\#168](https://github.com/TheHive-Project/Cortex/issues/168)
+
+**Fixed bugs:**
+
+- SSO: Authentication module not found  [\#181](https://github.com/TheHive-Project/Cortex/issues/181)
+- Akka Dispatcher Blocked [\#170](https://github.com/TheHive-Project/Cortex/issues/170)
+
+**Closed issues:**
+
+- Use files to communicate with analyzer/responder [\#176](https://github.com/TheHive-Project/Cortex/issues/176)
+- Provide analyzers and responders packaged with docker [\#175](https://github.com/TheHive-Project/Cortex/issues/175)
+- Single sign-on support for Cortex [\#165](https://github.com/TheHive-Project/Cortex/issues/165)
+- File extraction [\#120](https://github.com/TheHive-Project/Cortex/issues/120)
+
+## [2.1.3](https://github.com/TheHive-Project/Cortex/tree/2.1.3) (2018-12-20)
 [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.2...2.1.3)
 
 **Implemented enhancements:**

app/org/thp/cortex/Module.scala

@@ -1,20 +1,23 @@
 package org.thp.cortex
+
 import com.google.inject.AbstractModule
 import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder }
 import play.api.libs.concurrent.AkkaGuiceSupport
 import play.api.{ Configuration, Environment, Logger, Mode }
-
 import scala.collection.JavaConverters._
+
 import com.google.inject.name.Names
 import org.reflections.Reflections
 import org.reflections.scanners.SubTypesScanner
 import org.reflections.util.ConfigurationBuilder
 import org.thp.cortex.models.{ AuditedModel, Migration }
-import org.thp.cortex.services.{ AuditActor, CortexAuthSrv, UserSrv }
+import org.thp.cortex.services._
+
 import org.elastic4play.models.BaseModelDef
 import org.elastic4play.services.auth.MultiAuthSrv
 import org.elastic4play.services.{ AuthSrv, MigrationOperations }
 import org.thp.cortex.controllers.{ AssetCtrl, AssetCtrlDev, AssetCtrlProd }
+import services.mappers.{ MultiUserMapperSrv, UserMapper }
 
 class Module(environment: Environment, configuration: Configuration) extends AbstractModule with ScalaModule with AkkaGuiceSupport {
 
@@ -50,16 +53,26 @@ class Module(environment: Environment, configuration: Configuration) extends Abs
       .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers) || c.isMemberClass)
       .filterNot(c ⇒ c == classOf[MultiAuthSrv] || c == classOf[CortexAuthSrv])
       .foreach { authSrvClass ⇒
+        logger.info(s"Loading authentication module $authSrvClass")
         authBindings.addBinding.to(authSrvClass)
       }
 
+    val ssoMapperBindings = ScalaMultibinder.newSetBinder[UserMapper](binder)
+    reflectionClasses
+      .getSubTypesOf(classOf[UserMapper])
+      .asScala
+      .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers) || c.isMemberClass)
+      .filterNot(c ⇒ c == classOf[MultiUserMapperSrv])
+      .foreach(mapperCls ⇒ ssoMapperBindings.addBinding.to(mapperCls))
+
     if (environment.mode == Mode.Prod)
       bind[AssetCtrl].to[AssetCtrlProd]
     else
       bind[AssetCtrl].to[AssetCtrlDev]
 
     bind[org.elastic4play.services.UserSrv].to[UserSrv]
     bind[Int].annotatedWith(Names.named("databaseVersion")).toInstance(models.modelVersion)
+    bind[UserMapper].to[MultiUserMapperSrv]
 
     bind[AuthSrv].to[CortexAuthSrv]
     bind[MigrationOperations].to[Migration]

app/org/thp/cortex/controllers/AnalyzerCtrl.scala

@@ -1,18 +1,16 @@
 package org.thp.cortex.controllers
 
-import javax.inject.{ Inject, Singleton }
 import scala.concurrent.{ ExecutionContext, Future }
 
-import play.api.libs.json.{ JsNumber, JsObject, JsString, Json }
+import play.api.libs.json.{ JsObject, JsString, Json }
 import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
 
 import akka.stream.Materializer
-import akka.stream.scaladsl.Sink
-import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition }
+import javax.inject.{ Inject, Singleton }
+import org.thp.cortex.models.{ Roles, Worker }
 import org.thp.cortex.services.{ UserSrv, WorkerSrv }
 
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
-import org.elastic4play.models.JsonFormat.baseModelEntityWrites
 import org.elastic4play.services.JsonFormat.queryReads
 import org.elastic4play.services.{ QueryDSL, QueryDef }
 
@@ -33,75 +31,42 @@ class AnalyzerCtrl @Inject() (
     val sort = request.body.getStrings("sort").getOrElse(Nil)
     val isAdmin = request.roles.contains(Roles.orgAdmin)
     val (analyzers, analyzerTotal) = workerSrv.findAnalyzersForUser(request.userId, query, range, sort)
-    val enrichedAnalyzers = analyzers.mapAsync(2)(analyzerJson(isAdmin))
-    renderer.toOutput(OK, enrichedAnalyzers, analyzerTotal)
+    renderer.toOutput(OK, analyzers.map(analyzerJson(isAdmin)), analyzerTotal)
   }
 
   def get(analyzerId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒
     val isAdmin = request.roles.contains(Roles.orgAdmin)
     workerSrv.getForUser(request.userId, analyzerId)
-      .flatMap(analyzerJson(isAdmin))
-      .map(renderer.toOutput(OK, _))
-  }
-
-  private val emptyAnalyzerDefinitionJson = Json.obj(
-    "version" → "0.0",
-    "description" → "unknown",
-    "dataTypeList" → Nil,
-    "author" → "unknown",
-    "url" → "unknown",
-    "license" → "unknown")
-
-  private def analyzerJson(analyzer: Worker, analyzerDefinition: Option[WorkerDefinition]) = {
-    analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad ⇒
-      Json.obj(
-        "maxTlp" → (analyzer.config \ "max_tlp").asOpt[JsNumber],
-        "maxPap" → (analyzer.config \ "max_pap").asOpt[JsNumber],
-        "version" → ad.version,
-        "description" → ad.description,
-        "author" → ad.author,
-        "url" → ad.url,
-        "license" → ad.license,
-        "baseConfig" → ad.baseConfiguration)
-    } + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId())) // For compatibility reason
+      .map(a ⇒ renderer.toOutput(OK, analyzerJson(isAdmin)(a)))
   }
 
-  private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): Future[JsObject] = {
-    workerSrv.getDefinition(analyzer.workerDefinitionId())
-      .map(analyzerDefinition ⇒ analyzerJson(analyzer, Some(analyzerDefinition)))
-      .recover { case _ ⇒ analyzerJson(analyzer, None) }
-      .map {
-        case a if isAdmin ⇒ a + ("configuration" → Json.parse(analyzer.configuration()))
-        case a            ⇒ a
-      }
+  private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): JsObject = {
+    if (isAdmin)
+      analyzer.toJson + ("configuration" → Json.parse(analyzer.configuration())) + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId()))
+    else
+      analyzer.toJson + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId()))
   }
 
   def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒
     import org.elastic4play.services.QueryDSL._
-    workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
-      ._1
-      .mapAsyncUnordered(2) { analyzer ⇒
-        workerSrv.getDefinition(analyzer.workerDefinitionId())
-          .map(ad ⇒ analyzerJson(analyzer, Some(ad)))
-      }
-      .runWith(Sink.seq)
-      .map(analyzers ⇒ renderer.toOutput(OK, analyzers))
+    val (responderList, responderCount) = workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
+    renderer.toOutput(OK, responderList.map(analyzerJson(isAdmin = false)), responderCount)
   }
 
   def create(analyzerDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒
     for {
       organizationId ← userSrv.getOrganizationId(request.userId)
-      workerDefinition ← workerSrv.getDefinition(analyzerDefinitionId)
+      workerDefinition ← Future.fromTry(workerSrv.getDefinition(analyzerDefinitionId))
       analyzer ← workerSrv.create(organizationId, workerDefinition, request.body)
-    } yield renderer.toOutput(CREATED, analyzerJson(analyzer, Some(workerDefinition)))
+    } yield renderer.toOutput(CREATED, analyzerJson(isAdmin = false)(analyzer))
   }
 
-  def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒
+  def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { _ ⇒
     val (analyzers, analyzerTotal) = workerSrv.listAnalyzerDefinitions
     renderer.toOutput(OK, analyzers, analyzerTotal)
   }
 
-  def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒
+  def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { _ ⇒
     workerSrv.rescan()
     NoContent
   }
@@ -117,7 +82,6 @@ class AnalyzerCtrl @Inject() (
     for {
       analyzer ← workerSrv.getForUser(request.userId, analyzerId)
       updatedAnalyzer ← workerSrv.update(analyzer, request.body)
-      updatedAnalyzerJson ← analyzerJson(isAdmin = true)(updatedAnalyzer)
-    } yield renderer.toOutput(OK, updatedAnalyzerJson)
+    } yield renderer.toOutput(OK, analyzerJson(isAdmin = true)(updatedAnalyzer))
   }
 }

app/org/thp/cortex/controllers/AuthenticationCtrl.scala

@@ -1,18 +1,18 @@
 package org.thp.cortex.controllers
 
-import javax.inject.{ Inject, Singleton }
-
 import scala.concurrent.{ ExecutionContext, Future }
 
 import play.api.mvc._
 
+import javax.inject.{ Inject, Singleton }
+import org.thp.cortex.models.UserStatus
 import org.thp.cortex.services.UserSrv
 
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
 import org.elastic4play.database.DBIndex
 import org.elastic4play.services.AuthSrv
-import org.elastic4play.{ MissingAttributeError, Timed }
 import org.elastic4play.services.JsonFormat.authContextWrites
+import org.elastic4play.{ AuthorizationError, MissingAttributeError, OAuth2Redirect, Timed }
 
 @Singleton
 class AuthenticationCtrl @Inject() (
@@ -38,6 +38,27 @@ class AuthenticationCtrl @Inject() (
     }
   }
 
+  @Timed
+  def ssoLogin: Action[AnyContent] = Action.async { implicit request ⇒
+    dbIndex.getIndexStatus.flatMap {
+      case false ⇒ Future.successful(Results.Status(520))
+      case _ ⇒
+        (for {
+          authContext ← authSrv.authenticate()
+          user ← userSrv.get(authContext.userId)
+        } yield {
+          if (user.status() == UserStatus.Ok)
+            authenticated.setSessingUser(Ok, authContext)
+          else
+            throw AuthorizationError("Your account is locked")
+        }) recover {
+          // A bit of a hack with the status code, so that Angular doesn't reject the origin
+          case OAuth2Redirect(redirectUrl, qp) ⇒ Redirect(redirectUrl, qp, status = OK)
+          case e                               ⇒ throw e
+        }
+    }
+  }
+
   @Timed
   def logout = Action {
     Ok.withNewSession

app/org/thp/cortex/controllers/JobCtrl.scala

@@ -1,31 +1,31 @@
 package org.thp.cortex.controllers
 
-import javax.inject.{ Inject, Named, Singleton }
+import scala.concurrent.duration.{ Duration, FiniteDuration }
+import scala.concurrent.{ ExecutionContext, Future }
+
+import play.api.http.Status
+import play.api.libs.json.{ JsObject, JsString, JsValue, Json }
+import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
 
 import akka.actor.{ ActorRef, ActorSystem }
 import akka.pattern.ask
 import akka.stream.Materializer
 import akka.stream.scaladsl.Sink
 import akka.util.Timeout
+import javax.inject.{ Inject, Named, Singleton }
+import org.thp.cortex.models.{ Job, JobStatus, Roles }
+import org.thp.cortex.services.AuditActor.{ JobEnded, Register }
+import org.thp.cortex.services.JobSrv
+
 import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
 import org.elastic4play.models.JsonFormat.baseModelEntityWrites
 import org.elastic4play.services.JsonFormat.queryReads
 import org.elastic4play.services.{ QueryDSL, QueryDef }
 import org.elastic4play.utils.RichFuture
-import org.thp.cortex.models.{ Job, JobStatus, Roles }
-import org.thp.cortex.services.AuditActor.{ JobEnded, Register }
-import org.thp.cortex.services.{ JobSrv, UserSrv }
-import play.api.http.Status
-import play.api.libs.json.{ JsObject, JsString, JsValue, Json }
-import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
-
-import scala.concurrent.duration.{ Duration, FiniteDuration }
-import scala.concurrent.{ ExecutionContext, Future }
 
 @Singleton
 class JobCtrl @Inject() (
     jobSrv: JobSrv,
-    userSrv: UserSrv,
     @Named("audit") auditActor: ActorRef,
     fieldsBodyParser: FieldsBodyParser,
     authenticated: Authenticated,

app/org/thp/cortex/controllers/ResponderCtrl.scala

@@ -6,7 +6,6 @@ import play.api.libs.json.{ JsNumber, JsObject, Json }
 import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }
 
 import akka.stream.Materializer
-import akka.stream.scaladsl.Sink
 import javax.inject.{ Inject, Singleton }
 import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition }
 import org.thp.cortex.services.{ UserSrv, WorkerSrv }
@@ -33,15 +32,13 @@ class ResponderCtrl @Inject() (
     val sort = request.body.getStrings("sort").getOrElse(Nil)
     val isAdmin = request.roles.contains(Roles.orgAdmin)
     val (responders, responderTotal) = workerSrv.findRespondersForUser(request.userId, query, range, sort)
-    val enrichedResponders = responders.mapAsync(2)(responderJson(isAdmin))
-    renderer.toOutput(OK, enrichedResponders, responderTotal)
+    renderer.toOutput(OK, responders.map(responderJson(isAdmin)), responderTotal)
   }
 
   def get(responderId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒
     val isAdmin = request.roles.contains(Roles.orgAdmin)
     workerSrv.getForUser(request.userId, responderId)
-      .flatMap(responderJson(isAdmin))
-      .map(renderer.toOutput(OK, _))
+      .map(responder ⇒ renderer.toOutput(OK, responderJson(isAdmin)(responder)))
   }
 
   private val emptyResponderDefinitionJson = Json.obj(
@@ -66,42 +63,33 @@ class ResponderCtrl @Inject() (
     }
   }
 
-  private def responderJson(isAdmin: Boolean)(responder: Worker): Future[JsObject] = {
-    workerSrv.getDefinition(responder.workerDefinitionId())
-      .map(responderDefinition ⇒ responderJson(responder, Some(responderDefinition)))
-      .recover { case _ ⇒ responderJson(responder, None) }
-      .map {
-        case a if isAdmin ⇒ a + ("configuration" → Json.parse(responder.configuration()))
-        case a            ⇒ a
-      }
+  private def responderJson(isAdmin: Boolean)(responder: Worker): JsObject = {
+    if (isAdmin)
+      responder.toJson + ("configuration" → Json.parse(responder.configuration()))
+    else
+      responder.toJson
   }
 
   def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒
     import org.elastic4play.services.QueryDSL._
-    workerSrv.findRespondersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
-      ._1
-      .mapAsyncUnordered(2) { responder ⇒
-        workerSrv.getDefinition(responder.workerDefinitionId())
-          .map(ad ⇒ responderJson(responder, Some(ad)))
-      }
-      .runWith(Sink.seq)
-      .map(responders ⇒ renderer.toOutput(OK, responders))
+    val (responderList, responderCount) = workerSrv.findRespondersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
+    renderer.toOutput(OK, responderList.map(responderJson(false)), responderCount)
   }
 
   def create(responderDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒
     for {
       organizationId ← userSrv.getOrganizationId(request.userId)
-      workerDefinition ← workerSrv.getDefinition(responderDefinitionId)
+      workerDefinition ← Future.fromTry(workerSrv.getDefinition(responderDefinitionId))
       responder ← workerSrv.create(organizationId, workerDefinition, request.body)
     } yield renderer.toOutput(CREATED, responderJson(responder, Some(workerDefinition)))
   }
 
-  def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒
+  def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { _ ⇒
     val (responders, responderTotal) = workerSrv.listResponderDefinitions
     renderer.toOutput(OK, responders, responderTotal)
   }
 
-  def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒
+  def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { _ ⇒
     workerSrv.rescan()
     NoContent
   }
@@ -117,7 +105,6 @@ class ResponderCtrl @Inject() (
     for {
       responder ← workerSrv.getForUser(request.userId, responderId)
       updatedResponder ← workerSrv.update(responder, request.body)
-      updatedResponderJson ← responderJson(isAdmin = true)(updatedResponder)
-    } yield renderer.toOutput(OK, updatedResponderJson)
+    } yield renderer.toOutput(OK, responderJson(isAdmin = true)(updatedResponder))
   }
 }