Merge branch 'release/3.2.0-rc1'

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Change Log
 
+## [3.2.0-RC1](https://github.com/TheHive-Project/TheHive/tree/3.2.0-RC1)
+
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.1.2...3.2.0-RC1)
+
+**Implemented enhancements:**
+
+- Add ability to add a log in responder operation [\#795](https://github.com/TheHive-Project/TheHive/issues/795)
+- Add responder actions in dashboard [\#794](https://github.com/TheHive-Project/TheHive/issues/794)
+- Show observable description on mouseover observables [\#793](https://github.com/TheHive-Project/TheHive/issues/793)
+- Update Play [\#791](https://github.com/TheHive-Project/TheHive/issues/791)
+- Show tags of observables in Alert preview [\#778](https://github.com/TheHive-Project/TheHive/issues/778)
+- Observable Value gets cleared when changing its type \(importing it from an analyser result\) [\#763](https://github.com/TheHive-Project/TheHive/issues/763)
+- Add confirmation dialogs when running a responder [\#762](https://github.com/TheHive-Project/TheHive/issues/762)
+- Whitelist of tags for MISP alerts [\#481](https://github.com/TheHive-Project/TheHive/issues/481)
+
+**Fixed bugs:**
+
+- MISP synchronization fails if event contains attachment with invalid name [\#801](https://github.com/TheHive-Project/TheHive/issues/801)
+- Observable creation doesn't allow multiline observables [\#790](https://github.com/TheHive-Project/TheHive/issues/790)
+- A user with "write" permission can delete a case using API [\#773](https://github.com/TheHive-Project/TheHive/issues/773)
+- Basic authentication method should be disabled by default [\#772](https://github.com/TheHive-Project/TheHive/issues/772)
+- Case search from dashboard clic "invalid filters error" [\#761](https://github.com/TheHive-Project/TheHive/issues/761)
+- Intermittently losing Cortex [\#739](https://github.com/TheHive-Project/TheHive/issues/739)
+
+**Merged pull requests:**
+
+- Added Integration with FireEye iSIGHT [\#755](https://github.com/TheHive-Project/TheHive/pull/755) ([garanews](https://github.com/garanews))
+
 ## [3.1.2](https://github.com/TheHive-Project/TheHive/tree/3.1.2) (2018-10-12)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.1.1...3.1.2)
 

README.md

@@ -81,6 +81,9 @@ TheHive Project provides [DigitalShadows2TH](https://github.com/TheHive-Project/
 ### Integration with Zerofox
 [Zerofox2TH](https://github.com/TheHive-Project/Zerofox2TH) is a free, open source [ZeroFOX](https://www.zerofox.com/) alert feeder for TheHive, written by TheHive Project. You can use it to feed ZeroFOX alerts into TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
 
+### Integration with FireEye iSIGHT
+[FireEye2TH](https://github.com/LDO-CERT/FireEye2TH) is a free, open source [FireEye iSIGHT](https://www.fireeye.com/) alert feeder for TheHive, written by LDO-CERT. You can use it to feed FireEye iSIGHT alerts into TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
+
 # License
 TheHive is an open source and free software released under the [AGPL](https://github.com/TheHive-Project/TheHive/blob/master/LICENSE) (Affero General Public License). We, TheHive Project, are committed to ensure that TheHive will remain a free and open source project on the long-run.
 

project/Dependencies.scala

@@ -20,7 +20,7 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.6.3"
+    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.7.0"
     val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.11"
     val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.11"
   }

project/plugins.sbt

@@ -1,7 +1,7 @@
 // Comment to get more information during initialization
 logLevel := Level.Info
 
-addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.18")
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.20")
 
 addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.1")
 

thehive-backend/app/controllers/CaseCtrl.scala

@@ -88,13 +88,13 @@ class CaseCtrl @Inject() (
   }
 
   @Timed
-  def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒
+  def delete(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒
     caseSrv.delete(id)
       .map(_ ⇒ NoContent)
   }
 
   @Timed
-  def realDelete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒
+  def realDelete(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒
     caseSrv.realDelete(id)
       .map(_ ⇒ NoContent)
   }

thehive-backend/app/controllers/DescribeCtrl.scala

@@ -40,7 +40,7 @@ class DescribeCtrl @Inject() (
       .getOrElse(NotFound(s"Model $modelName not found"))
   }
 
-  private val allModels: Seq[String] = Seq("case", "case_artifact", "case_task", "case_task_log", "alert", "case_artifact_job", "audit")
+  private val allModels: Seq[String] = Seq("case", "case_artifact", "case_task", "case_task_log", "alert", "case_artifact_job", "audit", "action")
   def describeAll: Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒
     val entityDefinitions = modelSrv.list
       .collect {

thehive-backend/conf/reference.conf

@@ -57,7 +57,16 @@ datastore {
 }
 
 auth {
-	# "type" parameter contains authentication provider. It can be multi-valued (useful for migration)
+	method {
+		# basic authentication is disabled by default
+		basic = false
+		# authentication using API key
+		key = true
+		# authentication using a client x509 certificate
+		pki = true
+	}
+
+	# "provider" parameter contains authentication provider. It is multi-valued (useful for migration)
 	# available auth types are:
 	# services.LocalAuthSrv : passwords are stored in user entity (in ElasticSearch). No configuration are required.
 	# ad : use ActiveDirectory to authenticate users. Configuration is under "auth.ad" key

thehive-cortex/app/connectors/cortex/services/ActionOperation.scala

@@ -8,10 +8,11 @@ import play.api.libs.json._
 import akka.actor.ActorSystem
 import akka.stream.Materializer
 import akka.stream.scaladsl.Sink
+import connectors.cortex.services.ActionOperationStatus.Type
 import javax.inject.{ Inject, Provider, Singleton }
 import models._
 import org.elasticsearch.index.engine.VersionConflictEngineException
-import services.{ AlertSrv, ArtifactSrv, CaseSrv, TaskSrv }
+import services._
 
 import org.elastic4play.controllers.Fields
 import org.elastic4play.database.ModifyConfig
@@ -33,27 +34,31 @@ trait ActionOperation {
 }
 
 case class AddTagToCase(tag: String, status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
-  def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): AddTagToCase = copy(status = newStatus, message = newMessage)
+  override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): AddTagToCase = copy(status = newStatus, message = newMessage)
 }
 
 case class AddTagToArtifact(tag: String, status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
-  def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): AddTagToArtifact = copy(status = newStatus, message = newMessage)
+  override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): AddTagToArtifact = copy(status = newStatus, message = newMessage)
 }
 
 case class CreateTask(fields: JsObject, status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
-  def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): CreateTask = copy(status = newStatus, message = newMessage)
+  override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): CreateTask = copy(status = newStatus, message = newMessage)
 }
 
 case class AddCustomFields(name: String, tpe: String, value: JsValue, status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
   override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): AddCustomFields = copy(status = newStatus, message = newMessage)
 }
 
 case class CloseTask(status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
-  def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): CloseTask = copy(status = newStatus, message = newMessage)
+  override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): CloseTask = copy(status = newStatus, message = newMessage)
 }
 
 case class MarkAlertAsRead(status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
-  def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): MarkAlertAsRead = copy(status = newStatus, message = newMessage)
+  override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): MarkAlertAsRead = copy(status = newStatus, message = newMessage)
+}
+
+case class AddLogToTask(content: String, owner: Option[String], status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
+  override def updateStatus(newStatus: Type, newMessage: String): ActionOperation = copy(status = newStatus, message = newMessage)
 }
 
 object ActionOperation {
@@ -63,6 +68,7 @@ object ActionOperation {
   val addCustomFieldsWrites = Json.writes[AddCustomFields]
   val closeTaskWrites = Json.writes[CloseTask]
   val markAlertAsReadWrites = Json.writes[MarkAlertAsRead]
+  val addLogToTaskWrites = Json.writes[AddLogToTask]
   implicit val actionOperationReads: Reads[ActionOperation] = Reads[ActionOperation](json ⇒
     (json \ "type").asOpt[String].fold[JsResult[ActionOperation]](JsError("type is missing in action operation")) {
       case "AddTagToCase"     ⇒ (json \ "tag").validate[String].map(tag ⇒ AddTagToCase(tag))
@@ -75,7 +81,11 @@ object ActionOperation {
       } yield AddCustomFields(name, tpe, value)
       case "CloseTask"       ⇒ JsSuccess(CloseTask())
       case "MarkAlertAsRead" ⇒ JsSuccess(MarkAlertAsRead())
-      case other             ⇒ JsError(s"Unknown operation $other")
+      case "AddLogToTask" ⇒ for {
+        content ← (json \ "content").validate[String]
+        owner ← (json \ "owner").validateOpt[String]
+      } yield AddLogToTask(content, owner)
+      case other ⇒ JsError(s"Unknown operation $other")
     })
   implicit val actionOperationWrites: Writes[ActionOperation] = Writes[ActionOperation] {
     case a: AddTagToCase     ⇒ addTagToCaseWrites.writes(a)
@@ -84,6 +94,7 @@ object ActionOperation {
     case a: AddCustomFields  ⇒ addCustomFieldsWrites.writes(a)
     case a: CloseTask        ⇒ closeTaskWrites.writes(a)
     case a: MarkAlertAsRead  ⇒ markAlertAsReadWrites.writes(a)
+    case a: AddLogToTask     ⇒ addLogToTaskWrites.writes(a)
     case a                   ⇒ Json.obj("unsupported operation" → a.toString)
   }
 }
@@ -92,6 +103,7 @@ object ActionOperation {
 class ActionOperationSrv @Inject() (
     caseSrv: CaseSrv,
     taskSrv: TaskSrv,
+    logSrv: LogSrv,
     alertSrvProvider: Provider[AlertSrv],
     findSrv: FindSrv,
     artifactSrv: ArtifactSrv,
@@ -169,6 +181,11 @@ class ActionOperationSrv @Inject() (
               case alert: Alert ⇒ alertSrv.markAsRead(alert).map(_ ⇒ operation.updateStatus(ActionOperationStatus.Success, ""))
               case _            ⇒ Future.failed(BadRequestError("Alert not found"))
             }
+          case AddLogToTask(content, owner, _, _) ⇒
+            for {
+              task ← findTaskEntity(entity)
+              _ ← logSrv.create(task, Fields.empty.set("message", content).set("owner", owner.map(JsString)))
+            } yield operation.updateStatus(ActionOperationStatus.Success, "")
           case o ⇒ Future.successful(operation.updateStatus(ActionOperationStatus.Failure, s"Operation $o not supported"))
         }
       }

thehive-cortex/app/connectors/cortex/services/CortexAnalyzerSrv.scala

@@ -25,7 +25,7 @@ import org.elastic4play.controllers.Fields
 import org.elastic4play.database.{ DBRemove, ModifyConfig }
 import org.elastic4play.services.JsonFormat.attachmentFormat
 import org.elastic4play.services._
-import org.elastic4play.utils.RetryOnError
+import org.elastic4play.utils.Retry
 import org.elastic4play.{ InternalError, NotFoundError }
 
 @Singleton
@@ -226,7 +226,7 @@ class CortexAnalyzerSrv @Inject() (
         .toOption
         .flatMap(r ⇒ (r \ "summary").asOpt[JsObject])
         .map { jobSummary ⇒
-          RetryOnError() {
+          Retry()(classOf[Exception]) {
             for {
               artifact ← artifactSrv.get(job.artifactId())
               reports = Try(Json.parse(artifact.reports()).asOpt[JsObject]).toOption.flatten.getOrElse(JsObject.empty)

thehive-cortex/app/connectors/cortex/services/CortexClient.scala

@@ -10,7 +10,6 @@ import play.api.libs.json.{ JsObject, JsValue, Json }
 import play.api.libs.ws.{ WSAuthScheme, WSRequest, WSResponse }
 import play.api.mvc.MultipartFormData.{ DataPart, FilePart }
 
-import akka.actor.ActorSystem
 import akka.stream.scaladsl.Source
 import connectors.cortex.models.JsonFormat._
 import connectors.cortex.models._
@@ -19,7 +18,6 @@ import models.HealthStatus
 import services.CustomWSAPI
 
 import org.elastic4play.NotFoundError
-import org.elastic4play.utils.RichFuture
 
 object CortexConfig {
   def getCortexClient(name: String, configuration: Configuration, ws: CustomWSAPI): Option[CortexClient] = {
@@ -187,27 +185,25 @@ class CortexClient(val name: String, baseUrl: String, authentication: Option[Cor
     request(s"api/job/$jobId/waitreport", _.withQueryStringParameters("atMost" → atMost.toString).get, _.json.as[JsObject])
   }
 
-  def getVersion()(implicit system: ActorSystem, ec: ExecutionContext): Future[Option[String]] = {
+  def getVersion()(implicit ec: ExecutionContext): Future[Option[String]] = {
     request("api/status", _.get, identity)
       .map {
         case resp if resp.status / 100 == 2 ⇒ (resp.json \ "versions" \ "Cortex").asOpt[String]
         case _                              ⇒ None
       }
       .recover { case _ ⇒ None }
-      .withTimeout(1.seconds, None)
   }
 
-  def getCurrentUser()(implicit system: ActorSystem, ec: ExecutionContext): Future[Option[String]] = {
+  def getCurrentUser()(implicit ec: ExecutionContext): Future[Option[String]] = {
     request("api/user/current", _.get, identity)
       .map {
         case resp if resp.status / 100 == 2 ⇒ (resp.json \ "id").asOpt[String]
         case _                              ⇒ None
       }
       .recover { case _ ⇒ None }
-      .withTimeout(1.seconds, None)
   }
 
-  def status()(implicit system: ActorSystem, ec: ExecutionContext): Future[JsObject] =
+  def status()(implicit ec: ExecutionContext): Future[JsObject] =
     for {
       version ← getVersion()
       versionValue = version.getOrElse("")
@@ -222,7 +218,7 @@ class CortexClient(val name: String, baseUrl: String, authentication: Option[Cor
         "status" → status)
     }
 
-  def health()(implicit system: ActorSystem, ec: ExecutionContext): Future[HealthStatus.Type] = {
+  def health()(implicit ec: ExecutionContext): Future[HealthStatus.Type] = {
     getVersion()
       .map {
         case None ⇒ HealthStatus.Error

thehive-misp/app/connectors/misp/MispConfig.scala

@@ -33,14 +33,15 @@ class MispConfig(val interval: FiniteDuration, val connections: Seq[MispConnecti
       maxSize = mispConnectionConfig.getOptional[ConfigMemorySize]("max-size").map(_.toBytes)
       excludedOrganisations = mispConnectionConfig.getOptional[Seq[String]]("exclusion.organisation").getOrElse(Nil)
       excludedTags = mispConnectionConfig.getOptional[Seq[String]]("exclusion.tags").fold(Set.empty[String])(_.toSet)
+      whitelistTags = mispConnectionConfig.getOptional[Seq[String]]("whitelist.tags").fold(Set.empty[String])(_.toSet)
       purpose = mispConnectionConfig.getOptional[String]("purpose")
         .fold(MispPurpose.ImportAndExport) { purposeName ⇒
           Try(MispPurpose.withName(purposeName)).getOrElse {
             Logger(getClass).error(s"Incorrect value for MISP purpose ($name.purpose), one of (${MispPurpose.values.mkString(", ")}) was expected. Using default value: ImportAndExport ")
             MispPurpose.ImportAndExport
           }
         }
-    } yield MispConnection(name, url, key, instanceWS, caseTemplate, artifactTags, maxAge, maxAttributes, maxSize, excludedOrganisations, excludedTags, purpose))
+    } yield MispConnection(name, url, key, instanceWS, caseTemplate, artifactTags, maxAge, maxAttributes, maxSize, excludedOrganisations, excludedTags, whitelistTags, purpose))
 
   @Inject def this(configuration: Configuration, httpSrv: CustomWSAPI) =
     this(

thehive-misp/app/connectors/misp/MispConnection.scala

@@ -9,12 +9,9 @@ import play.api.Logger
 import play.api.libs.json.{ JsObject, Json }
 import play.api.libs.ws.WSRequest
 
-import akka.actor.ActorSystem
 import models.HealthStatus
 import services.CustomWSAPI
 
-import org.elastic4play.utils.RichFuture
-
 object MispPurpose extends Enumeration {
   val ImportOnly, ExportOnly, ImportAndExport = Value
 }
@@ -30,6 +27,7 @@ case class MispConnection(
     maxSize: Option[Long],
     excludedOrganisations: Seq[String],
     excludedTags: Set[String],
+    whitelistTags: Set[String],
     purpose: MispPurpose.Value) {
 
   private[MispConnection] lazy val logger = Logger(getClass)
@@ -46,6 +44,8 @@ case class MispConnection(
        |    max size:       ${maxSize.getOrElse("<not set>")}
        |    excluded orgs:  ${excludedOrganisations.mkString}
        |    excluded tags:  ${excludedTags.mkString}
+       |    whitelist tags: ${whitelistTags.mkString}
+       |  purpose:          $purpose
        |""".stripMargin)
 
   private[misp] def apply(url: String): WSRequest =
@@ -71,7 +71,11 @@ case class MispConnection(
   }
 
   def isExcluded(event: MispAlert): Boolean = {
-    if (excludedOrganisations.contains(event.source)) {
+    if (whitelistTags.nonEmpty && whitelistTags.intersect(event.tags.toSet).isEmpty) {
+      logger.debug(s"event ${event.sourceRef} is ignored because it doesn't contain any of whitelist tags (${whitelistTags.mkString(",")})")
+      true
+    }
+    else if (excludedOrganisations.contains(event.source)) {
       logger.debug(s"event ${event.sourceRef} is ignored because its organisation (${event.source}) is excluded")
       true
     }
@@ -85,17 +89,16 @@ case class MispConnection(
     }
   }
 
-  def getVersion()(implicit system: ActorSystem, ec: ExecutionContext): Future[Option[String]] = {
+  def getVersion()(implicit ec: ExecutionContext): Future[Option[String]] = {
     apply("servers/getVersion").get
       .map {
         case resp if resp.status / 100 == 2 ⇒ (resp.json \ "version").asOpt[String]
         case _                              ⇒ None
       }
       .recover { case _ ⇒ None }
-      .withTimeout(1.seconds, None)
   }
 
-  def status()(implicit system: ActorSystem, ec: ExecutionContext): Future[JsObject] = {
+  def status()(implicit ec: ExecutionContext): Future[JsObject] = {
     getVersion()
       .map {
         case Some(version) ⇒ Json.obj(
@@ -111,7 +114,7 @@ case class MispConnection(
       }
   }
 
-  def healthStatus()(implicit system: ActorSystem, ec: ExecutionContext): Future[HealthStatus.Type] = {
+  def healthStatus()(implicit ec: ExecutionContext): Future[HealthStatus.Type] = {
     getVersion()
       .map {
         case None ⇒ HealthStatus.Error

thehive-misp/app/connectors/misp/MispSrv.scala

@@ -298,7 +298,7 @@ class MispSrv @Inject() (
             None
           }
 
-        tempFile = tempSrv.newTemporaryFile("misp_malware", file.name)
+        tempFile = tempSrv.newTemporaryFile("misp", "malware")
         _ = logger.info(s"Extract malware file ${file.filepath} in file $tempFile")
         _ = zipFile.extractFile(contentFileHeader, tempFile.getParent.toString, null, tempFile.getFileName.toString)
       } yield FileInputValue(filename, tempFile, "application/octet-stream")).getOrElse(file)