Merge pull request #6 from TheHive-Project/master

Merge Master

.drone.yml

@@ -159,6 +159,7 @@ steps:
       strip_components: 3
     when:
       branch: [develop]
+      event: {exclude: [pull_request]}
 
   - name: deploy binaries in integration environment
     image: appleboy/drone-ssh
@@ -170,6 +171,7 @@ steps:
         - ./start thehive ${DRONE_BUILD_NUMBER}
     when:
       branch: [develop]
+      event: {exclude: [pull_request]}
 
   # Deploy binaries in staging environment
   - name: copy binaries in staging environment
@@ -183,6 +185,7 @@ steps:
       strip_components: 3
     when:
       branch: [master]
+      event: {exclude: [pull_request]}
 
   - name: deploy binaries in staging environment
     image: appleboy/drone-ssh
@@ -194,6 +197,7 @@ steps:
         - ./start thehive ${DRONE_BUILD_NUMBER}
     when:
       branch: [master]
+      event: {exclude: [pull_request]}
 
 volumes:
   - name: cache

AUTHORS

@@ -11,7 +11,7 @@ Contributors
 
 * CERT Banque de France (CERT-BDF)
 
-Copyright (C) 2017-2018 Nabil Adouani
-Copyright (C) 2014-2018 Thomas Franco
-Copyright (C) 2014-2018 Saâd Kadhi
-Copyright (C) 2014-2018 Jérôme Leonard
+Copyright (C) 2017-2019 Nabil Adouani
+Copyright (C) 2014-2019 Thomas Franco
+Copyright (C) 2014-2019 Saâd Kadhi
+Copyright (C) 2014-2019 Jérôme Leonard

CHANGELOG.md

@@ -1,7 +1,85 @@
 # Change Log
 
-## [3.3.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC2) (2019-02-07)
+## [3.3.0](https://github.com/TheHive-Project/TheHive/tree/3.3.0) (2019-03-19)
+
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC6...3.3.0)
+
+**Fixed bugs:**
+
+- Merge case by CaseID Broken [\#930](https://github.com/TheHive-Project/TheHive/issues/930)
+
+## [3.3.0-RC6](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC6) (2019-03-07)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC5...3.3.0-RC6)
+
+**Implemented enhancements:**
+
+- Add Tags to an Alert with Responder [\#912](https://github.com/TheHive-Project/TheHive/issues/912)
+- Dashboards - Add text widget [\#908](https://github.com/TheHive-Project/TheHive/issues/908)
+- Empty case still available when disabled [\#901](https://github.com/TheHive-Project/TheHive/issues/901)
+- Support for filtering Tags by prefix \(using asterisk, % or something\) in search dialog [\#666](https://github.com/TheHive-Project/TheHive/issues/666)
+
+**Closed issues:**
+
+- Dynamic \(auto-refresh\) of cases is break in 3.3.0-RC5 [\#907](https://github.com/TheHive-Project/TheHive/issues/907)
+- Hostname Artifact [\#900](https://github.com/TheHive-Project/TheHive/issues/900)
+- DOS issue: Firefox crashing TheHive  [\#899](https://github.com/TheHive-Project/TheHive/issues/899)
+
+## [3.3.0-RC5](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC5) (2019-02-23)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC4...3.3.0-RC5)
+
+**Implemented enhancements:**
+
+- Mouseover text for alert preview [\#897](https://github.com/TheHive-Project/TheHive/issues/897)
+
+**Fixed bugs:**
 
+- dashboard clicks are not correctly translated to tag filters [\#896](https://github.com/TheHive-Project/TheHive/issues/896)
+- Search results not visible [\#895](https://github.com/TheHive-Project/TheHive/issues/895)
+
+## [3.3.0-RC4](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC4) (2019-02-22)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC3...3.3.0-RC4)
+
+**Implemented enhancements:**
+
+- Use empty case modal when merging alerts and no templates are defined [\#893](https://github.com/TheHive-Project/TheHive/issues/893)
+
+**Fixed bugs:**
+
+- Issue with navigation from dashboard clickable donuts to search page [\#894](https://github.com/TheHive-Project/TheHive/issues/894)
+- Hide Empty Case Button Broken [\#890](https://github.com/TheHive-Project/TheHive/issues/890)
+
+## [3.3.0-RC3](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC3) (2019-02-21)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC2...3.3.0-RC3)
+
+**Implemented enhancements:**
+
+- Add a UI configuration admin section [\#888](https://github.com/TheHive-Project/TheHive/issues/888)
+- Add a Related Alerts link to case details view [\#884](https://github.com/TheHive-Project/TheHive/issues/884)
+- Update Copyright with year 2019 [\#879](https://github.com/TheHive-Project/TheHive/issues/879)
+- Provide a quick link to copy alert id [\#870](https://github.com/TheHive-Project/TheHive/issues/870)
+- \[BUG\] Audit trail for alert ignore [\#863](https://github.com/TheHive-Project/TheHive/issues/863)
+- Related artifacts: IOC/not IOC [\#838](https://github.com/TheHive-Project/TheHive/issues/838)
+- Feature: Add "auto-completion" to the UI [\#831](https://github.com/TheHive-Project/TheHive/issues/831)
+- Improvement: Upload of observables seem to fail "silently" [\#829](https://github.com/TheHive-Project/TheHive/issues/829)
+- Feature Request: link to and from Hive to MISP [\#820](https://github.com/TheHive-Project/TheHive/issues/820)
+- Disable clickable widgets in dashboard edit mode [\#485](https://github.com/TheHive-Project/TheHive/issues/485)
+- Ability to disable "New Case" -\> "Empty case" [\#449](https://github.com/TheHive-Project/TheHive/issues/449)
+
+**Fixed bugs:**
+
+- Drone build fails on pull-requests [\#882](https://github.com/TheHive-Project/TheHive/issues/882)
+- AKKA version missmatch [\#877](https://github.com/TheHive-Project/TheHive/issues/877)
+- Label Typo in Updated Alerts [\#874](https://github.com/TheHive-Project/TheHive/issues/874)
+- Log message related to MISP synchronization is confusing [\#871](https://github.com/TheHive-Project/TheHive/issues/871)
+- Cortex responders with DataType `thehive:case\_artifact` do not show up within thehive when attempting to run them for observables. [\#869](https://github.com/TheHive-Project/TheHive/issues/869)
+- Alert updates and tracking \(follow\) [\#856](https://github.com/TheHive-Project/TheHive/issues/856)
+
+**Merged pull requests:**
+
+- Update akka version [\#878](https://github.com/TheHive-Project/TheHive/pull/878) ([zpriddy](https://github.com/zpriddy))
+- Fix Update Label to Warning [\#873](https://github.com/TheHive-Project/TheHive/pull/873) ([zpriddy](https://github.com/zpriddy))
+
+## [3.3.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC2) (2019-02-07)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC1...3.3.0-RC2)
 
 **Fixed bugs:**

README.md

@@ -77,12 +77,24 @@ TheHive can be configured to import events from one or multiple [MISP](http://ww
 
 [Cortex](https://github.com/TheHive-Project/Cortex/) is the perfect companion for TheHive. Use one or several to analyze observables at scale and respond to incidents.
 
-### Integration with Digital Shadows
-TheHive Project provides [DigitalShadows2TH](https://github.com/TheHive-Project/DigitalShadows2TH), a free, open source [Digital Shadows](https://www.digitalshadows.com/) alert feeder for TheHive. You can use it to import Digital Shadows *incidents* and *intel-incidents* as alerts in TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
+### Alert Feeders by TheHive Project
 
-### Integration with Zerofox
+#### DigitalShadows2TH
+[DigitalShadows2TH](https://github.com/TheHive-Project/DigitalShadows2TH) is a free, open source [Digital Shadows](https://www.digitalshadows.com/) alert feeder for TheHive. You can use it to import Digital Shadows *incidents* and *intel-incidents* as alerts in TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
+
+#### Synapse
+[Synapse](https://github.com/TheHive-Project/Synapse) is a meta-alert feeder that allows you to centrally feed TheHive from multiple alert sources. It leverages TheHive's API to automate case and alert creation. Case creation from email or alert creation from SIEM event are typical use cases. Currently, Synapse allows you to integrate Exchange, O365 & QRadar.
+
+#### Zerofox2TH
 [Zerofox2TH](https://github.com/TheHive-Project/Zerofox2TH) is a free, open source [ZeroFOX](https://www.zerofox.com/) alert feeder for TheHive, written by TheHive Project. You can use it to feed ZeroFOX alerts into TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
 
+### Alert Feeders from the User Community
+
+### Integration with Crowdstrike Falcon (WIP)
+[Crowdstrike2TH](https://github.com/xg5-simon/CrowdStrike2TH) is a [Crowdstrike Falcon](https://www.crowdstrike.com/endpoint-security-products/) alert feeder for TheHive, written by [Simon](https://github.com/xg5-simon). You can use it to feed Crowdstrike alerts into TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
+
+**Note**: this is a work in progress. Currently, the code licensing is unclear. 
+
 ### Integration with FireEye iSIGHT
 [FireEye2TH](https://github.com/LDO-CERT/FireEye2TH) is a free, open source [FireEye iSIGHT](https://www.fireeye.com/) alert feeder for TheHive, written by LDO-CERT. You can use it to feed FireEye iSIGHT alerts into TheHive, where they can be previewed and transformed into new cases using pre-defined incident response templates or added into existing ones.
 

project/Dependencies.scala

@@ -20,8 +20,8 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.8.0-1"
-    val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.11"
-    val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.11"
+    val elastic4play = "org.thehive-project" %% "elastic4play" % "1.10.0"
+    val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.19"
+    val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.19"
   }
 }

thehive-backend/app/controllers/ArtifactCtrl.scala

@@ -38,31 +38,35 @@ class ArtifactCtrl @Inject() (
   private[ArtifactCtrl] lazy val logger = Logger(getClass)
 
   // extract a file from the archive and make sure its size matches the header (to protect against zip bombs)
-  private def extractAndCheckSize(zipFile: ZipFile, header: FileHeader)(implicit authContext: AuthContext): FileInputValue = {
-    val file = tempSrv.newTemporaryFile(header.getFileName, "-fromZipFile")
-
-    val input = zipFile.getInputStream(header)
-    val size = header.getUncompressedSize
-    val sizedInput: FilterInputStream = new FilterInputStream(input) {
-      var totalRead = 0
-
-      override def read(): Int = {
-        if (totalRead < size) {
-          totalRead += 1
-          super.read()
+  private def extractAndCheckSize(zipFile: ZipFile, header: FileHeader)(implicit authContext: AuthContext): Option[FileInputValue] = {
+    val fileName = header.getFileName
+    if (fileName.contains('/')) None
+    else {
+      val file = tempSrv.newTemporaryFile(fileName, "-fromZipFile")
+
+      val input = zipFile.getInputStream(header)
+      val size = header.getUncompressedSize
+      val sizedInput: FilterInputStream = new FilterInputStream(input) {
+        var totalRead = 0
+
+        override def read(): Int = {
+          if (totalRead < size) {
+            totalRead += 1
+            super.read()
+          }
+          else throw BadRequestError("Error extracting file: output size doesn't match header")
         }
-        else throw BadRequestError("Error extracting file: output size doesn't match header")
       }
+      Files.delete(file)
+      val fileSize = Files.copy(sizedInput, file)
+      if (fileSize != size) {
+        file.toFile.delete()
+        throw InternalError("Error extracting file: output size doesn't match header")
+      }
+      input.close()
+      val contentType = Option(Files.probeContentType(file)).getOrElse("application/octet-stream")
+      Some(FileInputValue(header.getFileName, file, contentType))
     }
-    Files.delete(file)
-    val fileSize = Files.copy(sizedInput, file)
-    if (fileSize != size) {
-      file.toFile.delete()
-      throw InternalError("Error extracting file: output size doesn't match header")
-    }
-    input.close()
-    val contentType = Option(Files.probeContentType(file)).getOrElse("application/octet-stream")
-    FileInputValue(header.getFileName, file, contentType)
   }
 
   @Timed
@@ -91,7 +95,7 @@ class ArtifactCtrl @Inject() (
               }
 
               val multiFields = files.filterNot(_.isDirectory)
-                .map(extractAndCheckSize(zipFile, _))
+                .flatMap(extractAndCheckSize(zipFile, _))
                 .map { fiv ⇒
                   fields
                     .unset("isZip")

thehive-backend/app/models/Artifact.scala

@@ -1,8 +1,8 @@
 package models
 
 import java.util.Date
-import javax.inject.{ Inject, Provider, Singleton }
 
+import javax.inject.{ Inject, Provider, Singleton }
 import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.Success
 
@@ -12,6 +12,7 @@ import play.api.libs.json.JsValue.jsValueToJsLookup
 import play.api.libs.json.Json.toJsFieldJsValueWrapper
 import play.api.libs.json._
 
+import akka.stream.scaladsl.Sink
 import akka.stream.{ IOResult, Materializer }
 import akka.{ Done, NotUsed }
 import models.JsonFormat.artifactStatusFormat
@@ -114,9 +115,11 @@ class ArtifactModel @Inject() (
   override def getStats(entity: BaseEntity): Future[JsObject] = {
     entity match {
       case artifact: Artifact ⇒
-        val (_, total) = artifactSrv.get.findSimilar(artifact, Some("0-0"), Nil)
-        total.failed.foreach(t ⇒ logger.error("Artifact.getStats error", t))
-        total.map { t ⇒ Json.obj("seen" → t) }
+        val (similarArtifacts, total) = artifactSrv.get.findSimilar(artifact, Some("0-1"), Seq("-ioc"))
+        for {
+          ioc ← similarArtifacts.runWith(Sink.headOption).map(_.fold(false)(_.ioc()))
+          t ← total
+        } yield Json.obj("seen" → t, "ioc" → ioc)
       case _ ⇒ Future.successful(JsObject.empty)
     }
   }

thehive-backend/app/services/AlertSrv.scala

@@ -146,7 +146,7 @@ class AlertSrv(
 
   def update(alert: Alert, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Alert] = {
     val follow = fields.getBoolean("follow").getOrElse(alert.follow())
-    val newStatus = if (follow) AlertStatus.Updated else alert.status()
+    val newStatus = if (follow && alert.status() != AlertStatus.New) AlertStatus.Updated else alert.status()
     val updatedAlert = updateSrv(alert, fields.set("status", Json.toJson(newStatus)), modifyConfig)
     alert.caze() match {
       case Some(caseId) if follow ⇒

thehive-cortex/app/connectors/cortex/services/ActionOperation.scala

@@ -61,6 +61,10 @@ case class AddLogToTask(content: String, owner: Option[String], status: ActionOp
   override def updateStatus(newStatus: Type, newMessage: String): ActionOperation = copy(status = newStatus, message = newMessage)
 }
 
+case class AddTagToAlert(tag: String, status: ActionOperationStatus.Type = ActionOperationStatus.Waiting, message: String = "") extends ActionOperation {
+  override def updateStatus(newStatus: ActionOperationStatus.Type, newMessage: String): AddTagToAlert = copy(status = newStatus, message = newMessage)
+}
+
 object ActionOperation {
   val addTagToCaseWrites = Json.writes[AddTagToCase]
   val addTagToArtifactWrites = Json.writes[AddTagToArtifact]
@@ -69,6 +73,7 @@ object ActionOperation {
   val closeTaskWrites = Json.writes[CloseTask]
   val markAlertAsReadWrites = Json.writes[MarkAlertAsRead]
   val addLogToTaskWrites = Json.writes[AddLogToTask]
+  val addTagToAlertWrites = Json.writes[AddTagToAlert]
   implicit val actionOperationReads: Reads[ActionOperation] = Reads[ActionOperation](json ⇒
     (json \ "type").asOpt[String].fold[JsResult[ActionOperation]](JsError("type is missing in action operation")) {
       case "AddTagToCase"     ⇒ (json \ "tag").validate[String].map(tag ⇒ AddTagToCase(tag))
@@ -85,7 +90,8 @@ object ActionOperation {
         content ← (json \ "content").validate[String]
         owner ← (json \ "owner").validateOpt[String]
       } yield AddLogToTask(content, owner)
-      case other ⇒ JsError(s"Unknown operation $other")
+      case "AddTagToAlert" ⇒ (json \ "tag").validate[String].map(tag ⇒ AddTagToAlert(tag))
+      case other           ⇒ JsError(s"Unknown operation $other")
     })
   implicit val actionOperationWrites: Writes[ActionOperation] = Writes[ActionOperation] {
     case a: AddTagToCase     ⇒ addTagToCaseWrites.writes(a)
@@ -95,6 +101,7 @@ object ActionOperation {
     case a: CloseTask        ⇒ closeTaskWrites.writes(a)
     case a: MarkAlertAsRead  ⇒ markAlertAsReadWrites.writes(a)
     case a: AddLogToTask     ⇒ addLogToTaskWrites.writes(a)
+    case a: AddTagToAlert    ⇒ addTagToAlertWrites.writes(a)
     case a                   ⇒ Json.obj("unsupported operation" → a.toString)
   }
 }
@@ -139,6 +146,18 @@ class ActionOperationSrv @Inject() (
     }
   }
 
+  def findArtifactEntity(entity: BaseEntity): Future[Artifact] = {
+    import org.elastic4play.services.QueryDSL._
+
+    (entity, entity.model) match {
+      case (a: Artifact, _) ⇒ Future.successful(a)
+      case (_, model: ChildModelDef[_, _, _, _]) ⇒
+        findSrv(model.parentModel, "_id" ~= entity.parentId.getOrElse(throw InternalError(s"Child entity $entity has no parent ID")), Some("0-1"), Nil)
+          ._1.runWith(Sink.head).flatMap(findArtifactEntity _)
+      case _ ⇒ Future.failed(BadRequestError("Artifact not found"))
+    }
+  }
+
   def execute(entity: BaseEntity, operation: ActionOperation)(implicit authContext: AuthContext): Future[ActionOperation] = {
     if (operation.status == ActionOperationStatus.Waiting) {
       Retry()(classOf[VersionConflictEngineException]) {
@@ -186,6 +205,15 @@ class ActionOperationSrv @Inject() (
               task ← findTaskEntity(entity)
               _ ← logSrv.create(task, Fields.empty.set("message", content).set("owner", owner.map(JsString)))
             } yield operation.updateStatus(ActionOperationStatus.Success, "")
+          case AddTagToAlert(tag, _, _) ⇒
+            entity match {
+              case initialAlert: Alert ⇒
+                for {
+                  alert ← alertSrv.get(initialAlert.id)
+                  _ ← alertSrv.update(alert.id, Fields.empty.set("tags", Json.toJson((alert.tags() :+ tag).distinct)), ModifyConfig(retryOnConflict = 0, version = Some(alert.version)))
+                } yield operation.updateStatus(ActionOperationStatus.Success, "")
+              case _ ⇒ Future.failed(BadRequestError("Alert not found"))
+            }
           case o ⇒ Future.successful(operation.updateStatus(ActionOperationStatus.Failure, s"Operation $o not supported"))
         }
       }

thehive-cortex/app/connectors/cortex/services/CortexActionSrv.scala

@@ -72,10 +72,14 @@ class CortexActionSrv @Inject() (
 
   def findResponderFor(entityType: String, entityId: String): Future[Seq[Responder]] = {
     for {
-      (tlp, pap) ← getEntity(entityType, entityId)
-        .flatMap(actionOperationSrv.findCaseEntity)
-        .map { caze ⇒ (caze.tlp(), caze.pap()) }
-        .recover { case _ ⇒ (0L, 0L) }
+      entity ← getEntity(entityType, entityId)
+      artifactTlp ← actionOperationSrv
+        .findArtifactEntity(entity)
+        .map(a ⇒ Some(a.tlp()))
+        .recover { case _ ⇒ None }
+      (tlp, pap) ← actionOperationSrv.findCaseEntity(entity)
+        .map { caze ⇒ (artifactTlp.getOrElse(caze.tlp()), caze.pap()) }
+        .recover { case _ ⇒ (artifactTlp.getOrElse(0L), 0L) }
       query = Json.obj(
         "dataTypeList" → s"thehive:$entityType")
       responders ← findResponders(query)

thehive-misp/app/connectors/misp/MispConnection.scala

@@ -106,11 +106,13 @@ case class MispConnection(
           "name" → name,
           "version" → version,
           "status" → "OK",
+          "url" → baseUrl,
           "purpose" → purpose.toString)
         case None ⇒ Json.obj(
           "name" → name,
           "version" → "",
           "status" → "ERROR",
+          "url" → baseUrl,
           "purpose" → purpose.toString)
       }
   }