Merge branch 'hotfix/3.0.4'

CHANGELOG.md

@@ -1,6 +1,43 @@
 # Change Log
 
-## [3.0.3](https://github.com/TheHive-Project/TheHive/tree/3.0.3) (2017-12-22)
+## [3.0.4](https://github.com/TheHive-Project/TheHive/tree/3.0.4) (2018-02-05)
+[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.0.3...3.0.4)
+
+**Implemented enhancements:**
+
+- Add compatibility with Cortex 2 [\#466](https://github.com/TheHive-Project/TheHive/issues/466)
+- Make alerts searchable through the global search field [\#456](https://github.com/TheHive-Project/TheHive/issues/456)
+- Make counts on Counter dashboard's widget clickable [\#455](https://github.com/TheHive-Project/TheHive/issues/455)
+- MISP feeds cause the growing of ES audit docs [\#450](https://github.com/TheHive-Project/TheHive/issues/450)
+- Case metrics sort  [\#418](https://github.com/TheHive-Project/TheHive/issues/418)
+- StreamSrv: Unexpected message : StreamNotFound [\#414](https://github.com/TheHive-Project/TheHive/issues/414)
+- Filter MISP Events Using MISP Tags & More Before Creating Alerts [\#370](https://github.com/TheHive-Project/TheHive/issues/370)
+- Single-Sign On support [\#354](https://github.com/TheHive-Project/TheHive/issues/354)
+- OAuth2 single sign-on implementation \(BE + FE\) [\#430](https://github.com/TheHive-Project/TheHive/pull/430) ([saibot94](https://github.com/saibot94))
+
+**Fixed bugs:**
+
+- Remove uppercase filter on template name [\#464](https://github.com/TheHive-Project/TheHive/issues/464)
+- Fix the alert bulk update timeline message [\#463](https://github.com/TheHive-Project/TheHive/issues/463)
+- "too many substreams open" on alerts [\#462](https://github.com/TheHive-Project/TheHive/issues/462)
+- Fix MISP export error dialog column's wrap [\#460](https://github.com/TheHive-Project/TheHive/issues/460)
+- More than 20 users prevents assignment in tasks [\#459](https://github.com/TheHive-Project/TheHive/issues/459)
+- Type is not used when generating alert id [\#457](https://github.com/TheHive-Project/TheHive/issues/457)
+- Fix link to default report templates [\#454](https://github.com/TheHive-Project/TheHive/issues/454)
+- Make dashboard donuts clickable [\#453](https://github.com/TheHive-Project/TheHive/issues/453)
+- Refresh custom fields on open cases by background changes [\#440](https://github.com/TheHive-Project/TheHive/issues/440)
+- Bug: Case metrics not shown when creating case from template [\#417](https://github.com/TheHive-Project/TheHive/issues/417)
+- Observable report taxonomies bug [\#409](https://github.com/TheHive-Project/TheHive/issues/409)
+- File upload when /tmp is full [\#321](https://github.com/TheHive-Project/TheHive/issues/321)
+
+**Closed issues:**
+
+- GET request with Content-Type ends up in HTTP 400 [\#438](https://github.com/TheHive-Project/TheHive/issues/438)
+- Feature Request: Ability to bulk upload files as observables. [\#435](https://github.com/TheHive-Project/TheHive/issues/435)
+- Add metadata to MISP event when exporting case from TheHive [\#433](https://github.com/TheHive-Project/TheHive/issues/433)
+- How to limit by date amount of events pulled from MISP initially? [\#432](https://github.com/TheHive-Project/TheHive/issues/432)
+
+## [3.0.3](https://github.com/TheHive-Project/TheHive/tree/3.0.3) (2018-01-10)
 [Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.0.2...3.0.3)
 
 **Fixed bugs:**

build.sbt

@@ -102,7 +102,7 @@ linuxPackageMappings ++= Seq(
   packageMapping(
     file("package/thehive.conf") -> "/etc/init/thehive.conf",
     file("conf/application.sample") -> "/etc/thehive/application.conf",
-    file("conf/logback.xml") -> "/etc/thehive/logback.xml"
+    file("package/logback.xml") -> "/etc/thehive/logback.xml"
   ).withPerms("644").withConfig(),
   packageMapping(
     file("package/thehive") -> "/etc/init.d/thehive"
@@ -161,7 +161,7 @@ dockerEntrypoint := Seq("/opt/thehive/entrypoint")
 dockerExposedPorts := Seq(9000)
 mappings in Docker ++= Seq(
   file("package/docker/entrypoint") -> "/opt/thehive/entrypoint",
-  file("conf/logback.xml") -> "/etc/thehive/logback.xml",
+  file("package/logback.xml") -> "/etc/thehive/logback.xml",
   file("package/empty") -> "/var/log/thehive/application.log")
 mappings in Docker ~= (_.filterNot {
   case (_, filepath) => filepath == "/opt/thehive/conf/application.conf"

package/thehive

@@ -1,4 +1,6 @@
 #!/bin/sh -e
+# chkconfig: 2345 80 20
+# description: TheHive: a FOSS Information Security Incident Management Platform
 ### BEGIN INIT INFO
 # Provides:          thehive
 # Required-Start:    $remote_fs $syslog
@@ -22,12 +24,33 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin
 [ -r /etc/default/$NAME ] && . /etc/default/$NAME
 
 # Load the VERBOSE setting and other rcS variables
-. /lib/init/vars.sh
+[ -r /lib/init/vars.sh ] && . /lib/init/vars.sh
 
 # Define LSB log_* functions.
 # Depend on lsb-base (>= 3.2-14) to ensure that this file is present
 # and status_of_proc is working.
-. /lib/lsb/init-functions
+[ -r /lib/lsb/init-functions ] && . /lib/lsb/init-functions
+
+# Source function library.
+[ -r /etc/init.d/functions ] && . /etc/init.d/functions
+
+check_requirements() {
+  if ! which start-stop-daemon; then
+    echo Error: start-stop-daemon is missing, aborting
+    return 1
+  fi
+  INIT=$(readlink /proc/1/exe 2> /dev/null)
+  case ${INIT} in
+    *systemd*)
+      echo echo "Your are using systemd. This script should not be used"
+    *upstart*)
+      echo echo "Your are using upstart. This script should not be used"
+      return 1;;
+  esac
+  return 0
+}
+
+
 
 #
 # Function that starts the daemon/service
@@ -39,8 +62,7 @@ do_start() {
         #   2 if daemon could not be started
         start-stop-daemon --start --quiet --pidfile $PIDFILE --user $DAEMON_USER --exec $DAEMON --test > /dev/null \
                 || return 1
-        start-stop-daemon --start --quiet --pidfile $PIDFILE --user $DAEMON_USER --exec $DAEMON -- \
-                $DAEMON_ARGS \
+        start-stop-daemon --start --quiet --pidfile $PIDFILE --user $DAEMON_USER --exec $DAEMON -- $DAEMON_ARGS \
                 || return 2
 }
 
@@ -69,16 +91,10 @@ do_stop() {
         return "$RETVAL"
 }
 
-check_for_upstart() {
-  if init_is_upstart; then
-    echo "Your are using upstart. This script should not be used"
-    exit $1
-  fi
-}
+check_requirements || exit 1
 
 case "$1" in
   start)
-    check_for_upstart 1
     if [ "$ENABLED" = no ]; then
       echo "The service TheHive is disabled (/etc/default/thehive)"
       exit 1
@@ -93,7 +109,6 @@ case "$1" in
     ;;
 
   stop)
-    check_for_upstart 1
     [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
     do_stop
     case "$?" in
@@ -103,7 +118,6 @@ case "$1" in
     ;;
 
   status)
-    check_for_upstart 1
     status_of_proc -p $PIDFILE $DAEMON $NAME && exit 0 || exit $?
     ;;
 

project/Dependencies.scala

@@ -20,6 +20,6 @@ object Dependencies {
 
     val reflections = "org.reflections" % "reflections" % "0.9.11"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.2"
+    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.4.3"
   }
 }

project/build.properties

@@ -1 +1 @@
-sbt.version=1.0.3
+sbt.version=1.1.0

thehive-backend/app/models/Alert.scala

@@ -27,23 +27,23 @@ trait AlertAttributes {
   _: AttributeDef ⇒
   val artifactAttributes: Seq[Attribute[_]] = {
     val remoteAttachmentAttributes = Seq(
-      Attribute("alert", "reference", F.stringFmt, Nil, None, ""),
-      Attribute("alert", "filename", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
-      Attribute("alert", "contentType", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
-      Attribute("alert", "size", OptionalAttributeFormat(F.numberFmt), Nil, None, ""),
-      Attribute("alert", "hash", MultiAttributeFormat(F.stringFmt), Nil, None, ""),
-      Attribute("alert", "type", OptionalAttributeFormat(F.stringFmt), Nil, None, ""))
+      Attribute("alert", "reference", F.stringFmt, Seq(O.readonly), None, ""),
+      Attribute("alert", "filename", OptionalAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "contentType", OptionalAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "size", OptionalAttributeFormat(F.numberFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "hash", MultiAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "type", OptionalAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""))
 
     Seq(
-      Attribute("alert", "data", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
-      Attribute("alert", "dataType", F.stringFmt, Nil, None, ""),
-      Attribute("alert", "message", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
-      Attribute("alert", "startDate", OptionalAttributeFormat(F.dateFmt), Nil, None, ""),
-      Attribute("alert", "attachment", OptionalAttributeFormat(F.attachmentFmt), Nil, None, ""),
-      Attribute("alert", "remoteAttachment", OptionalAttributeFormat(F.objectFmt(remoteAttachmentAttributes)), Nil, None, ""),
-      Attribute("alert", "tlp", OptionalAttributeFormat(TlpAttributeFormat), Nil, None, ""),
-      Attribute("alert", "tags", MultiAttributeFormat(F.stringFmt), Nil, None, ""),
-      Attribute("alert", "ioc", OptionalAttributeFormat(F.booleanFmt), Nil, None, ""))
+      Attribute("alert", "data", OptionalAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "dataType", F.stringFmt, Seq(O.readonly), None, ""),
+      Attribute("alert", "message", OptionalAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "startDate", OptionalAttributeFormat(F.dateFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "attachment", OptionalAttributeFormat(F.attachmentFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "remoteAttachment", OptionalAttributeFormat(F.objectFmt(remoteAttachmentAttributes)), Seq(O.readonly), None, ""),
+      Attribute("alert", "tlp", OptionalAttributeFormat(TlpAttributeFormat), Seq(O.readonly), None, ""),
+      Attribute("alert", "tags", MultiAttributeFormat(F.stringFmt), Seq(O.readonly), None, ""),
+      Attribute("alert", "ioc", OptionalAttributeFormat(F.booleanFmt), Seq(O.readonly), None, ""))
   }
 
   val alertId: A[String] = attribute("_id", F.stringFmt, "Alert id", O.readonly)
@@ -58,7 +58,7 @@ trait AlertAttributes {
   val severity: A[Long] = attribute("severity", SeverityAttributeFormat, "Severity if the alert (0-3)", 2L)
   val tags: A[Seq[String]] = multiAttribute("tags", F.stringFmt, "Alert tags")
   val tlp: A[Long] = attribute("tlp", TlpAttributeFormat, "TLP level", 2L)
-  val artifacts: A[Seq[JsObject]] = multiAttribute("artifacts", F.objectFmt(artifactAttributes), "Artifact of the alert")
+  val artifacts: A[Seq[JsObject]] = multiAttribute("artifacts", F.objectFmt(artifactAttributes), "Artifact of the alert", O.unaudited)
   val caseTemplate: A[Option[String]] = optionalAttribute("caseTemplate", F.stringFmt, "Case template to use")
   val status: A[AlertStatus.Value] = attribute("status", F.enumFmt(AlertStatus), "Status of the alert", AlertStatus.New)
   val follow: A[Boolean] = attribute("follow", F.booleanFmt, "", true)
@@ -94,7 +94,7 @@ class AlertModel @Inject() (dblists: DBLists)
           attrs
         else {
           val hasher = Hasher("MD5")
-          val tpe = (attrs \ "tpe").asOpt[String].getOrElse("<null>")
+          val tpe = (attrs \ "type").asOpt[String].getOrElse("<null>")
           val source = (attrs \ "source").asOpt[String].getOrElse("<null>")
           val sourceRef = (attrs \ "sourceRef").asOpt[String].getOrElse("<null>")
           val _id = hasher.fromString(s"$tpe|$source|$sourceRef").head.toString()

thehive-backend/app/models/Migration.scala

@@ -18,6 +18,7 @@ import akka.stream.Materializer
 import akka.stream.scaladsl.Source
 import services.{ AlertSrv, DashboardSrv }
 
+import org.elastic4play.ConflictError
 import org.elastic4play.controllers.Fields
 import org.elastic4play.services.JsonFormat.attachmentFormat
 import org.elastic4play.services._
@@ -80,27 +81,32 @@ class Migration(
         dataTypeList.addItem(dt)
           .map(_ ⇒ ())
           .recover {
-            case error ⇒ logger.error(s"Failed to add dataType $dt during migration", error)
+            case _: ConflictError ⇒
+            case error            ⇒ logger.error(s"Failed to add dataType $dt during migration", error)
           }
       }
       .map(_ ⇒ ())
   }
 
   private def addDashboards(version: Int): Future[Unit] = {
-    userSrv.inInitAuthContext { implicit authContext ⇒
-      val dashboardsPath = environment.rootPath.toPath.resolve("migration").resolve("12").resolve("dashboards")
-      val dashboards = for {
-        dashboardFile ← Try(Files.newDirectoryStream(dashboardsPath, "*.json").asScala).getOrElse(Nil)
-        if Files.isReadable(dashboardFile)
-        dashboardJson ← Try(readJsonFile(dashboardFile).as[JsObject]).toOption
-        dashboardDefinition = (dashboardJson \ "definition").as[JsValue].toString
-        dash = dashboardSrv.create(Fields(dashboardJson + ("definition" -> JsString(dashboardDefinition))))
-          .map(_ ⇒ ())
-          .recover {
-            case error ⇒ logger.error(s"Failed to create dashboard $dashboardFile during migration", error)
-          }
-      } yield dash
-      Future.sequence(dashboards).map(_ ⇒ ())
+    dashboardSrv.find(QueryDSL.any, Some("0-0"), Nil)._2.flatMap {
+      case 0 ⇒
+        userSrv.inInitAuthContext { implicit authContext ⇒
+          val dashboardsPath = environment.rootPath.toPath.resolve("migration").resolve("12").resolve("dashboards")
+          val dashboards = for {
+            dashboardFile ← Try(Files.newDirectoryStream(dashboardsPath, "*.json").asScala).getOrElse(Nil)
+            if Files.isReadable(dashboardFile)
+            dashboardJson ← Try(readJsonFile(dashboardFile).as[JsObject]).toOption
+            dashboardDefinition = (dashboardJson \ "definition").as[JsValue].toString
+            dash = dashboardSrv.create(Fields(dashboardJson + ("definition" -> JsString(dashboardDefinition))))
+              .map(_ ⇒ ())
+              .recover {
+                case error ⇒ logger.error(s"Failed to create dashboard $dashboardFile during migration", error)
+              }
+          } yield dash
+          Future.sequence(dashboards).map(_ ⇒ ())
+        }
+      case _ ⇒ Future.successful(())
     }
   }
 
@@ -294,6 +300,49 @@ class Migration(
           caseTemplate - "metricNames" + ("metrics" -> metrics)
         },
         addAttribute("case_artifact", "sighted" -> JsFalse))
+    case ds @ DatabaseState(12) ⇒
+      Seq(
+        // Remove alert artifacts in audit trail
+        mapEntity("audit") {
+          case audit if (audit \ "objectType").asOpt[String].contains("alert") ⇒
+            (audit \ "details").asOpt[JsObject].fold(audit) { details ⇒
+              audit + ("details" -> (details - "artifacts"))
+            }
+          case audit ⇒ audit
+        },
+        // Regenerate all alert ID
+        mapEntity("alert") { alert ⇒
+          val alertId = JsString(generateAlertId(alert))
+          alert + ("_id" → alertId) + ("_routing" -> alertId)
+        },
+        // and overwrite alert id in audit trail
+        Operation((f: String ⇒ Source[JsObject, NotUsed]) ⇒ {
+          case "audit" ⇒ f("audit").flatMapConcat {
+            case audit if (audit \ "objectType").asOpt[String].contains("alert") ⇒
+              val updatedAudit = (audit \ "objectId").asOpt[String].fold(Future.successful(audit)) { alertId ⇒
+                ds.getEntity("alert", alertId)
+                  .map { alert ⇒
+                    audit + ("objectId" -> JsString(generateAlertId(alert)))
+                  }
+                  .recover {
+                    case e ⇒
+                      logger.error(s"Get alert $alertId", e)
+                      audit
+                  }
+              }
+              Source.fromFuture(updatedAudit)
+            case audit ⇒ Source.single(audit)
+          }
+          case other ⇒ f(other)
+        }))
+  }
+
+  private def generateAlertId(alert: JsObject): String = {
+    val hasher = Hasher("MD5")
+    val tpe = (alert \ "type").asOpt[String].getOrElse("<null>")
+    val source = (alert \ "source").asOpt[String].getOrElse("<null>")
+    val sourceRef = (alert \ "sourceRef").asOpt[String].getOrElse("<null>")
+    hasher.fromString(s"$tpe|$source|$sourceRef").head.toString()
   }
 
   private def convertDate(json: JsValue): JsValue = {

thehive-backend/app/models/package.scala

@@ -1,5 +1,5 @@
 
 
 package object models {
-  val modelVersion = 12
+  val modelVersion = 13
 }