Merge branch 'release/2.12.0'

Conflicts:
	CHANGELOG.md
	ui/bower.json
	ui/package.json

CHANGELOG.md

@@ -1,7 +1,36 @@
 # Change Log
 
+## [2.12.0](https://github.com/CERT-BDF/TheHive/tree/2.12.0)
+
+[Full Changelog](https://github.com/CERT-BDF/TheHive/compare/2.11.3...2.12.0)
+
+**Implemented enhancements:**
+
+- Sort the analyzers list in observable details page [\#245](https://github.com/CERT-BDF/TheHive/issues/245)
+- More options to sort cases [\#243](https://github.com/CERT-BDF/TheHive/issues/243)
+- Alert Preview and management improvements [\#232](https://github.com/CERT-BDF/TheHive/issues/232)
+- Ability to Reopen Tasks [\#156](https://github.com/CERT-BDF/TheHive/issues/156)
+- Display short reports on the Observables tab [\#131](https://github.com/CERT-BDF/TheHive/issues/131)
+- Custom fields for case template [\#12](https://github.com/CERT-BDF/TheHive/issues/12)
+- Show case status and category \(FP, TP, IND\) in related cases  [\#229](https://github.com/CERT-BDF/TheHive/issues/229)
+- Open External Links in New Tab [\#228](https://github.com/CERT-BDF/TheHive/issues/228)
+- Observable analyzers view reports. [\#191](https://github.com/CERT-BDF/TheHive/issues/191)
+- Specifying tags on statistics page or performing a search [\#186](https://github.com/CERT-BDF/TheHive/issues/186)
+- Choose case template while importing events from MISP [\#175](https://github.com/CERT-BDF/TheHive/issues/175)
+- Use local font files [\#250](https://github.com/CERT-BDF/TheHive/issues/250)
+
+**Fixed bugs:**
+
+- Fix case metrics malformed definitions [\#248](https://github.com/CERT-BDF/TheHive/issues/248)
+- Sorting alerts by severity fails [\#242](https://github.com/CERT-BDF/TheHive/issues/242)
+- Alerting Panel: Typo Correction [\#240](https://github.com/CERT-BDF/TheHive/issues/240)
+- files in alerts are limited to 32kB [\#237](https://github.com/CERT-BDF/TheHive/issues/237)
+- Alert can contain inconsistent data [\#234](https://github.com/CERT-BDF/TheHive/issues/234)
+- Search do not work with non-latin characters [\#223](https://github.com/CERT-BDF/TheHive/issues/223)
+- report status not updated after finish [\#212](https://github.com/CERT-BDF/TheHive/issues/212)
+
 ## [2.11.3](https://github.com/CERT-BDF/TheHive/tree/2.11.3) (2017-06-14)
-[Full Changelog](https://github.com/CERT-BDF/TheHive/compare/debian/2.11.2-2...2.11.3)
+[Full Changelog](https://github.com/CERT-BDF/TheHive/compare/debian/2.11.2...2.11.3)
 
 **Fixed bugs:**
 

build.sbt

@@ -104,7 +104,7 @@ packageBin := {
   (packageBin in Rpm).value
 }
 // DEB //
-version in Debian := version.value + "-2"
+version in Debian := version.value + "-1"
 debianPackageRecommends := Seq("elasticsearch")
 debianPackageDependencies += "openjdk-8-jre-headless"
 maintainerScripts in Debian := maintainerScriptsFromDirectory(

contrib/report-templates/File_Info_1_0/long.html

@@ -9,7 +9,7 @@
     <!--  File Indenfication -->
     <div class="panel panel-info">
         <div class="panel-heading">
-            <strong>File Idenfitication</strong>
+            <strong>File Identification</strong>
         </div>
         <div class="panel-body">
             <dl class="dl-horizontal">

project/Dependencies.scala

@@ -30,6 +30,6 @@ object Dependencies {
     val reflections = "org.reflections" % "reflections" % "0.9.10"
     val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
     val akkaTest = "com.typesafe.akka" %% "akka-stream-testkit" % "2.4.4"
-    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.1.5"
+    val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.2.1"
   }
 }

thehive-backend/app/controllers/AlertCtrl.scala

@@ -10,16 +10,18 @@ import org.elastic4play.services._
 import org.elastic4play.{ BadRequestError, Timed }
 import play.api.Logger
 import play.api.http.Status
-import play.api.libs.json.JsArray
+import play.api.libs.json.{ JsArray, JsObject, Json }
 import play.api.mvc.{ Action, AnyContent, Controller }
-import services.AlertSrv
+import services.{ AlertSrv, CaseSrv }
+import services.JsonFormat.caseSimilarityWrites
 
 import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.Try
 
 @Singleton
 class AlertCtrl @Inject() (
     alertSrv: AlertSrv,
+    caseSrv: CaseSrv,
     auxSrv: AuxSrv,
     authenticated: Authenticated,
     renderer: Renderer,
@@ -35,17 +37,39 @@ class AlertCtrl @Inject() (
       .map(alert ⇒ renderer.toOutput(CREATED, alert))
   }
 
+  @Timed
+  def mergeWithCase(alertId: String, caseId: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒
+    for {
+      alert ← alertSrv.get(alertId)
+      caze ← caseSrv.get(caseId)
+      _ ← alertSrv.mergeWithCase(alert, caze)
+    } yield renderer.toOutput(CREATED, caze)
+  }
+
   @Timed
   def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒
-    val withStats = for {
-      statsValues ← request.queryString.get("nstats")
-      firstValue ← statsValues.headOption
-    } yield Try(firstValue.toBoolean).getOrElse(firstValue == "1")
+    val withStats = request
+      .queryString
+      .get("nstats")
+      .flatMap(_.headOption)
+      .exists(v ⇒ Try(v.toBoolean).getOrElse(v == "1"))
+
+    val withSimilarity = request
+      .queryString
+      .get("similarity")
+      .flatMap(_.headOption)
+      .exists(v ⇒ Try(v.toBoolean).getOrElse(v == "1"))
 
     for {
       alert ← alertSrv.get(id)
-      alertsWithStats ← auxSrv.apply(alert, 0, withStats.getOrElse(false), removeUnaudited = false)
-    } yield renderer.toOutput(OK, alertsWithStats)
+      alertsWithStats ← auxSrv.apply(alert, 0, withStats, removeUnaudited = false)
+      similarCases ← if (withSimilarity)
+        alertSrv.similarCases(alert)
+          .map(sc ⇒ Json.obj("similarCases" → Json.toJson(sc)))
+      else Future.successful(JsObject(Nil))
+    } yield {
+      renderer.toOutput(OK, alertsWithStats ++ similarCases)
+    }
   }
 
   @Timed
@@ -106,11 +130,12 @@ class AlertCtrl @Inject() (
   }
 
   @Timed
-  def createCase(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒
+  def createCase(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒
     for {
       alert ← alertSrv.get(id)
-      updatedAlert ← alertSrv.createCase(alert)
-    } yield renderer.toOutput(CREATED, updatedAlert)
+      customCaseTemplate = request.body.getString("caseTemplate")
+      caze ← alertSrv.createCase(alert, customCaseTemplate)
+    } yield renderer.toOutput(CREATED, caze)
   }
 
   @Timed

thehive-backend/app/controllers/ArtifactCtrl.scala

@@ -48,7 +48,7 @@ class ArtifactCtrl @Inject() (
 
   @Timed
   def get(id: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒
-    artifactSrv.get(id, request.body.getStrings("fields").map("dataType" +: _))
+    artifactSrv.get(id)
       .map(artifact ⇒ renderer.toOutput(OK, artifact))
   }
 

thehive-backend/app/models/Alert.scala

@@ -4,8 +4,11 @@ import java.util.Date
 import javax.inject.{ Inject, Singleton }
 
 import models.JsonFormat.alertStatusFormat
-import org.elastic4play.models.{ Attribute, AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, AttributeFormat ⇒ F, AttributeOption ⇒ O }
+import org.elastic4play.controllers.JsonInputValue
+import org.elastic4play.models.{ Attribute, AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, MultiAttributeFormat, OptionalAttributeFormat, AttributeFormat ⇒ F, AttributeOption ⇒ O }
+import org.elastic4play.services.DBLists
 import org.elastic4play.utils.Hasher
+import org.elastic4play.{ AttributeCheckingError, InvalidFormatAttributeError }
 import play.api.Logger
 import play.api.libs.json._
 import services.AuditedModel
@@ -20,7 +23,26 @@ object AlertStatus extends Enumeration with HiveEnumeration {
 
 trait AlertAttributes {
   _: AttributeDef ⇒
-  def artifactAttributes: Seq[Attribute[_]]
+  val artifactAttributes: Seq[Attribute[_]] = {
+    val remoteAttachmentAttributes = Seq(
+      Attribute("alert", "reference", F.stringFmt, Nil, None, ""),
+      Attribute("alert", "filename", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
+      Attribute("alert", "contentType", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
+      Attribute("alert", "size", OptionalAttributeFormat(F.numberFmt), Nil, None, ""),
+      Attribute("alert", "hash", MultiAttributeFormat(F.stringFmt), Nil, None, ""),
+      Attribute("alert", "type", OptionalAttributeFormat(F.stringFmt), Nil, None, ""))
+
+    Seq(
+      Attribute("alert", "data", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
+      Attribute("alert", "dataType", F.stringFmt, Nil, None, ""),
+      Attribute("alert", "message", OptionalAttributeFormat(F.stringFmt), Nil, None, ""),
+      Attribute("alert", "startDate", OptionalAttributeFormat(F.dateFmt), Nil, None, ""),
+      Attribute("alert", "attachment", OptionalAttributeFormat(F.attachmentFmt), Nil, None, ""),
+      Attribute("alert", "remoteAttachment", OptionalAttributeFormat(F.objectFmt(remoteAttachmentAttributes)), Nil, None, ""),
+      Attribute("alert", "tlp", OptionalAttributeFormat(F.numberFmt), Nil, None, ""),
+      Attribute("alert", "tags", MultiAttributeFormat(F.stringFmt), Nil, None, ""),
+      Attribute("alert", "ioc", OptionalAttributeFormat(F.stringFmt), Nil, None, ""))
+  }
 
   val alertId: A[String] = attribute("_id", F.stringFmt, "Alert id", O.readonly)
   val tpe: A[String] = attribute("type", F.stringFmt, "Type of the alert", O.readonly)
@@ -41,7 +63,7 @@ trait AlertAttributes {
 }
 
 @Singleton
-class AlertModel @Inject() (artifactModel: ArtifactModel)
+class AlertModel @Inject() (dblists: DBLists)
     extends ModelDef[AlertModel, Alert]("alert")
     with AlertAttributes
     with AuditedModel {
@@ -50,30 +72,38 @@ class AlertModel @Inject() (artifactModel: ArtifactModel)
   override val defaultSortBy: Seq[String] = Seq("-date")
   override val removeAttribute: JsObject = Json.obj("status" → AlertStatus.Ignored)
 
-  override def artifactAttributes: Seq[Attribute[_]] = artifactModel.attributes
-
   override def creationHook(parent: Option[BaseEntity], attrs: JsObject): Future[JsObject] = {
-    Future.successful {
-      if (attrs.keys.contains("_id"))
-        attrs
-      else {
-        val hasher = Hasher("MD5")
-        val tpe = (attrs \ "tpe").asOpt[String].getOrElse("<null>")
-        val source = (attrs \ "source").asOpt[String].getOrElse("<null>")
-        val sourceRef = (attrs \ "sourceRef").asOpt[String].getOrElse("<null>")
-        val _id = hasher.fromString(s"$tpe|$source|$sourceRef").head.toString()
-        attrs + ("_id" → JsString(_id))
-      } - "lastSyncDate" - "case" - "status" - "follow"
-    }
+    // check if data attribute is present on all artifacts
+    val missingDataErrors = (attrs \ "artifacts")
+      .asOpt[Seq[JsValue]]
+      .getOrElse(Nil)
+      .filter { a ⇒
+        ((a \ "data").toOption.isEmpty && (a \ "attachment").toOption.isEmpty && (a \ "remoteAttachment").toOption.isEmpty) ||
+          ((a \ "tags").toOption.isEmpty && (a \ "message").toOption.isEmpty)
+      }
+      .map(v ⇒ InvalidFormatAttributeError("artifacts", "artifact", JsonInputValue(v)))
+    if (missingDataErrors.nonEmpty)
+      Future.failed(AttributeCheckingError("alert", missingDataErrors))
+    else
+      Future.successful {
+        if (attrs.keys.contains("_id"))
+          attrs
+        else {
+          val hasher = Hasher("MD5")
+          val tpe = (attrs \ "tpe").asOpt[String].getOrElse("<null>")
+          val source = (attrs \ "source").asOpt[String].getOrElse("<null>")
+          val sourceRef = (attrs \ "sourceRef").asOpt[String].getOrElse("<null>")
+          val _id = hasher.fromString(s"$tpe|$source|$sourceRef").head.toString()
+          attrs + ("_id" → JsString(_id))
+        } - "lastSyncDate" - "case" - "status" - "follow"
+      }
   }
 }
 
 class Alert(model: AlertModel, attributes: JsObject)
     extends EntityDef[AlertModel, Alert](model, attributes)
     with AlertAttributes {
 
-  override def artifactAttributes: Seq[Attribute[_]] = Nil
-
   override def toJson: JsObject = super.toJson +
     ("artifacts" → JsArray(artifacts().map {
       // for file artifact, parse data as Json

thehive-backend/app/models/Artifact.scala

@@ -3,21 +3,26 @@ package models
 import java.util.Date
 import javax.inject.{ Inject, Provider, Singleton }
 
+import akka.{ Done, NotUsed }
+
 import scala.concurrent.{ ExecutionContext, Future }
 import scala.language.postfixOps
-import akka.stream.Materializer
-import play.api.libs.json.{ JsNull, JsObject, JsString, JsValue, JsArray }
+import akka.stream.{ IOResult, Materializer }
+import play.api.libs.json.{ JsArray, JsNull, JsObject, JsString, JsValue }
 import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup
 import play.api.libs.json.JsValue.jsValueToJsLookup
 import play.api.libs.json.Json
 import play.api.libs.json.Json.toJsFieldJsValueWrapper
-import org.elastic4play.BadRequestError
+import org.elastic4play.{ BadRequestError, InternalError }
 import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O }
-import org.elastic4play.services.{ Attachment, DBLists }
+import org.elastic4play.services.{ Attachment, AttachmentSrv, DBLists }
 import org.elastic4play.utils.MultiHash
 import models.JsonFormat.artifactStatusFormat
+import play.api.Logger
 import services.{ ArtifactSrv, AuditedModel }
 
+import scala.util.Success
+
 object ArtifactStatus extends Enumeration with HiveEnumeration {
   type Type = Value
   val Ok, Deleted = Value
@@ -28,7 +33,7 @@ trait ArtifactAttributes { _: AttributeDef ⇒
   val artifactId: A[String] = attribute("_id", F.stringFmt, "Artifact id", O.model)
   val data: A[Option[String]] = optionalAttribute("data", F.stringFmt, "Content of the artifact", O.readonly)
   val dataType: A[String] = attribute("dataType", F.listEnumFmt("artifactDataType")(dblists), "Type of the artifact", O.readonly)
-  val message: A[String] = attribute("message", F.textFmt, "Description of the artifact in the context of the case")
+  val message: A[Option[String]] = optionalAttribute("message", F.textFmt, "Description of the artifact in the context of the case")
   val startDate: A[Date] = attribute("startDate", F.dateFmt, "Creation date", new Date)
   val attachment: A[Option[Attachment]] = optionalAttribute("attachment", F.attachmentFmt, "Artifact file content", O.readonly)
   val tlp: A[Long] = attribute("tlp", F.numberFmt, "TLP level", 2L)
@@ -42,37 +47,68 @@ trait ArtifactAttributes { _: AttributeDef ⇒
 class ArtifactModel @Inject() (
     caseModel: CaseModel,
     val dblists: DBLists,
+    attachmentSrv: AttachmentSrv,
     artifactSrv: Provider[ArtifactSrv],
     implicit val mat: Materializer,
     implicit val ec: ExecutionContext) extends ChildModelDef[ArtifactModel, Artifact, CaseModel, Case](caseModel, "case_artifact") with ArtifactAttributes with AuditedModel {
+  private[ArtifactModel] lazy val logger = Logger(getClass)
   override val removeAttribute: JsObject = Json.obj("status" → ArtifactStatus.Deleted)
 
-  override def apply(attributes: JsObject) = {
+  override def apply(attributes: JsObject): Artifact = {
     val tags = (attributes \ "tags").asOpt[Seq[JsString]].getOrElse(Nil).distinct
     new Artifact(this, attributes + ("tags" → JsArray(tags)))
   }
 
   // this method modify request in order to hash artifact and manager file upload
   override def creationHook(parent: Option[BaseEntity], attrs: JsObject): Future[JsObject] = {
     val keys = attrs.keys
+    if (!keys.contains("message") && (attrs \ "tags").asOpt[Seq[JsValue]].forall(_.isEmpty))
+      throw BadRequestError(s"Artifact must contain a message or on ore more tags")
     if (keys.contains("data") == keys.contains("attachment"))
       throw BadRequestError(s"Artifact must contain data or attachment (but not both)")
-    computeId(parent, attrs).map { id ⇒
+    computeId(parent.getOrElse(throw InternalError(s"artifact $attrs has no parent")), attrs).map { id ⇒
       attrs + ("_id" → JsString(id))
     }
   }
 
-  def computeId(parent: Option[BaseEntity], attrs: JsObject): Future[String] = {
+  override def updateHook(entity: BaseEntity, updateAttrs: JsObject): Future[JsObject] = {
+    entity match {
+      case artifact: Artifact ⇒
+        val removeMessage = (updateAttrs \ "message").toOption.exists {
+          case JsNull       ⇒ true
+          case JsArray(Nil) ⇒ true
+          case _            ⇒ false
+        }
+        val removeTags = (updateAttrs \ "tags").toOption.exists {
+          case JsNull       ⇒ true
+          case JsArray(Nil) ⇒ true
+          case _            ⇒ false
+        }
+        if ((removeMessage && removeTags) ||
+          (removeMessage && artifact.tags().isEmpty) ||
+          (removeTags && artifact.message().isEmpty))
+          Future.failed(BadRequestError(s"Artifact must contain a message or on ore more tags"))
+        else
+          Future.successful(updateAttrs)
+    }
+  }
+  def computeId(parent: BaseEntity, attrs: JsObject): Future[String] = {
     // in order to make sure that there is no duplicated artifact, calculate its id from its content (dataType, data, attachment and parent)
     val mm = new MultiHash("MD5")
     mm.addValue((attrs \ "data").asOpt[JsValue].getOrElse(JsNull))
     mm.addValue((attrs \ "dataType").asOpt[JsValue].getOrElse(JsNull))
-    (attrs \ "attachment" \ "filepath").asOpt[String]
-      .fold(Future.successful(()))(file ⇒ mm.addFile(file))
-      .map { _ ⇒
-        mm.addValue(JsString(parent.fold("")(_.id)))
-        mm.digest.toString
-      }
+    for {
+      IOResult(_, done) ← (attrs \ "attachment" \ "filepath").asOpt[String]
+        .fold(Future.successful(IOResult(0, Success(Done))))(file ⇒ mm.addFile(file))
+      _ ← Future.fromTry(done)
+      _ ← (attrs \ "attachment" \ "id").asOpt[String]
+        .fold(Future.successful(NotUsed: NotUsed)) { fileId ⇒
+          mm.addFile(attachmentSrv.source(fileId))
+        }
+    } yield {
+      mm.addValue(JsString(parent.id))
+      mm.digest.toString
+    }
   }
 
   override def getStats(entity: BaseEntity): Future[JsObject] = {