Merge branch 'hotfix/2.11.3'

build.sbt

@@ -106,7 +106,7 @@ packageBin := {
 // DEB //
 version in Debian := version.value + "-2"
 debianPackageRecommends := Seq("elasticsearch")
-debianPackageDependencies += "java8-runtime-headless | java8-runtime"
+debianPackageDependencies += "openjdk-8-jre-headless"
 maintainerScripts in Debian := maintainerScriptsFromDirectory(
   baseDirectory.value / "package" / "debian",
   Seq(DebianConstants.Postinst, DebianConstants.Prerm, DebianConstants.Postrm)

thehive-backend/app/controllers/AlertCtrl.scala

@@ -105,6 +105,7 @@ class AlertCtrl @Inject() (
     } yield renderer.toOutput(OK, updatedAlert)
   }
 
+  @Timed
   def createCase(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒
     for {
       alert ← alertSrv.get(id)
@@ -118,8 +119,15 @@ class AlertCtrl @Inject() (
       .map { alert ⇒ renderer.toOutput(OK, alert) }
   }
 
+  @Timed
   def unfollowAlert(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒
     alertSrv.setFollowAlert(id, follow = false)
       .map { alert ⇒ renderer.toOutput(OK, alert) }
   }
+
+  @Timed
+  def fixStatus() = authenticated(Role.admin).async { implicit request ⇒
+    alertSrv.fixStatus()
+      .map(_ ⇒ NoContent)
+  }
 }

thehive-backend/app/services/AlertSrv.scala

@@ -90,6 +90,9 @@ class AlertSrv(
   def update(id: String, fields: Fields)(implicit authContext: AuthContext): Future[Alert] =
     updateSrv[AlertModel, Alert](alertModel, id, fields)
 
+  def update(alert: Alert, fields: Fields)(implicit authContext: AuthContext): Future[Alert] =
+    updateSrv(alert, fields)
+
   def bulkUpdate(ids: Seq[String], fields: Fields)(implicit authContext: AuthContext): Future[Seq[Try[Alert]]] = {
     updateSrv[AlertModel, Alert](alertModel, ids, fields)
   }
@@ -209,4 +212,36 @@ class AlertSrv(
   def setFollowAlert(alertId: String, follow: Boolean)(implicit authContext: AuthContext): Future[Alert] = {
     updateSrv[AlertModel, Alert](alertModel, alertId, Fields(Json.obj("follow" → follow)))
   }
+
+  def fixStatus()(implicit authContext: AuthContext): Future[Unit] = {
+    import org.elastic4play.services.QueryDSL._
+
+    val updatedStatusFields = Fields.empty.set("status", "Updated")
+    val (updateAlerts, updateAlertCount) = find("status" ~= "Update", Some("all"), Nil)
+    updateAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Update status"))
+    val updateAlertProcess = updateAlerts
+      .mapAsyncUnordered(3) { alert ⇒
+        logger.debug(s"Updating alert ${alert.id} (status: Update -> Updated)")
+        update(alert, updatedStatusFields)
+          .andThen {
+            case Failure(error) ⇒ logger.warn(s"""Fail to set "Updated" status to alert ${alert.id}""", error)
+          }
+      }
+
+    val ignoredStatusFields = Fields.empty.set("status", "Ignored")
+    val (ignoreAlerts, ignoreAlertCount) = find("status" ~= "Ignore", Some("all"), Nil)
+    ignoreAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Ignore status"))
+    val ignoreAlertProcess = ignoreAlerts
+      .mapAsyncUnordered(3) { alert ⇒
+        logger.debug(s"Updating alert ${alert.id} (status: Ignore -> Ignored)")
+        update(alert, ignoredStatusFields)
+          .andThen {
+            case Failure(error) ⇒ logger.warn(s"""Fail to set "Ignored" status to alert ${alert.id}""", error)
+          }
+      }
+
+    (updateAlertProcess ++ ignoreAlertProcess)
+      .runWith(Sink.ignore)
+      .map(_ ⇒ ())
+  }
 }

thehive-backend/app/services/StreamSrv.scala

@@ -43,7 +43,7 @@ object StreamActor {
   /* Ask messages, wait if there is no ready messages*/
   case object GetOperations
   /* Pending messages must be sent to sender */
-  case class Submit(senderRef: ActorRef)
+  case object Submit
   /* List of ready messages */
   case class StreamMessages(messages: Seq[JsObject])
   case object StreamNotFound
@@ -70,25 +70,25 @@ class StreamActor(
     def this(senderRef: ActorRef) = this(
       senderRef,
       FakeCancellable,
-      context.system.scheduler.scheduleOnce(refresh, self, Submit(senderRef)),
+      context.system.scheduler.scheduleOnce(refresh, self, Submit),
       false)
 
     /**
      * Renew timers
      */
-    def renew(): WaitingRequest = {
+    def renew: WaitingRequest = {
       if (itemCancellable.cancel()) {
         if (!hasResult && globalCancellable.cancel()) {
           new WaitingRequest(
             senderRef,
-            context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit(senderRef)),
-            context.system.scheduler.scheduleOnce(globalMaxWait, self, Submit(senderRef)),
+            context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit),
+            context.system.scheduler.scheduleOnce(globalMaxWait, self, Submit),
             true)
         }
         else
           new WaitingRequest(
             senderRef,
-            context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit(senderRef)),
+            context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit),
             globalCancellable,
             true)
       }
@@ -162,7 +162,7 @@ class StreamActor(
           aog :+ operation
         case _ ⇒
           logger.debug("Impossible")
-          ???
+          sys.error("")
       }
       context.become(receiveWithState(waitingRequest.map(_.renew), currentMessages + (requestId → Some(updatedOperationGroup))))
 
@@ -174,7 +174,7 @@ class StreamActor(
       }
       context.become(receiveWithState(Some(new WaitingRequest(sender)), currentMessages))
 
-    case Submit(senderRef) ⇒
+    case Submit ⇒
       waitingRequest match {
         case Some(wr) ⇒
           val (readyMessages, pendingMessages) = currentMessages.partition(_._2.fold(false)(_.isReady))
@@ -184,9 +184,9 @@ class StreamActor(
           logger.error("No request to submit !")
       }
 
-    case Initialize(requestId)     ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None)))
-    case operation: AuditOperation ⇒
-    case message                   ⇒ logger.warn(s"Unexpected message $message (${message.getClass})")
+    case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None)))
+    case _: AuditOperation     ⇒
+    case message               ⇒ logger.warn(s"Unexpected message $message (${message.getClass})")
   }
 
   def receive: Receive = receiveWithState(None, Map.empty[String, Option[StreamMessageGroup[_]]])

thehive-backend/conf/routes

@@ -54,6 +54,7 @@ GET      /api/alert                               controllers.AlertCtrl.find()
 POST     /api/alert/_search                       controllers.AlertCtrl.find()
 PATCH    /api/alert/_bulk                         controllers.AlertCtrl.bulkUpdate()
 POST     /api/alert/_stats                        controllers.AlertCtrl.stats()
+GET      /api/alert/_fixStatus                    controllers.AlertCtrl.fixStatus()
 POST     /api/alert                               controllers.AlertCtrl.create()
 GET      /api/alert/:alertId                      controllers.AlertCtrl.get(alertId)
 PATCH    /api/alert/:alertId                      controllers.AlertCtrl.update(alertId)

thehive-misp/app/connectors/misp/MispCtrl.scala

@@ -31,6 +31,7 @@ class MispCtrl @Inject() (
   private[MispCtrl] lazy val logger = Logger(getClass)
   val router = SimpleRouter {
     case GET(p"/_syncAlerts")    ⇒ syncAlerts
+    case GET(p"/_syncAllAlerts") ⇒ syncAllAlerts
     case GET(p"/_syncArtifacts") ⇒ syncArtifacts
     case r                       ⇒ throw NotFoundError(s"${r.uri} not found")
   }
@@ -41,6 +42,12 @@ class MispCtrl @Inject() (
       .map { m ⇒ Ok(Json.toJson(m)) }
   }
 
+  @Timed
+  def syncAllAlerts: Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒
+    mispSrv.fullSynchronize()
+      .map { m ⇒ Ok(Json.toJson(m)) }
+  }
+
   @Timed
   def syncArtifacts: Action[AnyContent] = authenticated(Role.admin) {
     eventSrv.publish(UpdateMispAlertArtifact())

thehive-misp/app/connectors/misp/MispSrv.scala

@@ -1,6 +1,5 @@
 package connectors.misp
 
-import java.text.SimpleDateFormat
 import java.util.Date
 import javax.inject.{ Inject, Provider, Singleton }
 
@@ -26,6 +25,7 @@ import play.api.libs.json._
 import play.api.{ Configuration, Environment, Logger }
 import services._
 
+import scala.collection.immutable
 import scala.concurrent.duration.{ DurationInt, DurationLong, FiniteDuration }
 import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.{ Failure, Success, Try }
@@ -157,81 +157,96 @@ class MispSrv @Inject() (
     // for each MISP server
     Source(mispConfig.connections.toList)
       // get last synchronization
-      .mapAsyncUnordered(1) { mcfg ⇒
-        alertSrv.stats(and("type" ~= "misp", "source" ~= mcfg.name), Seq(selectMax("lastSyncDate")))
-          .map { maxLastSyncDate ⇒ mcfg → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) }
-          .recover { case _ ⇒ mcfg → new Date(0) }
+      .mapAsyncUnordered(1) { mispConnection ⇒
+        alertSrv.stats(and("type" ~= "misp", "source" ~= mispConnection.name), Seq(selectMax("lastSyncDate")))
+          .map { maxLastSyncDate ⇒ mispConnection → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) }
+          .recover { case _ ⇒ mispConnection → new Date(0) }
       }
-      // get events that have been published after the last synchronization
       .flatMapConcat {
-        case (mcfg, lastSyncDate) ⇒
-          getEventsFromDate(mcfg, lastSyncDate).map((mcfg, lastSyncDate, _))
+        case (mispConnection, lastSyncDate) ⇒
+          synchronize(mispConnection, lastSyncDate)
       }
+      .runWith(Sink.seq)
+  }
+
+  def fullSynchronize()(implicit authContext: AuthContext): Future[immutable.Seq[Try[Alert]]] = {
+    Source(mispConfig.connections.toList)
+      .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(1)))
+      .runWith(Sink.seq)
+  }
+
+  def synchronize(mispConnection: MispConnection, lastSyncDate: Date)(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = {
+    logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate")
+    val fullSynchro = if (lastSyncDate.getTime == 1) Some(lastSyncDate) else None
+    // get events that have been published after the last synchronization
+    getEventsFromDate(mispConnection, lastSyncDate)
       // get related alert
-      .mapAsyncUnordered(1) {
-        case (mcfg, lastSyncDate, event) ⇒
-          logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}")
-          alertSrv.get("misp", event.source, event.sourceRef)
-            .map(a ⇒ (mcfg, lastSyncDate, event, a))
+      .mapAsyncUnordered(1) { event ⇒
+        logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}")
+        alertSrv.get("misp", event.source, event.sourceRef)
+          .map((event, _))
       }
       .mapAsyncUnordered(1) {
-        case (mcfg, lastSyncDate, event, alert) ⇒
-          logger.trace(s"MISP synchro ${mcfg.name} last sync at $lastSyncDate, event ${event.sourceRef}, alert ${alert.fold("no alert")("alert" + _.alertId())}")
-          logger.info(s"getting MISP event ${event.sourceRef}")
-          getAttributes(mcfg, event.sourceRef, alert.map(_ ⇒ lastSyncDate))
-            .map((mcfg, event, alert, _))
+        case (event, alert) ⇒
+          logger.trace(s"MISP synchro ${mispConnection.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}")
+          logger.info(s"getting MISP event ${event.source}:${event.sourceRef}")
+          getAttributes(mispConnection, event.sourceRef, fullSynchro.orElse(alert.map(_.lastSyncDate())))
+            .map((event, alert, _))
       }
       .mapAsyncUnordered(1) {
         // if there is no related alert, create a new one
-        case (mcfg, event, None, attrs) ⇒
-          logger.info(s"MISP event ${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)")
+        case (event, None, attrs) ⇒
+          logger.info(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)")
           val alertJson = Json.toJson(event).as[JsObject] +
             ("type" → JsString("misp")) +
-            ("caseTemplate" → mcfg.caseTemplate.fold[JsValue](JsNull)(JsString)) +
+            ("caseTemplate" → mispConnection.caseTemplate.fold[JsValue](JsNull)(JsString)) +
             ("artifacts" → JsArray(attrs))
           alertSrv.create(Fields(alertJson))
             .map(Success(_))
             .recover { case t ⇒ Failure(t) }
 
         // if a related alert exists, update it
-        case (_, event, Some(alert), attrs) ⇒
-          logger.info(s"MISP event ${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)")
+        case (event, Some(alert), attrs) ⇒
+          logger.info(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)")
           val alertJson = Json.toJson(event).as[JsObject] -
             "type" -
             "source" -
             "sourceRef" -
             "caseTemplate" -
             "date" +
             ("artifacts" → JsArray(attrs)) +
-            ("status" → (if (!alert.follow()) Json.toJson(alert.status())
+            // if this is a full synchronization, don't update alert status
+            ("status" → (if (!alert.follow() || fullSynchro.isDefined) Json.toJson(alert.status())
             else alert.status() match {
               case AlertStatus.New ⇒ Json.toJson(AlertStatus.New)
               case _               ⇒ Json.toJson(AlertStatus.Updated)
             }))
+          logger.debug(s"Update alert ${alert.id} with\n$alertJson")
           val fAlert = alertSrv.update(alert.id, Fields(alertJson))
           // if a case have been created, update it
           (alert.caze() match {
             case None ⇒ fAlert
             case Some(caze) ⇒
               for {
                 a ← fAlert
-                _ ← caseSrv.update(caze, Fields(alert.toCaseJson))
+                // if this is a full synchronization, don't update case status
+                caseFields = if (fullSynchro.isDefined) Fields(alert.toCaseJson).unset("status")
+                else Fields(alert.toCaseJson)
+                _ ← caseSrv.update(caze, caseFields)
                 _ ← artifactSrv.create(caze, attrs.map(Fields.apply))
               } yield a
           })
             .map(Success(_))
             .recover { case t ⇒ Failure(t) }
       }
-      .runWith(Sink.seq)
   }
 
   def getEventsFromDate(mispConnection: MispConnection, fromDate: Date): Source[MispAlert, NotUsed] = {
-    val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
-    val date = dateFormat.format(fromDate)
+    val date = fromDate.getTime / 1000
     Source
       .fromFuture {
         mispConnection("events/index")
-          .post(Json.obj("searchDatefrom" → date))
+          .post(Json.obj("searchpublish_timestamp" → date))
       }
       .mapConcat { response ⇒
         val eventJson = Json.parse(response.body)
@@ -249,7 +264,6 @@ class MispSrv @Inject() (
                 None
               }
           }
-          .filter(event ⇒ event.isPublished && event.date.after(fromDate))
 
         val eventJsonSize = eventJson.size
         val eventsSize = events.size
@@ -263,12 +277,16 @@ class MispSrv @Inject() (
     mispConnection: MispConnection,
     eventId: String,
     fromDate: Option[Date]): Future[Seq[JsObject]] = {
-    val date = fromDate.fold("null") { fd ⇒
-      val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
-      dateFormat.format(fd)
-    }
-    mispConnection(s"attributes/restSearch/json/null/null/null/null/null/$date/null/null/$eventId/false")
-      .get()
+
+    val date = fromDate.fold(0L)(_.getTime / 1000)
+
+    mispConnection(s"attributes/restSearch/json")
+      .post(Json.obj(
+        "request" → Json.obj(
+          "timestamp" → date,
+          "eventid" → eventId)))
+      // add ("deleted" → 1) to see also deleted attributes
+      // add ("deleted" → "only") to see only deleted attributes
       .map { response ⇒
         val refDate = fromDate.getOrElse(new Date(0))
         val artifactTags = JsString(s"src:${mispConnection.name}") +: JsArray(mispConnection.artifactTags.map(JsString))

ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js

@@ -81,7 +81,9 @@
             };
 
             $scope.addTask = function() {
-                $scope.openTaskDialog({order: $scope.template.tasks.length}, 'Add');
+                var order = $scope.template.tasks ? $scope.template.tasks.length : 0;
+
+                $scope.openTaskDialog({order: order}, 'Add');
             };
 
             $scope.editTask = function(task) {
@@ -175,7 +177,11 @@
 
             $scope.addTask = function() {
                 if(action === 'Add') {
+                    if($scope.template.tasks) {
                     $scope.template.tasks.push(task);
+                    } else {
+                        $scope.template.tasks = [task];
+                    }                    
                 }
 
                 $uibModalInstance.dismiss();

ui/app/views/components/header.component.html

@@ -45,7 +45,7 @@
                 <li ui-sref-active="active">
                     <a href ui-sref="app.alert-list">
                         Alerts
-                        <span class="badge alert-danger">{{(alertEvents.New.count || 0)  + (alertEvents.Update.count || 0)}}</span>
+                        <span class="badge alert-danger">{{(alertEvents.New.count || 0)  + (alertEvents.Updated.count || 0)}}</span>
                     </a>
                 </li>
                 <li class="hdivider hidden-xs"></li>

ui/app/views/partials/admin/case-templates.html

@@ -135,7 +135,7 @@ <h4 class="vpad10 text-primary">
                                     </div>
                                 </div>
                             </div>
-                            <div class="text-danger" ng-if="template.tasks.length === 0">
+                            <div class="text-danger" ng-if="!template.tasks || template.tasks.length === 0">
                                 <table class="table table-striped">
                                     <tr>
                                         <td>No tasks have been specified</td>

version.sbt

@@ -1 +1 @@
-version in ThisBuild := "2.11.2"
+version in ThisBuild := "2.11.3"