diff --git a/build.sbt b/build.sbt index ba7187d436..36624fb200 100644 --- a/build.sbt +++ b/build.sbt @@ -106,7 +106,7 @@ packageBin := { // DEB // version in Debian := version.value + "-2" debianPackageRecommends := Seq("elasticsearch") -debianPackageDependencies += "java8-runtime-headless | java8-runtime" +debianPackageDependencies += "openjdk-8-jre-headless" maintainerScripts in Debian := maintainerScriptsFromDirectory( baseDirectory.value / "package" / "debian", Seq(DebianConstants.Postinst, DebianConstants.Prerm, DebianConstants.Postrm) diff --git a/thehive-backend/app/controllers/AlertCtrl.scala b/thehive-backend/app/controllers/AlertCtrl.scala index aaf31c36cc..b89d0d7f01 100644 --- a/thehive-backend/app/controllers/AlertCtrl.scala +++ b/thehive-backend/app/controllers/AlertCtrl.scala @@ -105,6 +105,7 @@ class AlertCtrl @Inject() ( } yield renderer.toOutput(OK, updatedAlert) } + @Timed def createCase(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ for { alert ← alertSrv.get(id) @@ -118,8 +119,15 @@ class AlertCtrl @Inject() ( .map { alert ⇒ renderer.toOutput(OK, alert) } } + @Timed def unfollowAlert(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ alertSrv.setFollowAlert(id, follow = false) .map { alert ⇒ renderer.toOutput(OK, alert) } } + + @Timed + def fixStatus() = authenticated(Role.admin).async { implicit request ⇒ + alertSrv.fixStatus() + .map(_ ⇒ NoContent) + } } \ No newline at end of file diff --git a/thehive-backend/app/services/AlertSrv.scala b/thehive-backend/app/services/AlertSrv.scala index 6e6c9d181a..6aff658e2b 100644 --- a/thehive-backend/app/services/AlertSrv.scala +++ b/thehive-backend/app/services/AlertSrv.scala @@ -90,6 +90,9 @@ class AlertSrv( def update(id: String, fields: Fields)(implicit authContext: AuthContext): Future[Alert] = updateSrv[AlertModel, Alert](alertModel, id, fields) + def update(alert: Alert, fields: Fields)(implicit authContext: AuthContext): Future[Alert] = + updateSrv(alert, fields) + def bulkUpdate(ids: Seq[String], fields: Fields)(implicit authContext: AuthContext): Future[Seq[Try[Alert]]] = { updateSrv[AlertModel, Alert](alertModel, ids, fields) } @@ -209,4 +212,36 @@ class AlertSrv( def setFollowAlert(alertId: String, follow: Boolean)(implicit authContext: AuthContext): Future[Alert] = { updateSrv[AlertModel, Alert](alertModel, alertId, Fields(Json.obj("follow" → follow))) } + + def fixStatus()(implicit authContext: AuthContext): Future[Unit] = { + import org.elastic4play.services.QueryDSL._ + + val updatedStatusFields = Fields.empty.set("status", "Updated") + val (updateAlerts, updateAlertCount) = find("status" ~= "Update", Some("all"), Nil) + updateAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Update status")) + val updateAlertProcess = updateAlerts + .mapAsyncUnordered(3) { alert ⇒ + logger.debug(s"Updating alert ${alert.id} (status: Update -> Updated)") + update(alert, updatedStatusFields) + .andThen { + case Failure(error) ⇒ logger.warn(s"""Fail to set "Updated" status to alert ${alert.id}""", error) + } + } + + val ignoredStatusFields = Fields.empty.set("status", "Ignored") + val (ignoreAlerts, ignoreAlertCount) = find("status" ~= "Ignore", Some("all"), Nil) + ignoreAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Ignore status")) + val ignoreAlertProcess = ignoreAlerts + .mapAsyncUnordered(3) { alert ⇒ + logger.debug(s"Updating alert ${alert.id} (status: Ignore -> Ignored)") + update(alert, ignoredStatusFields) + .andThen { + case Failure(error) ⇒ logger.warn(s"""Fail to set "Ignored" status to alert ${alert.id}""", error) + } + } + + (updateAlertProcess ++ ignoreAlertProcess) + .runWith(Sink.ignore) + .map(_ ⇒ ()) + } } \ No newline at end of file diff --git a/thehive-backend/app/services/StreamSrv.scala b/thehive-backend/app/services/StreamSrv.scala index 0782b1511a..f8047c7430 100644 --- a/thehive-backend/app/services/StreamSrv.scala +++ b/thehive-backend/app/services/StreamSrv.scala @@ -43,7 +43,7 @@ object StreamActor { /* Ask messages, wait if there is no ready messages*/ case object GetOperations /* Pending messages must be sent to sender */ - case class Submit(senderRef: ActorRef) + case object Submit /* List of ready messages */ case class StreamMessages(messages: Seq[JsObject]) case object StreamNotFound @@ -70,25 +70,25 @@ class StreamActor( def this(senderRef: ActorRef) = this( senderRef, FakeCancellable, - context.system.scheduler.scheduleOnce(refresh, self, Submit(senderRef)), + context.system.scheduler.scheduleOnce(refresh, self, Submit), false) /** * Renew timers */ - def renew(): WaitingRequest = { + def renew: WaitingRequest = { if (itemCancellable.cancel()) { if (!hasResult && globalCancellable.cancel()) { new WaitingRequest( senderRef, - context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit(senderRef)), - context.system.scheduler.scheduleOnce(globalMaxWait, self, Submit(senderRef)), + context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit), + context.system.scheduler.scheduleOnce(globalMaxWait, self, Submit), true) } else new WaitingRequest( senderRef, - context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit(senderRef)), + context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit), globalCancellable, true) } @@ -162,7 +162,7 @@ class StreamActor( aog :+ operation case _ ⇒ logger.debug("Impossible") - ??? + sys.error("") } context.become(receiveWithState(waitingRequest.map(_.renew), currentMessages + (requestId → Some(updatedOperationGroup)))) @@ -174,7 +174,7 @@ class StreamActor( } context.become(receiveWithState(Some(new WaitingRequest(sender)), currentMessages)) - case Submit(senderRef) ⇒ + case Submit ⇒ waitingRequest match { case Some(wr) ⇒ val (readyMessages, pendingMessages) = currentMessages.partition(_._2.fold(false)(_.isReady)) @@ -184,9 +184,9 @@ class StreamActor( logger.error("No request to submit !") } - case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None))) - case operation: AuditOperation ⇒ - case message ⇒ logger.warn(s"Unexpected message $message (${message.getClass})") + case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None))) + case _: AuditOperation ⇒ + case message ⇒ logger.warn(s"Unexpected message $message (${message.getClass})") } def receive: Receive = receiveWithState(None, Map.empty[String, Option[StreamMessageGroup[_]]]) diff --git a/thehive-backend/conf/routes b/thehive-backend/conf/routes index e57c00e6e4..06d2b27400 100644 --- a/thehive-backend/conf/routes +++ b/thehive-backend/conf/routes @@ -54,6 +54,7 @@ GET /api/alert controllers.AlertCtrl.find() POST /api/alert/_search controllers.AlertCtrl.find() PATCH /api/alert/_bulk controllers.AlertCtrl.bulkUpdate() POST /api/alert/_stats controllers.AlertCtrl.stats() +GET /api/alert/_fixStatus controllers.AlertCtrl.fixStatus() POST /api/alert controllers.AlertCtrl.create() GET /api/alert/:alertId controllers.AlertCtrl.get(alertId) PATCH /api/alert/:alertId controllers.AlertCtrl.update(alertId) diff --git a/thehive-misp/app/connectors/misp/MispCtrl.scala b/thehive-misp/app/connectors/misp/MispCtrl.scala index c684825cd2..d64931b70c 100644 --- a/thehive-misp/app/connectors/misp/MispCtrl.scala +++ b/thehive-misp/app/connectors/misp/MispCtrl.scala @@ -31,6 +31,7 @@ class MispCtrl @Inject() ( private[MispCtrl] lazy val logger = Logger(getClass) val router = SimpleRouter { case GET(p"/_syncAlerts") ⇒ syncAlerts + case GET(p"/_syncAllAlerts") ⇒ syncAllAlerts case GET(p"/_syncArtifacts") ⇒ syncArtifacts case r ⇒ throw NotFoundError(s"${r.uri} not found") } @@ -41,6 +42,12 @@ class MispCtrl @Inject() ( .map { m ⇒ Ok(Json.toJson(m)) } } + @Timed + def syncAllAlerts: Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ + mispSrv.fullSynchronize() + .map { m ⇒ Ok(Json.toJson(m)) } + } + @Timed def syncArtifacts: Action[AnyContent] = authenticated(Role.admin) { eventSrv.publish(UpdateMispAlertArtifact()) diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index 452efe2c35..9e41b52af1 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -1,6 +1,5 @@ package connectors.misp -import java.text.SimpleDateFormat import java.util.Date import javax.inject.{ Inject, Provider, Singleton } @@ -26,6 +25,7 @@ import play.api.libs.json._ import play.api.{ Configuration, Environment, Logger } import services._ +import scala.collection.immutable import scala.concurrent.duration.{ DurationInt, DurationLong, FiniteDuration } import scala.concurrent.{ ExecutionContext, Future } import scala.util.{ Failure, Success, Try } @@ -157,45 +157,57 @@ class MispSrv @Inject() ( // for each MISP server Source(mispConfig.connections.toList) // get last synchronization - .mapAsyncUnordered(1) { mcfg ⇒ - alertSrv.stats(and("type" ~= "misp", "source" ~= mcfg.name), Seq(selectMax("lastSyncDate"))) - .map { maxLastSyncDate ⇒ mcfg → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) } - .recover { case _ ⇒ mcfg → new Date(0) } + .mapAsyncUnordered(1) { mispConnection ⇒ + alertSrv.stats(and("type" ~= "misp", "source" ~= mispConnection.name), Seq(selectMax("lastSyncDate"))) + .map { maxLastSyncDate ⇒ mispConnection → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) } + .recover { case _ ⇒ mispConnection → new Date(0) } } - // get events that have been published after the last synchronization .flatMapConcat { - case (mcfg, lastSyncDate) ⇒ - getEventsFromDate(mcfg, lastSyncDate).map((mcfg, lastSyncDate, _)) + case (mispConnection, lastSyncDate) ⇒ + synchronize(mispConnection, lastSyncDate) } + .runWith(Sink.seq) + } + + def fullSynchronize()(implicit authContext: AuthContext): Future[immutable.Seq[Try[Alert]]] = { + Source(mispConfig.connections.toList) + .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(1))) + .runWith(Sink.seq) + } + + def synchronize(mispConnection: MispConnection, lastSyncDate: Date)(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = { + logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate") + val fullSynchro = if (lastSyncDate.getTime == 1) Some(lastSyncDate) else None + // get events that have been published after the last synchronization + getEventsFromDate(mispConnection, lastSyncDate) // get related alert - .mapAsyncUnordered(1) { - case (mcfg, lastSyncDate, event) ⇒ - logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") - alertSrv.get("misp", event.source, event.sourceRef) - .map(a ⇒ (mcfg, lastSyncDate, event, a)) + .mapAsyncUnordered(1) { event ⇒ + logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") + alertSrv.get("misp", event.source, event.sourceRef) + .map((event, _)) } .mapAsyncUnordered(1) { - case (mcfg, lastSyncDate, event, alert) ⇒ - logger.trace(s"MISP synchro ${mcfg.name} last sync at $lastSyncDate, event ${event.sourceRef}, alert ${alert.fold("no alert")("alert" + _.alertId())}") - logger.info(s"getting MISP event ${event.sourceRef}") - getAttributes(mcfg, event.sourceRef, alert.map(_ ⇒ lastSyncDate)) - .map((mcfg, event, alert, _)) + case (event, alert) ⇒ + logger.trace(s"MISP synchro ${mispConnection.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}") + logger.info(s"getting MISP event ${event.source}:${event.sourceRef}") + getAttributes(mispConnection, event.sourceRef, fullSynchro.orElse(alert.map(_.lastSyncDate()))) + .map((event, alert, _)) } .mapAsyncUnordered(1) { // if there is no related alert, create a new one - case (mcfg, event, None, attrs) ⇒ - logger.info(s"MISP event ${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)") + case (event, None, attrs) ⇒ + logger.info(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)") val alertJson = Json.toJson(event).as[JsObject] + ("type" → JsString("misp")) + - ("caseTemplate" → mcfg.caseTemplate.fold[JsValue](JsNull)(JsString)) + + ("caseTemplate" → mispConnection.caseTemplate.fold[JsValue](JsNull)(JsString)) + ("artifacts" → JsArray(attrs)) alertSrv.create(Fields(alertJson)) .map(Success(_)) .recover { case t ⇒ Failure(t) } // if a related alert exists, update it - case (_, event, Some(alert), attrs) ⇒ - logger.info(s"MISP event ${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)") + case (event, Some(alert), attrs) ⇒ + logger.info(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)") val alertJson = Json.toJson(event).as[JsObject] - "type" - "source" - @@ -203,11 +215,13 @@ class MispSrv @Inject() ( "caseTemplate" - "date" + ("artifacts" → JsArray(attrs)) + - ("status" → (if (!alert.follow()) Json.toJson(alert.status()) + // if this is a full synchronization, don't update alert status + ("status" → (if (!alert.follow() || fullSynchro.isDefined) Json.toJson(alert.status()) else alert.status() match { case AlertStatus.New ⇒ Json.toJson(AlertStatus.New) case _ ⇒ Json.toJson(AlertStatus.Updated) })) + logger.debug(s"Update alert ${alert.id} with\n$alertJson") val fAlert = alertSrv.update(alert.id, Fields(alertJson)) // if a case have been created, update it (alert.caze() match { @@ -215,23 +229,24 @@ class MispSrv @Inject() ( case Some(caze) ⇒ for { a ← fAlert - _ ← caseSrv.update(caze, Fields(alert.toCaseJson)) + // if this is a full synchronization, don't update case status + caseFields = if (fullSynchro.isDefined) Fields(alert.toCaseJson).unset("status") + else Fields(alert.toCaseJson) + _ ← caseSrv.update(caze, caseFields) _ ← artifactSrv.create(caze, attrs.map(Fields.apply)) } yield a }) .map(Success(_)) .recover { case t ⇒ Failure(t) } } - .runWith(Sink.seq) } def getEventsFromDate(mispConnection: MispConnection, fromDate: Date): Source[MispAlert, NotUsed] = { - val dateFormat = new SimpleDateFormat("yyyy-MM-dd") - val date = dateFormat.format(fromDate) + val date = fromDate.getTime / 1000 Source .fromFuture { mispConnection("events/index") - .post(Json.obj("searchDatefrom" → date)) + .post(Json.obj("searchpublish_timestamp" → date)) } .mapConcat { response ⇒ val eventJson = Json.parse(response.body) @@ -249,7 +264,6 @@ class MispSrv @Inject() ( None } } - .filter(event ⇒ event.isPublished && event.date.after(fromDate)) val eventJsonSize = eventJson.size val eventsSize = events.size @@ -263,12 +277,16 @@ class MispSrv @Inject() ( mispConnection: MispConnection, eventId: String, fromDate: Option[Date]): Future[Seq[JsObject]] = { - val date = fromDate.fold("null") { fd ⇒ - val dateFormat = new SimpleDateFormat("yyyy-MM-dd") - dateFormat.format(fd) - } - mispConnection(s"attributes/restSearch/json/null/null/null/null/null/$date/null/null/$eventId/false") - .get() + + val date = fromDate.fold(0L)(_.getTime / 1000) + + mispConnection(s"attributes/restSearch/json") + .post(Json.obj( + "request" → Json.obj( + "timestamp" → date, + "eventid" → eventId))) + // add ("deleted" → 1) to see also deleted attributes + // add ("deleted" → "only") to see only deleted attributes .map { response ⇒ val refDate = fromDate.getOrElse(new Date(0)) val artifactTags = JsString(s"src:${mispConnection.name}") +: JsArray(mispConnection.artifactTags.map(JsString)) diff --git a/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js b/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js index 6032f443ab..cae892a826 100644 --- a/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js +++ b/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js @@ -81,7 +81,9 @@ }; $scope.addTask = function() { - $scope.openTaskDialog({order: $scope.template.tasks.length}, 'Add'); + var order = $scope.template.tasks ? $scope.template.tasks.length : 0; + + $scope.openTaskDialog({order: order}, 'Add'); }; $scope.editTask = function(task) { @@ -175,7 +177,11 @@ $scope.addTask = function() { if(action === 'Add') { + if($scope.template.tasks) { $scope.template.tasks.push(task); + } else { + $scope.template.tasks = [task]; + } } $uibModalInstance.dismiss(); diff --git a/ui/app/views/components/header.component.html b/ui/app/views/components/header.component.html index fb224b6b8a..3de2d0e7ac 100644 --- a/ui/app/views/components/header.component.html +++ b/ui/app/views/components/header.component.html @@ -45,7 +45,7 @@
  • Alerts - {{(alertEvents.New.count || 0) + (alertEvents.Update.count || 0)}} + {{(alertEvents.New.count || 0) + (alertEvents.Updated.count || 0)}}
    • diff --git a/ui/app/views/partials/admin/case-templates.html b/ui/app/views/partials/admin/case-templates.html index 03b9894db4..e96ec2db77 100644 --- a/ui/app/views/partials/admin/case-templates.html +++ b/ui/app/views/partials/admin/case-templates.html @@ -135,7 +135,7 @@

    -
    +
    diff --git a/version.sbt b/version.sbt index 3bccd1fbe1..9306e5fc8e 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.11.2" +version in ThisBuild := "2.11.3"
    No tasks have been specified