From 75b17bbb54146e12f8cb2424a38474d05353ea3d Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Jun 2017 16:05:16 +0200 Subject: [PATCH 1/9] #236 Retrieve MISP events according to their publish date --- .../app/connectors/misp/MispSrv.scala | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index 452efe2c35..4a4ce27e1b 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -165,20 +165,21 @@ class MispSrv @Inject() ( // get events that have been published after the last synchronization .flatMapConcat { case (mcfg, lastSyncDate) ⇒ - getEventsFromDate(mcfg, lastSyncDate).map((mcfg, lastSyncDate, _)) + getEventsFromDate(mcfg, lastSyncDate) + .map((mcfg, _)) } // get related alert .mapAsyncUnordered(1) { - case (mcfg, lastSyncDate, event) ⇒ + case (mcfg, event) ⇒ logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") alertSrv.get("misp", event.source, event.sourceRef) - .map(a ⇒ (mcfg, lastSyncDate, event, a)) + .map((mcfg, event, _)) } .mapAsyncUnordered(1) { - case (mcfg, lastSyncDate, event, alert) ⇒ - logger.trace(s"MISP synchro ${mcfg.name} last sync at $lastSyncDate, event ${event.sourceRef}, alert ${alert.fold("no alert")("alert" + _.alertId())}") + case (mcfg, event, alert) ⇒ + logger.trace(s"MISP synchro ${mcfg.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}") logger.info(s"getting MISP event ${event.sourceRef}") - getAttributes(mcfg, event.sourceRef, alert.map(_ ⇒ lastSyncDate)) + getAttributes(mcfg, event.sourceRef, alert.map(_.lastSyncDate())) .map((mcfg, event, alert, _)) } .mapAsyncUnordered(1) { @@ -226,12 +227,11 @@ class MispSrv @Inject() ( } def getEventsFromDate(mispConnection: MispConnection, fromDate: Date): Source[MispAlert, NotUsed] = { - val dateFormat = new SimpleDateFormat("yyyy-MM-dd") - val date = dateFormat.format(fromDate) + val date = fromDate.getTime / 1000 Source .fromFuture { mispConnection("events/index") - .post(Json.obj("searchDatefrom" → date)) + .post(Json.obj("searchpublish_timestamp" → date)) } .mapConcat { response ⇒ val eventJson = Json.parse(response.body) @@ -249,7 +249,6 @@ class MispSrv @Inject() ( None } } - .filter(event ⇒ event.isPublished && event.date.after(fromDate)) val eventJsonSize = eventJson.size val eventsSize = events.size @@ -263,12 +262,16 @@ class MispSrv @Inject() ( mispConnection: MispConnection, eventId: String, fromDate: Option[Date]): Future[Seq[JsObject]] = { - val date = fromDate.fold("null") { fd ⇒ - val dateFormat = new SimpleDateFormat("yyyy-MM-dd") - dateFormat.format(fd) - } - mispConnection(s"attributes/restSearch/json/null/null/null/null/null/$date/null/null/$eventId/false") - .get() + + val date = fromDate.fold(0L)(_.getTime / 1000) + + mispConnection(s"attributes/restSearch/json") + .post(Json.obj( + "request" → Json.obj( + "timestamp" → date, + "eventid" → eventId))) + // add ("deleted" → 1) to see also deleted attributes + // add ("deleted" → "only") to see only deleted attributes .map { response ⇒ val refDate = fromDate.getOrElse(new Date(0)) val artifactTags = JsString(s"src:${mispConnection.name}") +: JsArray(mispConnection.artifactTags.map(JsString)) From 064b7e0075cdbd6661a24b493bc1073855f0a2b7 Mon Sep 17 00:00:00 2001 From: To-om Date: Tue, 13 Jun 2017 15:24:02 +0200 Subject: [PATCH 2/9] #236 Add API to resynchronize all MISP alerts. This API also fix alert status name (d451ed0) --- thehive-backend/app/services/AlertSrv.scala | 3 + thehive-backend/app/services/StreamSrv.scala | 22 ++--- .../app/connectors/misp/MispCtrl.scala | 7 ++ .../app/connectors/misp/MispSrv.scala | 82 ++++++++++++++----- 4 files changed, 81 insertions(+), 33 deletions(-) diff --git a/thehive-backend/app/services/AlertSrv.scala b/thehive-backend/app/services/AlertSrv.scala index 6e6c9d181a..fdecccb3ed 100644 --- a/thehive-backend/app/services/AlertSrv.scala +++ b/thehive-backend/app/services/AlertSrv.scala @@ -90,6 +90,9 @@ class AlertSrv( def update(id: String, fields: Fields)(implicit authContext: AuthContext): Future[Alert] = updateSrv[AlertModel, Alert](alertModel, id, fields) + def update(alert: Alert, fields: Fields)(implicit authContext: AuthContext): Future[Alert] = + updateSrv(alert, fields) + def bulkUpdate(ids: Seq[String], fields: Fields)(implicit authContext: AuthContext): Future[Seq[Try[Alert]]] = { updateSrv[AlertModel, Alert](alertModel, ids, fields) } diff --git a/thehive-backend/app/services/StreamSrv.scala b/thehive-backend/app/services/StreamSrv.scala index 0782b1511a..f8047c7430 100644 --- a/thehive-backend/app/services/StreamSrv.scala +++ b/thehive-backend/app/services/StreamSrv.scala @@ -43,7 +43,7 @@ object StreamActor { /* Ask messages, wait if there is no ready messages*/ case object GetOperations /* Pending messages must be sent to sender */ - case class Submit(senderRef: ActorRef) + case object Submit /* List of ready messages */ case class StreamMessages(messages: Seq[JsObject]) case object StreamNotFound @@ -70,25 +70,25 @@ class StreamActor( def this(senderRef: ActorRef) = this( senderRef, FakeCancellable, - context.system.scheduler.scheduleOnce(refresh, self, Submit(senderRef)), + context.system.scheduler.scheduleOnce(refresh, self, Submit), false) /** * Renew timers */ - def renew(): WaitingRequest = { + def renew: WaitingRequest = { if (itemCancellable.cancel()) { if (!hasResult && globalCancellable.cancel()) { new WaitingRequest( senderRef, - context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit(senderRef)), - context.system.scheduler.scheduleOnce(globalMaxWait, self, Submit(senderRef)), + context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit), + context.system.scheduler.scheduleOnce(globalMaxWait, self, Submit), true) } else new WaitingRequest( senderRef, - context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit(senderRef)), + context.system.scheduler.scheduleOnce(nextItemMaxWait, self, Submit), globalCancellable, true) } @@ -162,7 +162,7 @@ class StreamActor( aog :+ operation case _ ⇒ logger.debug("Impossible") - ??? + sys.error("") } context.become(receiveWithState(waitingRequest.map(_.renew), currentMessages + (requestId → Some(updatedOperationGroup)))) @@ -174,7 +174,7 @@ class StreamActor( } context.become(receiveWithState(Some(new WaitingRequest(sender)), currentMessages)) - case Submit(senderRef) ⇒ + case Submit ⇒ waitingRequest match { case Some(wr) ⇒ val (readyMessages, pendingMessages) = currentMessages.partition(_._2.fold(false)(_.isReady)) @@ -184,9 +184,9 @@ class StreamActor( logger.error("No request to submit !") } - case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None))) - case operation: AuditOperation ⇒ - case message ⇒ logger.warn(s"Unexpected message $message (${message.getClass})") + case Initialize(requestId) ⇒ context.become(receiveWithState(waitingRequest, currentMessages + (requestId → None))) + case _: AuditOperation ⇒ + case message ⇒ logger.warn(s"Unexpected message $message (${message.getClass})") } def receive: Receive = receiveWithState(None, Map.empty[String, Option[StreamMessageGroup[_]]]) diff --git a/thehive-misp/app/connectors/misp/MispCtrl.scala b/thehive-misp/app/connectors/misp/MispCtrl.scala index c684825cd2..d64931b70c 100644 --- a/thehive-misp/app/connectors/misp/MispCtrl.scala +++ b/thehive-misp/app/connectors/misp/MispCtrl.scala @@ -31,6 +31,7 @@ class MispCtrl @Inject() ( private[MispCtrl] lazy val logger = Logger(getClass) val router = SimpleRouter { case GET(p"/_syncAlerts") ⇒ syncAlerts + case GET(p"/_syncAllAlerts") ⇒ syncAllAlerts case GET(p"/_syncArtifacts") ⇒ syncArtifacts case r ⇒ throw NotFoundError(s"${r.uri} not found") } @@ -41,6 +42,12 @@ class MispCtrl @Inject() ( .map { m ⇒ Ok(Json.toJson(m)) } } + @Timed + def syncAllAlerts: Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ + mispSrv.fullSynchronize() + .map { m ⇒ Ok(Json.toJson(m)) } + } + @Timed def syncArtifacts: Action[AnyContent] = authenticated(Role.admin) { eventSrv.publish(UpdateMispAlertArtifact()) diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index 4a4ce27e1b..ba1da6d8c6 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -1,6 +1,5 @@ package connectors.misp -import java.text.SimpleDateFormat import java.util.Date import javax.inject.{ Inject, Provider, Singleton } @@ -157,46 +156,86 @@ class MispSrv @Inject() ( // for each MISP server Source(mispConfig.connections.toList) // get last synchronization - .mapAsyncUnordered(1) { mcfg ⇒ - alertSrv.stats(and("type" ~= "misp", "source" ~= mcfg.name), Seq(selectMax("lastSyncDate"))) - .map { maxLastSyncDate ⇒ mcfg → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) } - .recover { case _ ⇒ mcfg → new Date(0) } + .mapAsyncUnordered(1) { mispConnection ⇒ + alertSrv.stats(and("type" ~= "misp", "source" ~= mispConnection.name), Seq(selectMax("lastSyncDate"))) + .map { maxLastSyncDate ⇒ mispConnection → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) } + .recover { case _ ⇒ mispConnection → new Date(0) } } - // get events that have been published after the last synchronization .flatMapConcat { - case (mcfg, lastSyncDate) ⇒ - getEventsFromDate(mcfg, lastSyncDate) - .map((mcfg, _)) + case (mispConnection, lastSyncDate) ⇒ + synchronize(mispConnection, lastSyncDate) } + .runWith(Sink.seq) + } + + def fullSynchronize()(implicit authContext: AuthContext) = { + import org.elastic4play.services.QueryDSL._ + + val updatedStatusFields = Fields.empty.set("status", "Updated") + val (updateAlerts, updateAlertCount) = alertSrv.find("status" ~= "Update", Some("all"), Nil) + updateAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Update status")) + updateAlerts + .mapAsyncUnordered(1) { alert ⇒ + logger.debug(s"Updating alert ${alert.id} (status: Update -> Updated") + alertSrv.update(alert, updatedStatusFields) + .recover { + case error ⇒ logger.warn(s"""Fail to set "Updated" status to alert ${alert.id}""", error) + } + } + .runWith(Sink.ignore) + + val ignoredStatusFields = Fields.empty.set("status", "Ignored") + val (ignoreAlerts, ignoreAlertCount) = alertSrv.find("status" ~= "Ignore", Some("all"), Nil) + ignoreAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Ignore status")) + ignoreAlerts + .mapAsyncUnordered(1) { alert ⇒ + logger.debug(s"Updating alert ${alert.id} (status: Ignore -> Ignored") + alertSrv.update(alert, ignoredStatusFields) + .recover { + case error ⇒ logger.warn(s"""Fail to set "Ignored" status to alert ${alert.id}""", error) + } + } + .runWith(Sink.ignore) + + Source(mispConfig.connections.toList) + .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(0))) + .runWith(Sink.seq) + } + + def synchronize(mispConnection: MispConnection, lastSyncDate: Date)(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = { + logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate") + val fullSynchro = if (lastSyncDate.getTime == 0) Some(lastSyncDate) else None + // get events that have been published after the last synchronization + getEventsFromDate(mispConnection, lastSyncDate) // get related alert .mapAsyncUnordered(1) { - case (mcfg, event) ⇒ + case event ⇒ logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") alertSrv.get("misp", event.source, event.sourceRef) - .map((mcfg, event, _)) + .map((event, _)) } .mapAsyncUnordered(1) { - case (mcfg, event, alert) ⇒ - logger.trace(s"MISP synchro ${mcfg.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}") - logger.info(s"getting MISP event ${event.sourceRef}") - getAttributes(mcfg, event.sourceRef, alert.map(_.lastSyncDate())) - .map((mcfg, event, alert, _)) + case (event, alert) ⇒ + logger.trace(s"MISP synchro ${mispConnection.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}") + logger.info(s"getting MISP event ${event.source}:${event.sourceRef}") + getAttributes(mispConnection, event.sourceRef, fullSynchro.orElse(alert.map(_.lastSyncDate()))) + .map((event, alert, _)) } .mapAsyncUnordered(1) { // if there is no related alert, create a new one - case (mcfg, event, None, attrs) ⇒ - logger.info(s"MISP event ${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)") + case (event, None, attrs) ⇒ + logger.info(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)") val alertJson = Json.toJson(event).as[JsObject] + ("type" → JsString("misp")) + - ("caseTemplate" → mcfg.caseTemplate.fold[JsValue](JsNull)(JsString)) + + ("caseTemplate" → mispConnection.caseTemplate.fold[JsValue](JsNull)(JsString)) + ("artifacts" → JsArray(attrs)) alertSrv.create(Fields(alertJson)) .map(Success(_)) .recover { case t ⇒ Failure(t) } // if a related alert exists, update it - case (_, event, Some(alert), attrs) ⇒ - logger.info(s"MISP event ${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)") + case (event, Some(alert), attrs) ⇒ + logger.info(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)") val alertJson = Json.toJson(event).as[JsObject] - "type" - "source" - @@ -223,7 +262,6 @@ class MispSrv @Inject() ( .map(Success(_)) .recover { case t ⇒ Failure(t) } } - .runWith(Sink.seq) } def getEventsFromDate(mispConnection: MispConnection, fromDate: Date): Source[MispAlert, NotUsed] = { From 516d7502e2f168b299c98ee7b77e8d59bb902870 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Jun 2017 10:36:36 +0200 Subject: [PATCH 3/9] #238 Change Debian package dependencies to select OpenJDK-8 explicitly --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index ba7187d436..247d46066a 100644 --- a/build.sbt +++ b/build.sbt @@ -104,9 +104,9 @@ packageBin := { (packageBin in Rpm).value } // DEB // -version in Debian := version.value + "-2" +version in Debian := version.value + "-1" debianPackageRecommends := Seq("elasticsearch") -debianPackageDependencies += "java8-runtime-headless | java8-runtime" +debianPackageDependencies += "openjdk-8-jre-headless" maintainerScripts in Debian := maintainerScriptsFromDirectory( baseDirectory.value / "package" / "debian", Seq(DebianConstants.Postinst, DebianConstants.Prerm, DebianConstants.Postrm) From 3c5f0c9efa3ad74ce4c8e7e22377002c498ff4e6 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Jun 2017 10:36:57 +0200 Subject: [PATCH 4/9] Bump version --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index 3bccd1fbe1..9306e5fc8e 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.11.2" +version in ThisBuild := "2.11.3" From 689801f2ccb5704104def6abb39ef414b51e3ff1 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Jun 2017 10:40:17 +0200 Subject: [PATCH 5/9] #236 Wait end of alert status fix before starting full synchronization --- .../app/connectors/misp/MispSrv.scala | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index ba1da6d8c6..cba0ba96b2 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -25,6 +25,7 @@ import play.api.libs.json._ import play.api.{ Configuration, Environment, Logger } import services._ +import scala.collection.immutable import scala.concurrent.duration.{ DurationInt, DurationLong, FiniteDuration } import scala.concurrent.{ ExecutionContext, Future } import scala.util.{ Failure, Success, Try } @@ -168,13 +169,13 @@ class MispSrv @Inject() ( .runWith(Sink.seq) } - def fullSynchronize()(implicit authContext: AuthContext) = { + def fullSynchronize()(implicit authContext: AuthContext): Future[immutable.Seq[Try[Alert]]] = { import org.elastic4play.services.QueryDSL._ val updatedStatusFields = Fields.empty.set("status", "Updated") val (updateAlerts, updateAlertCount) = alertSrv.find("status" ~= "Update", Some("all"), Nil) updateAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Update status")) - updateAlerts + val updateAlertProcess = updateAlerts .mapAsyncUnordered(1) { alert ⇒ logger.debug(s"Updating alert ${alert.id} (status: Update -> Updated") alertSrv.update(alert, updatedStatusFields) @@ -187,7 +188,7 @@ class MispSrv @Inject() ( val ignoredStatusFields = Fields.empty.set("status", "Ignored") val (ignoreAlerts, ignoreAlertCount) = alertSrv.find("status" ~= "Ignore", Some("all"), Nil) ignoreAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Ignore status")) - ignoreAlerts + val ignoreAlertProcess = ignoreAlerts .mapAsyncUnordered(1) { alert ⇒ logger.debug(s"Updating alert ${alert.id} (status: Ignore -> Ignored") alertSrv.update(alert, ignoredStatusFields) @@ -197,9 +198,13 @@ class MispSrv @Inject() ( } .runWith(Sink.ignore) - Source(mispConfig.connections.toList) - .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(0))) - .runWith(Sink.seq) + updateAlertProcess + .flatMap(_ ⇒ ignoreAlertProcess) + .flatMap { _ ⇒ + Source(mispConfig.connections.toList) + .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(0))) + .runWith(Sink.seq) + } } def synchronize(mispConnection: MispConnection, lastSyncDate: Date)(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = { @@ -208,11 +213,10 @@ class MispSrv @Inject() ( // get events that have been published after the last synchronization getEventsFromDate(mispConnection, lastSyncDate) // get related alert - .mapAsyncUnordered(1) { - case event ⇒ - logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") - alertSrv.get("misp", event.source, event.sourceRef) - .map((event, _)) + .mapAsyncUnordered(1) { event ⇒ + logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") + alertSrv.get("misp", event.source, event.sourceRef) + .map((event, _)) } .mapAsyncUnordered(1) { case (event, alert) ⇒ @@ -248,6 +252,7 @@ class MispSrv @Inject() ( case AlertStatus.New ⇒ Json.toJson(AlertStatus.New) case _ ⇒ Json.toJson(AlertStatus.Updated) })) + logger.debug(s"Update alert ${alert.id} with\n$alertJson") val fAlert = alertSrv.update(alert.id, Fields(alertJson)) // if a case have been created, update it (alert.caze() match { From 0bd6c542dc6e3cd95dc1d7de7e1de428ace20615 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Jun 2017 13:07:16 +0200 Subject: [PATCH 6/9] #236 Add API to fix alert status --- .../app/controllers/AlertCtrl.scala | 8 ++++ thehive-backend/app/services/AlertSrv.scala | 32 +++++++++++++ thehive-backend/conf/routes | 1 + .../app/connectors/misp/MispSrv.scala | 48 ++++--------------- 4 files changed, 51 insertions(+), 38 deletions(-) diff --git a/thehive-backend/app/controllers/AlertCtrl.scala b/thehive-backend/app/controllers/AlertCtrl.scala index aaf31c36cc..b89d0d7f01 100644 --- a/thehive-backend/app/controllers/AlertCtrl.scala +++ b/thehive-backend/app/controllers/AlertCtrl.scala @@ -105,6 +105,7 @@ class AlertCtrl @Inject() ( } yield renderer.toOutput(OK, updatedAlert) } + @Timed def createCase(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ for { alert ← alertSrv.get(id) @@ -118,8 +119,15 @@ class AlertCtrl @Inject() ( .map { alert ⇒ renderer.toOutput(OK, alert) } } + @Timed def unfollowAlert(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ alertSrv.setFollowAlert(id, follow = false) .map { alert ⇒ renderer.toOutput(OK, alert) } } + + @Timed + def fixStatus() = authenticated(Role.admin).async { implicit request ⇒ + alertSrv.fixStatus() + .map(_ ⇒ NoContent) + } } \ No newline at end of file diff --git a/thehive-backend/app/services/AlertSrv.scala b/thehive-backend/app/services/AlertSrv.scala index fdecccb3ed..6aff658e2b 100644 --- a/thehive-backend/app/services/AlertSrv.scala +++ b/thehive-backend/app/services/AlertSrv.scala @@ -212,4 +212,36 @@ class AlertSrv( def setFollowAlert(alertId: String, follow: Boolean)(implicit authContext: AuthContext): Future[Alert] = { updateSrv[AlertModel, Alert](alertModel, alertId, Fields(Json.obj("follow" → follow))) } + + def fixStatus()(implicit authContext: AuthContext): Future[Unit] = { + import org.elastic4play.services.QueryDSL._ + + val updatedStatusFields = Fields.empty.set("status", "Updated") + val (updateAlerts, updateAlertCount) = find("status" ~= "Update", Some("all"), Nil) + updateAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Update status")) + val updateAlertProcess = updateAlerts + .mapAsyncUnordered(3) { alert ⇒ + logger.debug(s"Updating alert ${alert.id} (status: Update -> Updated)") + update(alert, updatedStatusFields) + .andThen { + case Failure(error) ⇒ logger.warn(s"""Fail to set "Updated" status to alert ${alert.id}""", error) + } + } + + val ignoredStatusFields = Fields.empty.set("status", "Ignored") + val (ignoreAlerts, ignoreAlertCount) = find("status" ~= "Ignore", Some("all"), Nil) + ignoreAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Ignore status")) + val ignoreAlertProcess = ignoreAlerts + .mapAsyncUnordered(3) { alert ⇒ + logger.debug(s"Updating alert ${alert.id} (status: Ignore -> Ignored)") + update(alert, ignoredStatusFields) + .andThen { + case Failure(error) ⇒ logger.warn(s"""Fail to set "Ignored" status to alert ${alert.id}""", error) + } + } + + (updateAlertProcess ++ ignoreAlertProcess) + .runWith(Sink.ignore) + .map(_ ⇒ ()) + } } \ No newline at end of file diff --git a/thehive-backend/conf/routes b/thehive-backend/conf/routes index e57c00e6e4..27e581637d 100644 --- a/thehive-backend/conf/routes +++ b/thehive-backend/conf/routes @@ -63,6 +63,7 @@ POST /api/alert/:alertId/markAsUnread controllers.AlertCtrl.markAsUn POST /api/alert/:alertId/createCase controllers.AlertCtrl.createCase(alertId) POST /api/alert/:alertId/follow controllers.AlertCtrl.followAlert(alertId) POST /api/alert/:alertId/unfollow controllers.AlertCtrl.unfollowAlert(alertId) +GET /api/alert/_fixStatus controllers.AlertCtrl.fixStatus() GET /api/flow controllers.FlowCtrl.flow(rootId: Option[String], count: Option[Int]) diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index cba0ba96b2..9e41b52af1 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -170,46 +170,14 @@ class MispSrv @Inject() ( } def fullSynchronize()(implicit authContext: AuthContext): Future[immutable.Seq[Try[Alert]]] = { - import org.elastic4play.services.QueryDSL._ - - val updatedStatusFields = Fields.empty.set("status", "Updated") - val (updateAlerts, updateAlertCount) = alertSrv.find("status" ~= "Update", Some("all"), Nil) - updateAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Update status")) - val updateAlertProcess = updateAlerts - .mapAsyncUnordered(1) { alert ⇒ - logger.debug(s"Updating alert ${alert.id} (status: Update -> Updated") - alertSrv.update(alert, updatedStatusFields) - .recover { - case error ⇒ logger.warn(s"""Fail to set "Updated" status to alert ${alert.id}""", error) - } - } - .runWith(Sink.ignore) - - val ignoredStatusFields = Fields.empty.set("status", "Ignored") - val (ignoreAlerts, ignoreAlertCount) = alertSrv.find("status" ~= "Ignore", Some("all"), Nil) - ignoreAlertCount.foreach(c ⇒ logger.info(s"Updating $c alert with Ignore status")) - val ignoreAlertProcess = ignoreAlerts - .mapAsyncUnordered(1) { alert ⇒ - logger.debug(s"Updating alert ${alert.id} (status: Ignore -> Ignored") - alertSrv.update(alert, ignoredStatusFields) - .recover { - case error ⇒ logger.warn(s"""Fail to set "Ignored" status to alert ${alert.id}""", error) - } - } - .runWith(Sink.ignore) - - updateAlertProcess - .flatMap(_ ⇒ ignoreAlertProcess) - .flatMap { _ ⇒ - Source(mispConfig.connections.toList) - .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(0))) - .runWith(Sink.seq) - } + Source(mispConfig.connections.toList) + .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(1))) + .runWith(Sink.seq) } def synchronize(mispConnection: MispConnection, lastSyncDate: Date)(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = { logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate") - val fullSynchro = if (lastSyncDate.getTime == 0) Some(lastSyncDate) else None + val fullSynchro = if (lastSyncDate.getTime == 1) Some(lastSyncDate) else None // get events that have been published after the last synchronization getEventsFromDate(mispConnection, lastSyncDate) // get related alert @@ -247,7 +215,8 @@ class MispSrv @Inject() ( "caseTemplate" - "date" + ("artifacts" → JsArray(attrs)) + - ("status" → (if (!alert.follow()) Json.toJson(alert.status()) + // if this is a full synchronization, don't update alert status + ("status" → (if (!alert.follow() || fullSynchro.isDefined) Json.toJson(alert.status()) else alert.status() match { case AlertStatus.New ⇒ Json.toJson(AlertStatus.New) case _ ⇒ Json.toJson(AlertStatus.Updated) @@ -260,7 +229,10 @@ class MispSrv @Inject() ( case Some(caze) ⇒ for { a ← fAlert - _ ← caseSrv.update(caze, Fields(alert.toCaseJson)) + // if this is a full synchronization, don't update case status + caseFields = if (fullSynchro.isDefined) Fields(alert.toCaseJson).unset("status") + else Fields(alert.toCaseJson) + _ ← caseSrv.update(caze, caseFields) _ ← artifactSrv.create(caze, attrs.map(Fields.apply)) } yield a }) From f591c5c43a214448d1de52381e8d410c6d05950f Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Jun 2017 14:02:42 +0200 Subject: [PATCH 7/9] #236 Change route order to make fixStatus available --- thehive-backend/conf/routes | 2 +- ui/app/views/components/header.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive-backend/conf/routes b/thehive-backend/conf/routes index 27e581637d..06d2b27400 100644 --- a/thehive-backend/conf/routes +++ b/thehive-backend/conf/routes @@ -54,6 +54,7 @@ GET /api/alert controllers.AlertCtrl.find() POST /api/alert/_search controllers.AlertCtrl.find() PATCH /api/alert/_bulk controllers.AlertCtrl.bulkUpdate() POST /api/alert/_stats controllers.AlertCtrl.stats() +GET /api/alert/_fixStatus controllers.AlertCtrl.fixStatus() POST /api/alert controllers.AlertCtrl.create() GET /api/alert/:alertId controllers.AlertCtrl.get(alertId) PATCH /api/alert/:alertId controllers.AlertCtrl.update(alertId) @@ -63,7 +64,6 @@ POST /api/alert/:alertId/markAsUnread controllers.AlertCtrl.markAsUn POST /api/alert/:alertId/createCase controllers.AlertCtrl.createCase(alertId) POST /api/alert/:alertId/follow controllers.AlertCtrl.followAlert(alertId) POST /api/alert/:alertId/unfollow controllers.AlertCtrl.unfollowAlert(alertId) -GET /api/alert/_fixStatus controllers.AlertCtrl.fixStatus() GET /api/flow controllers.FlowCtrl.flow(rootId: Option[String], count: Option[Int]) diff --git a/ui/app/views/components/header.component.html b/ui/app/views/components/header.component.html index fb224b6b8a..3de2d0e7ac 100644 --- a/ui/app/views/components/header.component.html +++ b/ui/app/views/components/header.component.html @@ -45,7 +45,7 @@
  • Alerts - {{(alertEvents.New.count || 0) + (alertEvents.Update.count || 0)}} + {{(alertEvents.New.count || 0) + (alertEvents.Updated.count || 0)}}
    • From 319641c5b1f34a9f059681056abc65b4c6bdb3ac Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Wed, 14 Jun 2017 14:23:21 +0200 Subject: [PATCH 8/9] #239 Fix the case template tasks issue --- .../scripts/controllers/admin/AdminCaseTemplatesCtrl.js | 8 +++++++- ui/app/views/partials/admin/case-templates.html | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js b/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js index 6032f443ab..cae892a826 100644 --- a/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js +++ b/ui/app/scripts/controllers/admin/AdminCaseTemplatesCtrl.js @@ -81,7 +81,9 @@ }; $scope.addTask = function() { - $scope.openTaskDialog({order: $scope.template.tasks.length}, 'Add'); + var order = $scope.template.tasks ? $scope.template.tasks.length : 0; + + $scope.openTaskDialog({order: order}, 'Add'); }; $scope.editTask = function(task) { @@ -175,7 +177,11 @@ $scope.addTask = function() { if(action === 'Add') { + if($scope.template.tasks) { $scope.template.tasks.push(task); + } else { + $scope.template.tasks = [task]; + } } $uibModalInstance.dismiss(); diff --git a/ui/app/views/partials/admin/case-templates.html b/ui/app/views/partials/admin/case-templates.html index 03b9894db4..e96ec2db77 100644 --- a/ui/app/views/partials/admin/case-templates.html +++ b/ui/app/views/partials/admin/case-templates.html @@ -135,7 +135,7 @@

    -
    +
    From 054240333c1dcea9ed94ca14f47d4dc758547be0 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Jun 2017 15:05:30 +0200 Subject: [PATCH 9/9] Bump deb package version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 247d46066a..36624fb200 100644 --- a/build.sbt +++ b/build.sbt @@ -104,7 +104,7 @@ packageBin := { (packageBin in Rpm).value } // DEB // -version in Debian := version.value + "-1" +version in Debian := version.value + "-2" debianPackageRecommends := Seq("elasticsearch") debianPackageDependencies += "openjdk-8-jre-headless" maintainerScripts in Debian := maintainerScriptsFromDirectory(
    No tasks have been specified