#370 Add filter on MISP event during import

MISP event can be excluded according to the following filters:
 - the maximum number of attributes (max-attributes)
 - the maximum size of the event json message
 - the age of the last publication
 - the organisation is black-listed
 - one of the tags is black-listed

The filters are configurable in each connexion settings:
misp {
  "MISP-SERVER-ID" {
    url = "http://127.0.0.1"
    key = "MISP-KEY"

    # filters:
    max-attributes = 1000
    max-size = 1 MiB
    max-age = 7 days
    exclusion {
     organisation = ["bad organisation", "other orga"]
     tags = ["tag1", "tag2"]
    }
  }
}

thehive-misp/app/connectors/misp/MispConfig.scala

@@ -2,11 +2,12 @@ package connectors.misp
 
 import javax.inject.{ Inject, Singleton }
 
-import scala.concurrent.duration.{ DurationInt, FiniteDuration }
+import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration }
 import scala.util.Try
 
 import play.api.Configuration
 
+import com.typesafe.config.ConfigMemorySize
 import services.CustomWSAPI
 
 @Singleton
@@ -28,7 +29,12 @@ class MispConfig(val interval: FiniteDuration, val connections: Seq[MispConnecti
       instanceWS = mispWS.withConfig(mispConnectionConfig)
       artifactTags = mispConnectionConfig.getOptional[Seq[String]]("tags").getOrElse(defaultArtifactTags)
       caseTemplate = mispConnectionConfig.getOptional[String]("caseTemplate").orElse(defaultCaseTemplate)
-    } yield MispConnection(name, url, key, instanceWS, caseTemplate, artifactTags))
+      maxAge = mispConnectionConfig.getOptional[Duration]("max-age")
+      maxAttributes = mispConnectionConfig.getOptional[Int]("max-attributes")
+      maxSize = mispConnectionConfig.getOptional[ConfigMemorySize]("max-size").map(_.toBytes)
+      excludedOrganisations = mispConnectionConfig.getOptional[Seq[String]]("exclusion.organisation").getOrElse(Nil)
+      excludedTags = mispConnectionConfig.getOptional[Seq[String]]("exclusion.tags").fold(Set.empty[String])(_.toSet)
+    } yield MispConnection(name, url, key, instanceWS, caseTemplate, artifactTags, maxAge, maxAttributes, maxSize, excludedOrganisations, excludedTags))
 
   @Inject def this(configuration: Configuration, httpSrv: CustomWSAPI) =
     this(

thehive-misp/app/connectors/misp/MispConnection.scala

@@ -1,5 +1,7 @@
 package connectors.misp
 
+import java.util.Date
+
 import scala.concurrent.duration._
 import scala.concurrent.{ ExecutionContext, Future }
 
@@ -19,23 +21,60 @@ case class MispConnection(
     key: String,
     ws: CustomWSAPI,
     caseTemplate: Option[String],
-    artifactTags: Seq[String]) {
+    artifactTags: Seq[String],
+    maxAge: Option[Duration],
+    maxAttributes: Option[Int],
+    maxSize: Option[Long],
+    excludedOrganisations: Seq[String],
+    excludedTags: Set[String]) {
 
   private[MispConnection] lazy val logger = Logger(getClass)
 
   logger.info(
     s"""Add MISP connection $name
-       |\turl:           $baseUrl
-       |\tproxy:         ${ws.proxy}
-       |\tcase template: ${caseTemplate.getOrElse("<not set>")}
-       |\tartifact tags: ${artifactTags.mkString}""".stripMargin)
+       |  url:              $baseUrl
+       |  proxy:            ${ws.proxy}
+       |  case template:    ${caseTemplate.getOrElse("<not set>")}
+       |  artifact tags:    ${artifactTags.mkString}
+       |  filters:
+       |    max age:        ${maxAge.getOrElse("<not set>")}
+       |    max attributes: ${maxAttributes.getOrElse("<not set>")}
+       |    max size:       ${maxSize.getOrElse("<not set>")}
+       |    excluded orgs:  ${excludedOrganisations.mkString}
+       |    excluded tags:  ${excludedTags.mkString}
+       |""".stripMargin)
 
   private[misp] def apply(url: String): WSRequest =
     ws.url(s"$baseUrl/$url")
       .withHttpHeaders(
         "Authorization" → key,
         "Accept" → "application/json")
 
+  def syncFrom(date: Date): Date = {
+    maxAge.fold(date) { age ⇒
+      val now = new Date
+      val dateThreshold = new Date(now.getTime - age.toMillis)
+
+      if (date after dateThreshold) date
+      else dateThreshold
+    }
+  }
+
+  def isExcluded(event: MispAlert): Boolean = {
+    if (excludedOrganisations.contains(event.source)) {
+      logger.debug(s"event ${event.sourceRef} is ignored because its organisation (${event.source}) is excluded")
+      true
+    }
+    else {
+      val t = excludedTags.intersect(event.tags.toSet)
+      if (t.nonEmpty) {
+        logger.debug(s"event ${event.sourceRef} is ignored because one of its tags (${t.mkString(",")}) is excluded")
+        true
+      }
+      else false
+    }
+  }
+
   def getVersion()(implicit system: ActorSystem, ec: ExecutionContext): Future[Option[String]] = {
     apply("servers/getVersion").get
       .map {

thehive-misp/app/connectors/misp/MispSrv.scala

@@ -46,6 +46,7 @@ class MispSrv @Inject() (
     }
 
   def getEventsFromDate(mispConnection: MispConnection, fromDate: Date): Source[MispAlert, NotUsed] = {
+    logger.debug(s"Get MISP events from $fromDate")
     val date = fromDate.getTime / 1000
     Source
       .fromFuture {
@@ -62,11 +63,12 @@ class MispSrv @Inject() (
         val events = eventJson
           .flatMap { j ⇒
             j.asOpt[MispAlert]
-              .map(_.copy(source = mispConnection.name))
               .orElse {
                 logger.warn(s"MISP event can't be parsed\n$j")
                 None
               }
+              .filterNot(mispConnection.isExcluded)
+              .map(_.copy(source = mispConnection.name))
           }
 
         val eventJsonSize = eventJson.size
@@ -110,10 +112,17 @@ class MispSrv @Inject() (
           "eventid" → eventId)))
       // add ("deleted" → 1) to see also deleted attributes
       // add ("deleted" → "only") to see only deleted attributes
-      .map { response ⇒
+      .map(_.body)
+      .map {
+        case body if mispConnection.maxSize.fold(false)(body.length > _) ⇒
+          logger.debug(s"Size of event exceeds (${body.length}) the configured limit")
+          JsObject.empty
+        case body ⇒ Json.parse(body)
+      }
+      .map { jsBody ⇒
         val refDate = fromDate.getOrElse(new Date(0))
         val artifactTags = s"src:${mispConnection.name}" +: mispConnection.artifactTags
-        (Json.parse(response.body) \ "response" \\ "Attribute")
+        (jsBody \ "response" \\ "Attribute")
           .flatMap(_.as[Seq[MispAttribute]])
           .filter(_.date after refDate)
           .flatMap(convertAttribute)

thehive-misp/app/connectors/misp/MispSynchro.scala

@@ -124,7 +124,7 @@ class MispSynchro @Inject() (
   def synchronize(mispConnection: MispConnection, lastSyncDate: Option[Date])(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = {
     logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate")
     // get events that have been published after the last synchronization
-    mispSrv.getEventsFromDate(mispConnection, lastSyncDate.getOrElse(new Date(0)))
+    mispSrv.getEventsFromDate(mispConnection, mispConnection.syncFrom(lastSyncDate.getOrElse(new Date(0))))
       // get related alert
       .mapAsyncUnordered(1) { event ⇒
         logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}")
@@ -134,14 +134,22 @@ class MispSynchro @Inject() (
       .mapAsyncUnordered(1) {
         case (event, alert) ⇒
           logger.trace(s"MISP synchro ${mispConnection.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}")
-          logger.info(s"getting MISP event ${event.source}:${event.sourceRef}")
+          logger.debug(s"getting MISP event ${event.source}:${event.sourceRef}")
           mispSrv.getAttributesFromMisp(mispConnection, event.sourceRef, lastSyncDate.flatMap(_ ⇒ alert.map(_.lastSyncDate())))
             .map((event, alert, _))
       }
+      .filter {
+        // attrs is empty if the size of the http response exceed the configured limit (max-size)
+        case (_, _, attrs) if attrs.isEmpty ⇒ false
+        case (event, _, attrs) if mispConnection.maxAttributes.fold(false)(attrs.lengthCompare(_) > 0) ⇒
+          logger.debug(s"Event ${event.sourceRef} ignore because it has too many attributes (${attrs.length}>${mispConnection.maxAttributes.get})")
+          false
+        case _ ⇒ true
+      }
       .mapAsyncUnordered(1) {
         // if there is no related alert, create a new one
         case (event, None, attrs) ⇒
-          logger.info(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)")
+          logger.debug(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)")
           val alertJson = Json.toJson(event).as[JsObject] +
             ("type" → JsString("misp")) +
             ("caseTemplate" → mispConnection.caseTemplate.fold[JsValue](JsNull)(JsString)) +
@@ -151,7 +159,7 @@ class MispSynchro @Inject() (
             .recover { case t ⇒ Failure(t) }
 
         case (event, Some(alert), attrs) ⇒
-          logger.info(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)")
+          logger.debug(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)")
 
           alert.caze().fold[Future[Boolean]](Future.successful(lastSyncDate.isDefined && attrs.nonEmpty && alert.follow())) {
             case caze if alert.follow() ⇒

thehive-misp/conf/reference.conf

@@ -8,8 +8,15 @@ misp {
   #  url = ""
   #  # authentication key
   #  key = ""
-  #  #tags to be added to imported artifact
+  #  #tags to be added to imported artifacts
   #  tags = ["misp"]
+  #  max-attributes = 1000
+  #  max-size = 1 MiB
+  #  max-age = 7 days
+  #  exclusion {
+  #   organisation = ["", ""]
+  #   tags = ["", ""]
+  #  }
   #}
 
   # truststore to used to validate MISP certificate (if default truststore is not suffisient)