Fix MISP modules integration

Enable MISP modules in docker package Send config to MISP modules Catch exception in MISP modules loader Update documentation
TheHive-Project · May 16, 2017 · 9f8f1eb · 9f8f1eb
1 parent 7d676b2
commit 9f8f1eb
Show file tree

Hide file tree

Showing 11 changed files with 278 additions and 88 deletions.
diff --git a/app/models/JsonFormat.scala b/app/models/JsonFormat.scala
@@ -18,7 +18,8 @@ object JsonFormat {
   implicit val fileArtifactWrites: OWrites[FileArtifact] = OWrites[FileArtifact](fileArtifact ⇒ Json.obj(
     "attributes" → fileArtifact.attributes))
 
-  implicit val dataArtifactWrites: OWrites[DataArtifact] = Json.writes[DataArtifact]
+  implicit val dataArtifactWrites: OWrites[DataArtifact] = OWrites[DataArtifact](artifact ⇒
+    artifact.attributes + ("data" → JsString(artifact.data)))
   implicit val dataActifactReads: Reads[DataArtifact] = Json.reads[DataArtifact]
 
   val artifactWrites: OWrites[Artifact] = OWrites[Artifact] {

diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala
@@ -1,13 +1,15 @@
 package models
 
+import play.api.libs.json.JsObject
+
 case class MispModule(
     name: String,
     version: String,
     description: String,
     author: String,
     dataTypeList: Seq[String],
     inputAttributes: Seq[String],
-    config: Seq[String],
+    config: JsObject,
     loaderCommand: String) extends Analyzer {
   val license = "AGPL-3.0"
   val url = "https://github.com/MISP/misp-modules"

diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala
@@ -1,12 +1,13 @@
 package services
 
 import java.io.{ ByteArrayInputStream, FileInputStream, InputStream, SequenceInputStream }
-import javax.inject.Inject
+import javax.inject.{ Inject, Singleton }
 
 import akka.actor.ActorSystem
 import models.JsonFormat._
 import models._
 import org.apache.commons.codec.binary.{ Base64, Base64InputStream }
+import util.JsonConfig
 import play.api.libs.json.{ Json, _ }
 import play.api.{ Configuration, Logger }
 
@@ -15,8 +16,11 @@ import scala.concurrent.{ ExecutionContext, Future }
 import scala.sys.process._
 import scala.util.{ Failure, Success, Try }
 
+@Singleton
 class MispSrv(
-    loaderCommandOption: Option[String],
+    mispModulesEnabled: Boolean,
+    loaderCommand: String,
+    mispModuleConfig: JsObject,
     externalAnalyzerSrv: ExternalAnalyzerSrv,
     jobSrv: JobSrv,
     akkaSystem: ActorSystem) {
@@ -26,33 +30,37 @@ class MispSrv(
     externalAnalyzerSrv: ExternalAnalyzerSrv,
     jobSrv: JobSrv,
     akkaSystem: ActorSystem) = this(
-    configuration.getString("misp.modules.loader"),
+    configuration.getBoolean("misp.modules.enabled").getOrElse(false),
+    configuration.getString("misp.modules.loader").get,
+    JsonConfig.configWrites.writes(configuration.getConfig("misp.modules.config").getOrElse(Configuration.empty)),
     externalAnalyzerSrv,
     jobSrv,
     akkaSystem)
 
   private[MispSrv] lazy val logger = Logger(getClass)
   private[MispSrv] lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer")
 
-  lazy val list: Seq[MispModule] =
-    loaderCommandOption.fold(Seq.empty[MispModule]) { loaderCommand ⇒
-      Json.parse(s"$loaderCommand --list".!!)
-        .as[Seq[String]]
-        .map { moduleName ⇒
-          moduleName → (for {
-            moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!))
-            module ← Try(moduleInfo.as[MispModule](reads(loaderCommand)))
-          } yield module)
-        }
-        .flatMap {
-          case (moduleName, Failure(error)) ⇒
-            logger.warn(s"Load MISP module $moduleName fails", error)
-            Nil
-          case (_, Success(module)) ⇒
-            logger.info(s"Register MISP module ${module.name} ${module.version}")
-            Seq(module)
-        }
-    }
+  logger.info(s"MISP modules is ${if (mispModulesEnabled) "enabled" else "disabled"}, loader is $loaderCommand")
+
+  lazy val list: Seq[MispModule] = if (mispModulesEnabled) {
+    Json.parse(s"$loaderCommand --list".!!)
+      .as[Seq[String]]
+      .map { moduleName ⇒
+        moduleName → (for {
+          moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!))
+          module ← Try(moduleInfo.as[MispModule](reads(loaderCommand, mispModuleConfig)))
+        } yield module)
+      }
+      .flatMap {
+        case (moduleName, Failure(error)) ⇒
+          logger.warn(s"Load MISP module $moduleName fails", error)
+          Nil
+        case (_, Success(module)) ⇒
+          logger.info(s"Register MISP module ${module.name} ${module.version}")
+          Seq(module)
+      }
+  }
+  else Nil
 
   def get(moduleName: String): Option[MispModule] = list.find(_.name == moduleName)
 
@@ -89,30 +97,31 @@ class MispSrv(
   }
 
   def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = {
-    loaderCommandOption
-      .flatMap { loaderCommand ⇒
-        val artifact = toArtifact(mispType, data)
-        get(module)
-          .map { mispModule ⇒
-            val mispReport = Future {
-              val input = Json.obj(mispType → data)
-              val output = (s"$loaderCommand --run $module" #< input.toString).!!
-              Json.parse(output).as[JsObject]
-            }
-            jobSrv.create(mispModule, artifact, mispReport.map(toReport))
-            mispReport
-
+    val artifact = toArtifact(mispType, data)
+    val mispModule = if (mispModulesEnabled) {
+      get(module)
+        .map { mispModule ⇒
+          val mispReport = Future {
+            val input = Json.obj("config" → mispModule.config, mispType → data)
+            val output = (s"$loaderCommand --run $module" #< input.toString).!!
+            Json.parse(output).as[JsObject]
           }
-          .orElse {
-            externalAnalyzerSrv
-              .get(module)
-              .map { analyzer ⇒
-                externalAnalyzerSrv.analyze(analyzer, artifact)
-                  .map { report ⇒ toMispOutput(report) }
-              }
+          jobSrv.create(mispModule, artifact, mispReport.map(toReport))
+          mispReport
+
+        }
+    }
+    else None
+    mispModule
+      .orElse {
+        externalAnalyzerSrv
+          .get(module)
+          .map { analyzer ⇒
+            externalAnalyzerSrv.analyze(analyzer, artifact)
+              .map { report ⇒ toMispOutput(report) }
           }
       }
-      .getOrElse(Future.failed(new Exception(s"Module $module not found")))
+      .getOrElse(Future.failed(new Exception(s"Module $module not found"))) // TODO add appropriate exception
   }
 
   def analyze(module: MispModule, artifact: Artifact): Future[Report] = {
@@ -121,10 +130,13 @@ class MispSrv(
 
     val input = artifact match {
       case DataArtifact(data, _) ⇒
-        stringStream(Json.obj(dataType2mispType(artifact.dataType).head → data).toString)
+        val mispType = dataType2mispType(artifact.dataType)
+          .filter(module.inputAttributes.contains)
+          .head
+        stringStream((Json.obj("config" → module.config) + (mispType → JsString(data))).toString)
       case FileArtifact(data, _) ⇒
         new SequenceInputStream(Iterator(
-          stringStream("""{"attachment":""""),
+          stringStream(Json.obj("config" → module.config).toString.replaceFirst("}$", ""","attachment":"""")),
           new Base64InputStream(new FileInputStream(data), true),
           stringStream("\"}")).asJavaEnumeration)
     }
@@ -207,15 +219,26 @@ class MispSrv(
     else mispTypes
   }
 
-  private def reads(loaderCommand: String): Reads[MispModule] =
+  private def reads(loaderCommand: String, mispModuleConfig: JsObject): Reads[MispModule] =
     for {
       name ← (__ \ "name").read[String]
-      version ← (__ \ "meta" \ "version").read[String]
-      description ← (__ \ "meta" \ "description").read[String]
-      author ← (__ \ "meta" \ "author").read[String]
-      config ← (__ \ "meta" \ "config").read[Seq[String]]
+      version ← (__ \ "moduleinfo" \ "version").read[String]
+      description ← (__ \ "moduleinfo" \ "description").read[String]
+      author ← (__ \ "moduleinfo" \ "author").read[String]
+      config = (mispModuleConfig \ name).asOpt[JsObject].getOrElse(JsObject(Nil))
+      requiredConfig ← (__ \ "config").read[Set[String]]
+      missingConfig = requiredConfig -- config.keys
+      _ ← if (missingConfig.nonEmpty) {
+        val message = s"MISP module $name is disabled because the following configuration " +
+          s"item${if (missingConfig.size > 1) "s are" else " is"} missing: ${missingConfig.mkString(", ")}"
+        logger.warn(message)
+        Reads[Unit](_ ⇒ JsError(message))
+      }
+      else {
+        Reads[Unit](_ ⇒ JsSuccess(()))
+      }
       input ← (__ \ "mispattributes" \ "input").read[Seq[String]]
-      dataTypes = input.map(mispType2dataType)
+      dataTypes = input.map(mispType2dataType).distinct
     } yield MispModule(name, version, description, author, dataTypes, input, config, loaderCommand)
 
   private val typeLookup = Map(

diff --git a/build.sbt b/build.sbt
@@ -43,14 +43,21 @@ mappings in Universal ~= {
     file("package/cortex.service") -> "package/cortex.service",
     file("package/cortex.conf") -> "package/cortex.conf",
     file("package/cortex") -> "package/cortex",
-    file("package/logback.xml") -> "conf/logback.xml"
+    file("package/logback.xml") -> "conf/logback.xml",
+    file("contrib/misp-modules-loader.py") -> "contrib/misp-modules-loader.py"
   )
 }
 
 // Package //
-maintainer := "Thomas Franco <[email protected]"
-packageSummary := "-"
-packageDescription := """--""".stripMargin
+maintainer := "TheHive Project <[email protected]>"
+packageSummary := "Powerful Observable Analysis Engine"
+packageDescription := """Cortex tries to solve a common problem frequently encountered by SOCs, CSIRTs and security
+  | researchers in the course of threat intelligence, digital forensics and incident response: how to analyze
+  | observables they have collected, at scale, by querying a single tool instead of several?
+  |
+  | Cortex, an open source and free software, has been created by TheHive Project for this very purpose. Observables,
+  | such as IP and email addresses, URLs, domain names, files or hashes, can be analyzed one by one or in bulk mode
+  | using a Web interface. Analysts can also automate these operations thanks to the Cortex REST API. """.stripMargin
 defaultLinuxInstallLocation := "/opt"
 linuxPackageMappings ~= { _.map { pm =>
   val mappings = pm.mappings.filterNot {
@@ -62,7 +69,7 @@ linuxPackageMappings ~= { _.map { pm =>
   file("package/cortex.conf") -> "/etc/init/cortex.conf",
   file("package/cortex") -> "/etc/init.d/cortex",
   file("conf/application.sample") -> "/etc/cortex/application.conf",
-  file("conf/logback.xml") -> "/etc/cortex/logback.xml"
+  file("package/logback.xml") -> "/etc/cortex/logback.xml"
 ).withConfig()
 }
 
@@ -125,7 +132,11 @@ dockerCommands ~= { dc =>
           "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git && " +
           "cd /opt && " +
           "git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " +
-          "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt)"),
+          "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt) && " +
+          "apt-get install -y --no-install-recommends python3-setuptools python3-dev zlib1g-dev libxslt1-dev libxml2-dev libpq5 libjpeg-dev && git clone https://github.com/MISP/misp-modules.git && " +
+          "easy_install3 pip && " +
+          "(cd misp-modules && pip3 install -I -r REQUIREMENTS && pip3 install -I .) && " +
+          "rm -rf misp_modules /var/lib/apt/lists/*"),
       Cmd("ADD", "var", "/var"),
       Cmd("ADD", "etc", "/etc"),
       ExecCmd("RUN", "chown", "-R", "daemon:daemon", "/var/log/cortex")) ++

diff --git a/conf/application.sample b/conf/application.sample
@@ -7,12 +7,6 @@
 analyzer {
   path = "path/to/Cortex-Analyzers/analyzers"
   config {
-    global {
-      proxy {
-        #http="http://PROXY_ADDRESS:PORT",
-        #https="http://PROXY_ADDRESS:PORT"
-      }
-    }
     CIRCLPassiveDNS {
       #user= "..."
       #password= "..."
@@ -79,4 +73,81 @@ analyzer {
     # Max number of threads available for analyze
     parallelism-max = 4
   }
-}
+}
+
+misp.modules {
+  enabled = true
+
+  config {
+    shodan {
+      #apikey = ""
+    }
+    eupi {
+      #apikey = ""
+      #url = ""
+    }
+    passivetotal {
+      #username = ""
+      #api_key = ""
+    }
+    dns {
+      #nameserver = ""
+    }
+    whois {
+      #server = ""
+      #port = ""
+    }
+    sourcecache {
+      #archivepath = ""
+    }
+    geoip_country {
+    }
+    circl_passivessl {
+      #username = ""
+      #password = ""
+    }
+    iprep {
+      #apikey = ""
+    }
+    countrycode {
+    }
+    cve {
+    }
+    virustotal {
+      #apikey = ""
+      #event_limit = ""
+    }
+    ipasn {
+      #host = ""
+      #port = ""
+      #db = ""
+    }
+    circl_passivedns {
+      #username = ""
+      #password = ""
+    }
+    vmray_submit {
+      #apikey = ""
+      #url = ""
+      #shareable = ""
+      #do_not_reanalyze = ""
+      #do_not_include_vmrayjobids = ""
+    }
+    wiki {
+    }
+    domaintools {
+      #username = ""
+      #api_key = ""
+    }
+    reversedns {
+      #nameserver = ""
+    }
+    threatminer {
+    }
+    asn_history {
+      #host = ""
+      #port = ""
+      #db = ""
+    }
+  }
+}
diff --git a/conf/reference.conf b/conf/reference.conf
@@ -1,14 +1,20 @@
 # handler for errors (transform exception to related http status code
 play.http.errorHandler = services.ErrorHandler
 
+# MISP modules loader location
+misp.modules {
+  enabled = false
+  loader = ${play.server.dir}/"contrib/misp-modules-loader.py"
+}
+
 analyzer {
   # Directory that holds analyzers
   path = analyzers
   # Analyzer configuration
   config {
     dummy = dummy
   }
-  
+
   fork-join-executor {
     # Min number of threads available for analyze
     parallelism-min = 2