#21 Add misp integration

app/controllers/MispCtrl.scala

@@ -0,0 +1,29 @@
+package controllers
+
+import javax.inject.Inject
+
+import play.api.libs.json.{ JsObject, JsValue }
+import play.api.mvc.{ Action, AnyContent, Controller }
+import services.MispSrv
+
+import scala.concurrent.ExecutionContext
+
+class MispCtrl @Inject() (mispSrv: MispSrv, implicit val ec: ExecutionContext) extends Controller {
+
+  def modules: Action[AnyContent] = Action { _ ⇒
+    Ok(mispSrv.moduleList)
+  }
+
+  def query: Action[JsValue] = Action.async(parse.json) { request ⇒
+    val module = (request.body \ "module").asOpt[String].getOrElse(sys.error("module not present in request"))
+    val (mispType, dataJson) = request.body.as[JsObject].fields
+      .collectFirst {
+        case kv @ (k, _) if k != "module" ⇒ kv
+      }
+      .getOrElse(sys.error("invalid request"))
+    val data = dataJson.asOpt[String].getOrElse(sys.error("data has invalid type (expected string)"))
+    mispSrv.query(module, mispType, data)
+      .map(Ok(_))
+  }
+}
+

app/models/Artifact.scala

@@ -1,23 +1,31 @@
 package models
 
-import play.api.libs.json.JsObject
 import java.io.File
+import java.nio.file.Files
+
+import play.api.libs.json.JsObject
 
 abstract class Artifact(attributes: JsObject) {
-  def dataTypeFilter(filter: String): Boolean = (attributes \ "dataType").asOpt[String].fold(false)(_.toLowerCase.contains(filter.toLowerCase))
+  def dataType: String = (attributes \ "dataType").asOpt[String].getOrElse("other")
+  def dataTypeFilter(filter: String): Boolean = dataType.toLowerCase.contains(filter.toLowerCase)
   def dataFilter(filter: String): Boolean = false
 }
 class FileArtifact(val data: File, val attributes: JsObject) extends Artifact(attributes) {
-  override def finalize {
+  override def finalize() {
     data.delete()
   }
 }
 object FileArtifact {
-  def apply(data: File, attributes: JsObject) = {
+  def apply(data: File, attributes: JsObject): FileArtifact = {
     val tempFile = File.createTempFile("cortex-", "-datafile")
     data.renameTo(tempFile)
     new FileArtifact(tempFile, attributes)
   }
+  def apply(data: Array[Byte], attributes: JsObject): FileArtifact = {
+    val tempFile = File.createTempFile("cortex-", "-datafile")
+    Files.write(tempFile.toPath, data)
+    new FileArtifact(tempFile, attributes)
+  }
   def unapply(fileArtifact: FileArtifact) = Some(fileArtifact.data → fileArtifact.attributes)
 }
 case class DataArtifact(data: String, attributes: JsObject) extends Artifact(attributes) {

app/models/MispModule.scala

@@ -0,0 +1,73 @@
+package models
+
+import java.io._
+import java.nio.file.Path
+import java.nio.file.Files
+
+import org.apache.commons.codec.binary.{ Base64, Base64InputStream }
+import play.api.libs.json.{ JsObject, JsValue, Json }
+import services.MispSrv
+
+import scala.collection.JavaConverters._
+import scala.concurrent.{ ExecutionContext, Future }
+import scala.sys.process.{ BasicIO, Process, ProcessIO }
+import scala.sys.process._
+
+case class MispModule(
+  name: String,
+    version: String,
+    description: String,
+    dataTypeList: Seq[String],
+    author: String,
+    modulePath: Path,
+    loaderCommand: String)(implicit val ec: ExecutionContext) extends Analyzer {
+
+  val license = "AGPL-3.0"
+  val url = "https://github.com/MISP/misp-modules"
+
+  private def stringStream(string: String): InputStream = {
+    new ByteArrayInputStream(string.getBytes)
+  }
+  def analyze(artifact: Artifact): Future[JsObject] = {
+    val input = artifact match {
+      case DataArtifact(data, _) ⇒
+        stringStream(Json.obj(artifact.dataType → data).toString)
+      case FileArtifact(data, _) ⇒
+        new SequenceInputStream(Iterator(
+          stringStream("""{"file":""""),
+          new Base64InputStream(new FileInputStream(data), true),
+          stringStream("\"}")).asJavaEnumeration)
+    }
+    val output = (s"$loaderCommand --run $modulePath" #< input).!!
+    Future {
+      Json.parse(output).as[JsObject]
+    }
+  }
+}
+
+object MispModule {
+  def apply(
+    loaderCommand: String,
+    modulePath: Path,
+    mispSrv: MispSrv)(implicit ec: ExecutionContext): Option[MispModule] = {
+    val moduleInfo = Json.parse(s"$loaderCommand --info $modulePath".!!)
+    for {
+      name ← (moduleInfo \ "name").asOpt[String]
+      version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String]
+      description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String]
+      dataTypeList ← (moduleInfo \ "mispattributes" \ "input")
+        .asOpt[Seq[String]]
+        .map(_.map(mispSrv.mispType2dataType(_)))
+      author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String]
+    } yield MispModule(name, version, description, dataTypeList, author, modulePath, loaderCommand)
+  }
+}
+
+//{'mispattributes': {
+//   'input': ['ip-src', 'ip-dst', 'domain|ip'],
+//   'output': ['hostname']
+// },
+// 'moduleinfo': {
+//  'version': '0.1',
+// 'author': 'Andreas Muehlemann',
+// 'description': 'Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes', 'module-type': ['expansion', 'hover']}}

app/services/Analyzer.scala

@@ -2,37 +2,41 @@ package services
 
 import java.io.File
 import java.nio.file.{ Files, Path, Paths }
-
 import javax.inject.Inject
 
-import scala.collection.JavaConversions.iterableAsScalaIterable
-import scala.concurrent.ExecutionContext
-
 import akka.actor.ActorSystem
-
-import play.api.{ Configuration, Logger }
+import models.{ Analyzer, ExternalAnalyzer, MispModule }
 import play.api.libs.json.{ JsObject, JsValue, Json }
-
-import models.{ Analyzer, ExternalAnalyzer }
+import play.api.{ Configuration, Logger }
 import util.JsonConfig.configWrites
+
+import scala.collection.JavaConversions.iterableAsScalaIterable
+import scala.concurrent.ExecutionContext
 import scala.util.Try
 
 class AnalyzerSrv(
+    mispSrv: MispSrv,
     analyzerPath: Path,
     analyzerConfig: JsObject,
+    mispModulesPath: Path,
+    mispModuleLoaderCommand: Option[String],
     akkaSystem: ActorSystem) {
   @Inject def this(
+    mispSrv: MispSrv,
     configuration: Configuration,
     akkaSystem: ActorSystem) =
     this(
+      mispSrv,
       Paths.get(configuration.getString("analyzer.path").getOrElse(".")),
       configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)),
+      Paths.get(configuration.getString("misp.modules.path").getOrElse(".")),
+      configuration.getString("misp.modules.loader"),
       akkaSystem)
 
   lazy val log = Logger(getClass)
   lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer")
 
-  lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers
+  private lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers
   def list: Seq[Analyzer] = externalAnalyzers // ::: javaAnalyzers
   def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId)
   def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType))
@@ -79,6 +83,18 @@ class AnalyzerSrv(
     } yield ExternalAnalyzer(name, version, description, dataTypeList, author, url, license, absoluteCommand, globalConfig deepMerge baseConfig deepMerge config)(analyzeExecutionContext)
   }
 
+  private[services] def getMispModules: Seq[Analyzer] = {
+    for {
+      moduleFile ← Try(Files.newDirectoryStream(mispModulesPath).toSeq).getOrElse {
+        log.warn(s"MISP modules directory ($mispModulesPath) is not found")
+        Nil
+      }
+      loaderCommand ← mispModuleLoaderCommand
+      if Files.isRegularFile(moduleFile) && moduleFile.toString.endsWith(".py")
+      mispModule ← MispModule(loaderCommand, moduleFile, mispSrv)(analyzeExecutionContext)
+    } yield mispModule
+  }
+
   private[services] def readInfo(file: Path): JsValue = {
     val source = scala.io.Source.fromFile(file.toFile)
     try {

app/services/Job.scala

@@ -1,31 +1,26 @@
 package services
 
 import java.util.Date
-
 import javax.inject.{ Inject, Named }
 
-import scala.annotation.implicitNotFound
-import scala.concurrent.{ ExecutionContext, Future, Promise }
-import scala.concurrent.duration.{ DurationInt, FiniteDuration }
-import scala.concurrent.duration.Duration
-import scala.concurrent.duration.Duration.Infinite
-import scala.util.Random
-
 import akka.actor.{ Actor, ActorRef, ActorSystem, actorRef2Scala }
 import akka.pattern.ask
 import akka.util.Timeout
-
-import play.api.{ Configuration, Logger }
+import models.{ Analyzer, Artifact, Job, JobStatus }
 import play.api.libs.json.{ JsString, JsValue }
+import play.api.{ Configuration, Logger }
 
-import models.{ Analyzer, Artifact, Job, JobStatus }
+import scala.concurrent.duration.Duration.Infinite
+import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration }
+import scala.concurrent.{ ExecutionContext, Future, Promise }
+import scala.util.Random
 
 class JobSrv @Inject() (
     analyzerSrv: AnalyzerSrv,
     @Named("JobActor") jobActor: ActorRef,
     implicit val ec: ExecutionContext,
     implicit val system: ActorSystem) {
-  import JobActor._
+  import services.JobActor._
   implicit val timeout = Timeout(5.seconds)
 
   def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Future[(Int, Seq[Job])] = {
@@ -71,8 +66,8 @@ class JobSrv @Inject() (
       case _: Infinite ⇒ statusResult
       case duration: FiniteDuration ⇒
         val prom = Promise[(JobStatus.Type, JsValue)]()
-        val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.Failure, JsString("Timeout"))); () }
-        statusResult onComplete { case _ ⇒ timeout.cancel() }
+        val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.InProgress, JsString("Timeout"))); () }
+        statusResult.onComplete(_ ⇒ timeout.cancel())
         Future.firstCompletedOf(List(statusResult, prom.future))
     }
   }
@@ -95,7 +90,7 @@ class JobActor(
     analyzerSrv: AnalyzerSrv,
     implicit val ec: ExecutionContext) extends Actor {
 
-  import JobActor._
+  import services.JobActor._
   @Inject def this(
     configuration: Configuration,
     analyzerSrv: AnalyzerSrv,
@@ -128,7 +123,7 @@ class JobActor(
         dataTypeFilter.fold(true)(j.artifact.dataTypeFilter) &&
           dataFilter.fold(true)(j.artifact.dataFilter) &&
           analyzerFilter.fold(true)(j.analyzerId.contains))
-      sender ! JobList(filteredJobs.size, filteredJobs.drop(start).take(limit))
+      sender ! JobList(filteredJobs.size, filteredJobs.slice(start, start + limit))
     case GetJob(jobId) ⇒ sender ! jobs.find(_.id == jobId).getOrElse(JobNotFound)
     case RemoveJob(jobId) ⇒
       removeJob(jobs, jobId) match {
@@ -148,5 +143,5 @@ class JobActor(
       context.become(jobState(jobs.takeWhile(_.date after limitDate)))
   }
 
-  override def receive = jobState(Nil)
+  override def receive: Receive = jobState(Nil)
 }