From c32353434b2d99e187e4ed69a917c2026fc693cf Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 3 May 2017 12:16:30 +0200 Subject: [PATCH] #21 Add misp integration --- app/controllers/MispCtrl.scala | 29 +++++ app/models/Artifact.scala | 16 ++- app/models/MispModule.scala | 73 +++++++++++ app/services/Analyzer.scala | 34 +++-- app/services/Job.scala | 29 ++--- app/services/MispSrv.scala | 228 +++++++++++++++++++++++++++++++++ conf/routes | 4 + 7 files changed, 383 insertions(+), 30 deletions(-) create mode 100644 app/controllers/MispCtrl.scala create mode 100644 app/models/MispModule.scala create mode 100644 app/services/MispSrv.scala diff --git a/app/controllers/MispCtrl.scala b/app/controllers/MispCtrl.scala new file mode 100644 index 000000000..312708b65 --- /dev/null +++ b/app/controllers/MispCtrl.scala @@ -0,0 +1,29 @@ +package controllers + +import javax.inject.Inject + +import play.api.libs.json.{ JsObject, JsValue } +import play.api.mvc.{ Action, AnyContent, Controller } +import services.MispSrv + +import scala.concurrent.ExecutionContext + +class MispCtrl @Inject() (mispSrv: MispSrv, implicit val ec: ExecutionContext) extends Controller { + + def modules: Action[AnyContent] = Action { _ ⇒ + Ok(mispSrv.moduleList) + } + + def query: Action[JsValue] = Action.async(parse.json) { request ⇒ + val module = (request.body \ "module").asOpt[String].getOrElse(sys.error("module not present in request")) + val (mispType, dataJson) = request.body.as[JsObject].fields + .collectFirst { + case kv @ (k, _) if k != "module" ⇒ kv + } + .getOrElse(sys.error("invalid request")) + val data = dataJson.asOpt[String].getOrElse(sys.error("data has invalid type (expected string)")) + mispSrv.query(module, mispType, data) + .map(Ok(_)) + } +} + diff --git a/app/models/Artifact.scala b/app/models/Artifact.scala index c71a63439..b4a8f033e 100644 --- a/app/models/Artifact.scala +++ b/app/models/Artifact.scala @@ -1,23 +1,31 @@ package models -import play.api.libs.json.JsObject import java.io.File +import java.nio.file.Files + +import play.api.libs.json.JsObject abstract class Artifact(attributes: JsObject) { - def dataTypeFilter(filter: String): Boolean = (attributes \ "dataType").asOpt[String].fold(false)(_.toLowerCase.contains(filter.toLowerCase)) + def dataType: String = (attributes \ "dataType").asOpt[String].getOrElse("other") + def dataTypeFilter(filter: String): Boolean = dataType.toLowerCase.contains(filter.toLowerCase) def dataFilter(filter: String): Boolean = false } class FileArtifact(val data: File, val attributes: JsObject) extends Artifact(attributes) { - override def finalize { + override def finalize() { data.delete() } } object FileArtifact { - def apply(data: File, attributes: JsObject) = { + def apply(data: File, attributes: JsObject): FileArtifact = { val tempFile = File.createTempFile("cortex-", "-datafile") data.renameTo(tempFile) new FileArtifact(tempFile, attributes) } + def apply(data: Array[Byte], attributes: JsObject): FileArtifact = { + val tempFile = File.createTempFile("cortex-", "-datafile") + Files.write(tempFile.toPath, data) + new FileArtifact(tempFile, attributes) + } def unapply(fileArtifact: FileArtifact) = Some(fileArtifact.data → fileArtifact.attributes) } case class DataArtifact(data: String, attributes: JsObject) extends Artifact(attributes) { diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala new file mode 100644 index 000000000..4e01ec929 --- /dev/null +++ b/app/models/MispModule.scala @@ -0,0 +1,73 @@ +package models + +import java.io._ +import java.nio.file.Path +import java.nio.file.Files + +import org.apache.commons.codec.binary.{ Base64, Base64InputStream } +import play.api.libs.json.{ JsObject, JsValue, Json } +import services.MispSrv + +import scala.collection.JavaConverters._ +import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process.{ BasicIO, Process, ProcessIO } +import scala.sys.process._ + +case class MispModule( + name: String, + version: String, + description: String, + dataTypeList: Seq[String], + author: String, + modulePath: Path, + loaderCommand: String)(implicit val ec: ExecutionContext) extends Analyzer { + + val license = "AGPL-3.0" + val url = "https://github.com/MISP/misp-modules" + + private def stringStream(string: String): InputStream = { + new ByteArrayInputStream(string.getBytes) + } + def analyze(artifact: Artifact): Future[JsObject] = { + val input = artifact match { + case DataArtifact(data, _) ⇒ + stringStream(Json.obj(artifact.dataType → data).toString) + case FileArtifact(data, _) ⇒ + new SequenceInputStream(Iterator( + stringStream("""{"file":""""), + new Base64InputStream(new FileInputStream(data), true), + stringStream("\"}")).asJavaEnumeration) + } + val output = (s"$loaderCommand --run $modulePath" #< input).!! + Future { + Json.parse(output).as[JsObject] + } + } +} + +object MispModule { + def apply( + loaderCommand: String, + modulePath: Path, + mispSrv: MispSrv)(implicit ec: ExecutionContext): Option[MispModule] = { + val moduleInfo = Json.parse(s"$loaderCommand --info $modulePath".!!) + for { + name ← (moduleInfo \ "name").asOpt[String] + version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String] + description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String] + dataTypeList ← (moduleInfo \ "mispattributes" \ "input") + .asOpt[Seq[String]] + .map(_.map(mispSrv.mispType2dataType(_))) + author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String] + } yield MispModule(name, version, description, dataTypeList, author, modulePath, loaderCommand) + } +} + +//{'mispattributes': { +// 'input': ['ip-src', 'ip-dst', 'domain|ip'], +// 'output': ['hostname'] +// }, +// 'moduleinfo': { +// 'version': '0.1', +// 'author': 'Andreas Muehlemann', +// 'description': 'Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes', 'module-type': ['expansion', 'hover']}} \ No newline at end of file diff --git a/app/services/Analyzer.scala b/app/services/Analyzer.scala index c64eef8b2..d063747d9 100644 --- a/app/services/Analyzer.scala +++ b/app/services/Analyzer.scala @@ -2,37 +2,41 @@ package services import java.io.File import java.nio.file.{ Files, Path, Paths } - import javax.inject.Inject -import scala.collection.JavaConversions.iterableAsScalaIterable -import scala.concurrent.ExecutionContext - import akka.actor.ActorSystem - -import play.api.{ Configuration, Logger } +import models.{ Analyzer, ExternalAnalyzer, MispModule } import play.api.libs.json.{ JsObject, JsValue, Json } - -import models.{ Analyzer, ExternalAnalyzer } +import play.api.{ Configuration, Logger } import util.JsonConfig.configWrites + +import scala.collection.JavaConversions.iterableAsScalaIterable +import scala.concurrent.ExecutionContext import scala.util.Try class AnalyzerSrv( + mispSrv: MispSrv, analyzerPath: Path, analyzerConfig: JsObject, + mispModulesPath: Path, + mispModuleLoaderCommand: Option[String], akkaSystem: ActorSystem) { @Inject def this( + mispSrv: MispSrv, configuration: Configuration, akkaSystem: ActorSystem) = this( + mispSrv, Paths.get(configuration.getString("analyzer.path").getOrElse(".")), configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), + Paths.get(configuration.getString("misp.modules.path").getOrElse(".")), + configuration.getString("misp.modules.loader"), akkaSystem) lazy val log = Logger(getClass) lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") - lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers + private lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers def list: Seq[Analyzer] = externalAnalyzers // ::: javaAnalyzers def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) @@ -79,6 +83,18 @@ class AnalyzerSrv( } yield ExternalAnalyzer(name, version, description, dataTypeList, author, url, license, absoluteCommand, globalConfig deepMerge baseConfig deepMerge config)(analyzeExecutionContext) } + private[services] def getMispModules: Seq[Analyzer] = { + for { + moduleFile ← Try(Files.newDirectoryStream(mispModulesPath).toSeq).getOrElse { + log.warn(s"MISP modules directory ($mispModulesPath) is not found") + Nil + } + loaderCommand ← mispModuleLoaderCommand + if Files.isRegularFile(moduleFile) && moduleFile.toString.endsWith(".py") + mispModule ← MispModule(loaderCommand, moduleFile, mispSrv)(analyzeExecutionContext) + } yield mispModule + } + private[services] def readInfo(file: Path): JsValue = { val source = scala.io.Source.fromFile(file.toFile) try { diff --git a/app/services/Job.scala b/app/services/Job.scala index 1e8742da9..c66a7b455 100644 --- a/app/services/Job.scala +++ b/app/services/Job.scala @@ -1,31 +1,26 @@ package services import java.util.Date - import javax.inject.{ Inject, Named } -import scala.annotation.implicitNotFound -import scala.concurrent.{ ExecutionContext, Future, Promise } -import scala.concurrent.duration.{ DurationInt, FiniteDuration } -import scala.concurrent.duration.Duration -import scala.concurrent.duration.Duration.Infinite -import scala.util.Random - import akka.actor.{ Actor, ActorRef, ActorSystem, actorRef2Scala } import akka.pattern.ask import akka.util.Timeout - -import play.api.{ Configuration, Logger } +import models.{ Analyzer, Artifact, Job, JobStatus } import play.api.libs.json.{ JsString, JsValue } +import play.api.{ Configuration, Logger } -import models.{ Analyzer, Artifact, Job, JobStatus } +import scala.concurrent.duration.Duration.Infinite +import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration } +import scala.concurrent.{ ExecutionContext, Future, Promise } +import scala.util.Random class JobSrv @Inject() ( analyzerSrv: AnalyzerSrv, @Named("JobActor") jobActor: ActorRef, implicit val ec: ExecutionContext, implicit val system: ActorSystem) { - import JobActor._ + import services.JobActor._ implicit val timeout = Timeout(5.seconds) def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Future[(Int, Seq[Job])] = { @@ -71,8 +66,8 @@ class JobSrv @Inject() ( case _: Infinite ⇒ statusResult case duration: FiniteDuration ⇒ val prom = Promise[(JobStatus.Type, JsValue)]() - val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.Failure, JsString("Timeout"))); () } - statusResult onComplete { case _ ⇒ timeout.cancel() } + val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.InProgress, JsString("Timeout"))); () } + statusResult.onComplete(_ ⇒ timeout.cancel()) Future.firstCompletedOf(List(statusResult, prom.future)) } } @@ -95,7 +90,7 @@ class JobActor( analyzerSrv: AnalyzerSrv, implicit val ec: ExecutionContext) extends Actor { - import JobActor._ + import services.JobActor._ @Inject def this( configuration: Configuration, analyzerSrv: AnalyzerSrv, @@ -128,7 +123,7 @@ class JobActor( dataTypeFilter.fold(true)(j.artifact.dataTypeFilter) && dataFilter.fold(true)(j.artifact.dataFilter) && analyzerFilter.fold(true)(j.analyzerId.contains)) - sender ! JobList(filteredJobs.size, filteredJobs.drop(start).take(limit)) + sender ! JobList(filteredJobs.size, filteredJobs.slice(start, start + limit)) case GetJob(jobId) ⇒ sender ! jobs.find(_.id == jobId).getOrElse(JobNotFound) case RemoveJob(jobId) ⇒ removeJob(jobs, jobId) match { @@ -148,5 +143,5 @@ class JobActor( context.become(jobState(jobs.takeWhile(_.date after limitDate))) } - override def receive = jobState(Nil) + override def receive: Receive = jobState(Nil) } \ No newline at end of file diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala new file mode 100644 index 000000000..869c8c13a --- /dev/null +++ b/app/services/MispSrv.scala @@ -0,0 +1,228 @@ +package services + +import javax.inject.Inject + +import models.JsonFormat.dataActifactReads +import models.{ DataArtifact, FileArtifact } +import org.apache.commons.codec.binary.Base64 +import play.api.Logger +import play.api.libs.json.{ JsArray, JsObject, JsValue, Json } + +import scala.concurrent.{ ExecutionContext, Future } + +class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { + private[MispSrv] lazy val logger = Logger(getClass) + + def moduleList: JsValue = { + JsArray(analyzerSrv.list.map { analyzer ⇒ + Json.obj( + "name" → analyzer.id, + "type" → "expansion", + "mispattributes" → Json.obj( + "input" → analyzer.dataTypeList.flatMap(dataType2mispType).distinct, + "output" → Json.arr()), + "meta" → Json.obj( + "module-type" → Json.arr("expansion"), + "description" → analyzer.description, + "author" → analyzer.author, + "version" → analyzer.version, + "config" → Json.arr())) + }) + } + + def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = { + analyzerSrv.get(module).map { analyzer ⇒ + val artifact = mispType2dataType(mispType) match { + case "file" if mispType == "malware-sample" ⇒ ??? + case "file" ⇒ FileArtifact(Base64.decodeBase64(data), Json.obj( + "tlp" → 1, + "dataType" → "file")) + case dataType ⇒ DataArtifact(data, Json.obj( + "tlp" → 1, + "dataType" → dataType)) + } + + analyzer + .analyze(artifact) + .map { output ⇒ + logger.info(s"analyzer output:\n$output") + val success = (output \ "success") + .asOpt[Boolean] + .getOrElse(false) + if (success) { + Json.obj( + "results" → (output \ "artifacts") + .asOpt[Seq[JsObject]] + .getOrElse(Nil) + .map { artifact ⇒ + Json.obj( + "types" → dataType2mispType((artifact \ "type").as[String]), + "values" → Json.arr((artifact \ "value").as[String])) + }) + } + else { + val message = (output \ "error").asOpt[String].getOrElse(output.toString) + Json.obj( + "error" → message) + } + } + } + .getOrElse(Future.failed(new Exception(s"Module $module not found"))) + } + + def mispType2dataType(mispType: String): String = typeLookup.getOrElse(mispType, { + logger.warn(s"Misp type $mispType not recognized") + "other" + }) + + def dataType2mispType(dataType: String): Seq[String] = { + val mispTypes = typeLookup.filter(_._2 == dataType) + .keys + .toSeq + .distinct + + if (mispTypes.isEmpty) { + logger.warn(s"Data type $dataType not recognized") + Seq("other") + } + else mispTypes + } + + private val typeLookup = Map( + "md5" → "hash", + "sha1" → "hash", + "sha256" → "hash", + "filename" → "filename", + "pdb" → "other", + "filename|md5" → "other", + "filename|sha1" → "other", + "filename|sha256" → "other", + "ip-src" → "ip", + "ip-dst" → "ip", + "hostname" → "fqdn", + "domain" → "domain", + "domain|ip" → "other", + "email-src" → "mail", + "email-dst" → "mail", + "email-subject" → "mail_subject", + "email-attachment" → "other", + "float" → "other", + "url" → "url", + "http-method" → "other", + "user-agent" → "user-agent", + "regkey" → "registry", + "regkey|value" → "registry", + "AS" → "other", + "snort" → "other", + "pattern-in-file" → "other", + "pattern-in-traffic" → "other", + "pattern-in-memory" → "other", + "yara" → "other", + "sigma" → "other", + "vulnerability" → "other", + "attachment" → "file", + "malware-sample" → "file", + "link" → "other", + "comment" → "other", + "text" → "other", + "hex" → "other", + "other" → "other", + "named" → "other", + "mutex" → "other", + "target-user" → "other", + "target-email" → "mail", + "target-machine" → "hostname", + "target-org" → "other", + "target-location" → "other", + "target-external" → "other", + "btc" → "other", + "iban" → "other", + "bic" → "other", + "bank-account-nr" → "other", + "aba-rtn" → "other", + "bin" → "other", + "cc-number" → "other", + "prtn" → "other", + "threat-actor" → "other", + "campaign-name" → "other", + "campaign-id" → "other", + "malware-type" → "other", + "uri" → "uri_path", + "authentihash" → "other", + "ssdeep" → "hash", + "imphash" → "hash", + "pehash" → "hash", + "impfuzzy" → "hash", + "sha224" → "hash", + "sha384" → "hash", + "sha512" → "hash", + "sha512/224" → "hash", + "sha512/256" → "hash", + "tlsh" → "other", + "filename|authentihash" → "other", + "filename|ssdeep" → "other", + "filename|imphash" → "other", + "filename|impfuzzy" → "other", + "filename|pehash" → "other", + "filename|sha224" → "other", + "filename|sha384" → "other", + "filename|sha512" → "other", + "filename|sha512/224" → "other", + "filename|sha512/256" → "other", + "filename|tlsh" → "other", + "windows-scheduled-task" → "other", + "windows-service-name" → "other", + "windows-service-displayname" → "other", + "whois-registrant-email" → "mail", + "whois-registrant-phone" → "other", + "whois-registrant-name" → "other", + "whois-registrar" → "other", + "whois-creation-date" → "other", + "x509-fingerprint-sha1" → "other", + "dns-soa-email" → "other", + "size-in-bytes" → "other", + "counter" → "other", + "datetime" → "other", + "cpe" → "other", + "port" → "other", + "ip-dst|port" → "other", + "ip-src|port" → "other", + "hostname|port" → "other", + "email-dst-display-name" → "other", + "email-src-display-name" → "other", + "email-header" → "other", + "email-reply-to" → "other", + "email-x-mailer" → "other", + "email-mime-boundary" → "other", + "email-thread-index" → "other", + "email-message-id" → "other", + "github-username" → "other", + "github-repository" → "other", + "github-organisation" → "other", + "jabber-id" → "other", + "twitter-id" → "other", + "first-name" → "other", + "middle-name" → "other", + "last-name" → "other", + "date-of-birth" → "other", + "place-of-birth" → "other", + "gender" → "other", + "passport-number" → "other", + "passport-country" → "other", + "passport-expiration" → "other", + "redress-number" → "other", + "nationality" → "other", + "visa-number" → "other", + "issue-date-of-the-visa" → "other", + "primary-residence" → "other", + "country-of-residence" → "other", + "special-service-request" → "other", + "frequent-flyer-number" → "other", + "travel-details" → "other", + "payment-details" → "other", + "place-port-of-original-embarkation" → "other", + "place-port-of-clearance" → "other", + "place-port-of-onward-foreign-destination" → "other", + "passenger-name-record-locator-number" → "other", + "mobile-application-id" → "other") +} \ No newline at end of file diff --git a/conf/routes b/conf/routes index 00bf1381f..0b44c2cc4 100644 --- a/conf/routes +++ b/conf/routes @@ -2,6 +2,7 @@ # This file defines all application routes (Higher priority routes first) # ~~~~ + GET / controllers.Default.redirect(to = "/index.html") GET /api/analyzer controllers.AnalyzerCtrl.list GET /api/analyzer/:id controllers.AnalyzerCtrl.get(id) @@ -12,4 +13,7 @@ GET /api/job/:id controllers.JobCtrl.get(id) DELETE /api/job/:id controllers.JobCtrl.remove(id) GET /api/job/:id/report controllers.JobCtrl.report(id) GET /api/job/:id/waitreport controllers.JobCtrl.waitReport(id, atMost ?= "Inf") +GET /modules controllers.MispCtrl.modules +POST /query controllers.MispCtrl.query GET /*file controllers.AssetCtrl.get(file) +