From f84342fdd7cc65b101b11f9403bd1573223122ac Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 5 May 2017 13:29:48 +0200 Subject: [PATCH] #21 Restructure classes --- app/Module.scala | 11 +- .../{Analyzer.scala => AnalyzerCtrl.scala} | 17 +- app/controllers/Asset.scala | 6 +- app/controllers/{Job.scala => JobCtrl.scala} | 30 +-- app/controllers/MispCtrl.scala | 2 + app/models/Analyzer.scala | 6 +- app/models/ExternalAnalyzer.scala | 84 +------ app/models/Job.scala | 20 +- app/models/JsonFormat.scala | 79 +++++-- app/models/MispModule.scala | 93 +------- app/models/Report.scala | 9 + app/services/Analyzer.scala | 107 --------- app/services/AnalyzerSrv.scala | 37 +++ app/services/ExternalAnalyzerSrv.scala | 142 ++++++++++++ app/services/{Job.scala => JobSrv.scala} | 70 +++--- app/services/MispSrv.scala | 219 ++++++++++++++---- app/util/JsonConfig.scala | 14 +- build.sbt | 11 +- contrib/misp-modules-loader.py | 2 +- project/Bintray.scala | 20 +- project/FrontEnd.scala | 2 +- project/Release.scala | 7 +- 22 files changed, 537 insertions(+), 451 deletions(-) rename app/controllers/{Analyzer.scala => AnalyzerCtrl.scala} (89%) rename app/controllers/{Job.scala => JobCtrl.scala} (54%) create mode 100644 app/models/Report.scala delete mode 100644 app/services/Analyzer.scala create mode 100644 app/services/AnalyzerSrv.scala create mode 100644 app/services/ExternalAnalyzerSrv.scala rename app/services/{Job.scala => JobSrv.scala} (73%) diff --git a/app/Module.scala b/app/Module.scala index 5a042f5ef..3dacb8fda 100644 --- a/app/Module.scala +++ b/app/Module.scala @@ -1,16 +1,13 @@ -import play.api.{ Configuration, Environment, Mode } -import play.api.libs.concurrent.AkkaGuiceSupport - import com.google.inject.AbstractModule - -import net.codingwell.scalaguice.ScalaModule - import controllers.{ AssetCtrl, AssetCtrlDev, AssetCtrlProd } +import net.codingwell.scalaguice.ScalaModule +import play.api.libs.concurrent.AkkaGuiceSupport +import play.api.{ Configuration, Environment, Mode } import services.JobActor class Module(environment: Environment, configuration: Configuration) extends AbstractModule with ScalaModule with AkkaGuiceSupport { - override def configure() = { + override def configure(): Unit = { bindActor[JobActor]("JobActor") if (environment.mode == Mode.Prod) diff --git a/app/controllers/Analyzer.scala b/app/controllers/AnalyzerCtrl.scala similarity index 89% rename from app/controllers/Analyzer.scala rename to app/controllers/AnalyzerCtrl.scala index 644571788..8464291bf 100644 --- a/app/controllers/Analyzer.scala +++ b/app/controllers/AnalyzerCtrl.scala @@ -2,16 +2,14 @@ package controllers import javax.inject.Inject -import scala.annotation.implicitNotFound -import scala.concurrent.{ ExecutionContext, Future } - +import models.JsonFormat.{ analyzerWrites, dataActifactReads, jobWrites } +import models.{ DataArtifact, FileArtifact } import play.api.libs.json.{ JsObject, JsString, Json } import play.api.mvc.{ Action, AnyContent, Controller, Request } - -import models.{ DataArtifact, FileArtifact } -import models.JsonFormat.{ analyzerWrites, dataActifactReads, jobWrites } import services.{ AnalyzerSrv, JobSrv } +import scala.concurrent.{ ExecutionContext, Future } + class AnalyzerCtrl @Inject() ( analyzerSrv: AnalyzerSrv, jobSrv: JobSrv, @@ -38,7 +36,7 @@ class AnalyzerCtrl @Inject() ( private[controllers] def readFileArtifact(request: Request[AnyContent]) = { for { parts ← request.body.asMultipartFormData - filePart ← parts.file("data").headOption + filePart ← parts.file("data") attrList ← parts.dataParts.get("_json") attrStr ← attrList.headOption attr ← Json.parse(attrStr).asOpt[JsObject] @@ -47,11 +45,12 @@ class AnalyzerCtrl @Inject() ( ("filename" → JsString(filePart.filename))) } - def analyze(analyzerId: String) = Action.async { request ⇒ + def analyze(analyzerId: String): Action[AnyContent] = Action.async { request ⇒ readDataArtifact(request) .orElse(readFileArtifact(request)) .map { artifact ⇒ - jobSrv.create(artifact, analyzerId) + analyzerSrv.analyze(analyzerId, artifact) + //jobSrv.create(artifact, analyzerId) .map(j ⇒ Ok(Json.toJson(j))) } .getOrElse(Future.successful(BadRequest("???"))) diff --git a/app/controllers/Asset.scala b/app/controllers/Asset.scala index 102cd11ab..6a37553ec 100644 --- a/app/controllers/Asset.scala +++ b/app/controllers/Asset.scala @@ -4,7 +4,7 @@ import javax.inject.{ Inject, Singleton } import play.api.Environment import play.api.http.HttpErrorHandler -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc.{ Action, AnyContent } trait AssetCtrl { def get(file: String): Action[AnyContent] @@ -12,12 +12,12 @@ trait AssetCtrl { @Singleton class AssetCtrlProd @Inject() (errorHandler: HttpErrorHandler) extends Assets(errorHandler) with AssetCtrl { - def get(file: String) = at("/ui", file) + def get(file: String): Action[AnyContent] = at("/ui", file) } @Singleton class AssetCtrlDev @Inject() (environment: Environment) extends ExternalAssets(environment) with AssetCtrl { - def get(file: String) = { + def get(file: String): Action[AnyContent] = { if (file.startsWith("bower_components/")) { at("ui", file) } diff --git a/app/controllers/Job.scala b/app/controllers/JobCtrl.scala similarity index 54% rename from app/controllers/Job.scala rename to app/controllers/JobCtrl.scala index 2a96ca0a1..af20f5395 100644 --- a/app/controllers/Job.scala +++ b/app/controllers/JobCtrl.scala @@ -6,40 +6,38 @@ import scala.annotation.implicitNotFound import scala.concurrent.ExecutionContext import scala.concurrent.duration.Duration import scala.util.{ Failure, Success } - import play.api.libs.json.{ JsString, Json } -import play.api.mvc.{ Action, Controller } - -import models.JsonFormat.{ jobStatusWrites, jobWrites } +import play.api.mvc.{ Action, AnyContent, Controller } +import models.JsonFormat._ import services.JobSrv class JobCtrl @Inject() ( jobSrv: JobSrv, implicit val ec: ExecutionContext) extends Controller { - def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int) = Action.async { request ⇒ + def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Action[AnyContent] = Action.async { request ⇒ jobSrv.list(dataTypeFilter, dataFilter, analyzerFilter, start, limit).map { case (total, jobs) ⇒ Ok(Json.toJson(jobs)).withHeaders("X-Total" → total.toString) } } - def get(jobId: String) = Action.async { request ⇒ + def get(jobId: String): Action[AnyContent] = Action.async { request ⇒ jobSrv.get(jobId).map { job ⇒ Ok(Json.toJson(job)) } } - def remove(jobId: String) = Action.async { request ⇒ + def remove(jobId: String): Action[AnyContent] = Action.async { request ⇒ jobSrv.remove(jobId).map(_ ⇒ Ok("")) } - def report(jobId: String) = Action.async { request ⇒ + def report(jobId: String): Action[AnyContent] = Action.async { request ⇒ jobSrv .get(jobId) .map { job ⇒ val report = job.report.value match { - case Some(Success(report)) ⇒ report - case Some(Failure(error)) ⇒ JsString(error.getMessage) - case None ⇒ JsString("Running") + case Some(Success(r)) ⇒ Json.toJson(r) + case Some(Failure(error)) ⇒ JsString(error.getMessage) + case None ⇒ JsString("Running") } Ok(jobWrites.writes(job) + ("status" → jobStatusWrites.writes(job.status)) + @@ -47,12 +45,8 @@ class JobCtrl @Inject() ( } } - def waitReport(jobId: String, atMost: String) = Action.async { request ⇒ - for { - job ← jobSrv.get(jobId) - (status, report) ← jobSrv.waitReport(jobId, Duration(atMost)) - } yield Ok(jobWrites.writes(job) + - ("status" → jobStatusWrites.writes(job.status)) + - ("report" → report)) + def waitReport(jobId: String, atMost: String): Action[AnyContent] = Action.async { request ⇒ + jobSrv.waitReport(jobId, Duration(atMost)) + .map { job ⇒ Ok(Json.toJson(job)) } } } \ No newline at end of file diff --git a/app/controllers/MispCtrl.scala b/app/controllers/MispCtrl.scala index 312708b65..8792921e5 100644 --- a/app/controllers/MispCtrl.scala +++ b/app/controllers/MispCtrl.scala @@ -2,6 +2,7 @@ package controllers import javax.inject.Inject +import play.api.Logger import play.api.libs.json.{ JsObject, JsValue } import play.api.mvc.{ Action, AnyContent, Controller } import services.MispSrv @@ -10,6 +11,7 @@ import scala.concurrent.ExecutionContext class MispCtrl @Inject() (mispSrv: MispSrv, implicit val ec: ExecutionContext) extends Controller { + private[MispCtrl] lazy val logger = Logger(getClass) def modules: Action[AnyContent] = Action { _ ⇒ Ok(mispSrv.moduleList) } diff --git a/app/models/Analyzer.scala b/app/models/Analyzer.scala index ce853ac97..d13381f3b 100644 --- a/app/models/Analyzer.scala +++ b/app/models/Analyzer.scala @@ -1,10 +1,6 @@ package models -import scala.concurrent.Future -import play.api.libs.json.JsObject - abstract class Analyzer { - def analyze(artifact: Artifact): Future[JsObject] val name: String val version: String val description: String @@ -12,5 +8,5 @@ abstract class Analyzer { val author: String val url: String val license: String - val id = (name + "_" + version).replaceAll("\\.", "_") + val id: String = (name + "_" + version).replaceAll("\\.", "_") } diff --git a/app/models/ExternalAnalyzer.scala b/app/models/ExternalAnalyzer.scala index 2ce882b5a..97e2556d5 100644 --- a/app/models/ExternalAnalyzer.scala +++ b/app/models/ExternalAnalyzer.scala @@ -1,80 +1,16 @@ package models -import java.io.{ BufferedReader, InputStreamReader } import java.nio.file.Path -import scala.concurrent.{ ExecutionContext, Future, blocking } -import scala.sys.process.{ BasicIO, Process, ProcessIO } - -import akka.stream.Materializer - -import play.api.Logger -import play.api.libs.json.{ JsObject, JsString, Json } - -import com.fasterxml.jackson.core.JsonParseException -import com.fasterxml.jackson.databind.JsonMappingException +import play.api.libs.json.JsObject case class ExternalAnalyzer( - name: String, - version: String, - description: String, - dataTypeList: Seq[String], - author: String, - url: String, - license: String, - command: Path, - config: JsObject)(implicit val ec: ExecutionContext) extends Analyzer { - - val log = Logger(getClass) - private val osexec = if (System.getProperty("os.name").toLowerCase.contains("win")) - (c: String) ⇒ s"""cmd /c $c""" - else - (c: String) ⇒ s"""sh -c "./$c" """ - - override def analyze(artifact: Artifact): Future[JsObject] = { - Future { - val input = artifact match { - case FileArtifact(file, attributes) ⇒ attributes + ("file" → JsString(file.getAbsoluteFile.toString)) + ("config" → config) - case DataArtifact(data, attributes) ⇒ attributes + ("data" → JsString(data)) + ("config" → config) - } - val output = new StringBuffer - val error = new StringBuffer - try { - log.info(s"Execute ${osexec(command.getFileName.toString)} in ${command.getParent.toFile.getAbsoluteFile.getName}") - val exitValue = Process(osexec(command.getFileName.toString), command.getParent.toFile).run( - new ProcessIO( - { stdin ⇒ - try stdin.write(input.toString.getBytes("UTF-8")) - finally stdin.close() - }, - { stdout ⇒ - val reader = new BufferedReader(new InputStreamReader(stdout, "UTF-8")) - try BasicIO.processLinesFully { line ⇒ - output.append(line).append(System.lineSeparator()) - () - }(reader.readLine) - finally reader.close() - }, - { stderr ⇒ - val reader = new BufferedReader(new InputStreamReader(stderr, "UTF-8")) - try BasicIO.processLinesFully { line ⇒ - error.append(line).append(System.lineSeparator()) - () - }(reader.readLine) - finally reader.close() - })).exitValue - Json.parse(output.toString).as[JsObject] - } - catch { - case _: JsonMappingException ⇒ - error.append(output) - JsObject(Seq("errorMessage" → JsString(s"Error: Invalid output\n$error"))) - case _: JsonParseException ⇒ - error.append(output) - JsObject(Seq("errorMessage" → JsString(s"Error: Invalid output\n$error"))) - case t: Throwable ⇒ - JsObject(Seq("errorMessage" → JsString(t.getMessage + ":" + t.getStackTrace().mkString("", "\n\t", "\n")))) - } - } - } -} + name: String, + version: String, + description: String, + dataTypeList: Seq[String], + author: String, + url: String, + license: String, + command: Path, + config: JsObject) extends Analyzer \ No newline at end of file diff --git a/app/models/Job.scala b/app/models/Job.scala index 8d4460f64..bdc90644d 100644 --- a/app/models/Job.scala +++ b/app/models/Job.scala @@ -1,24 +1,22 @@ package models -import play.api.libs.json.JsObject -import scala.concurrent.Future import java.util.Date -import scala.util.Success -import scala.util.Failure + +import scala.concurrent.Future +import scala.util.{ Failure, Success } object JobStatus extends Enumeration { type Type = Value val InProgress, Success, Failure = Value } -case class Job(id: String, analyzerId: String, artifact: Artifact, report: Future[JsObject]) { + +case class Job(id: String, analyzer: Analyzer, artifact: Artifact, report: Future[Report]) { val date: Date = new Date() def status: JobStatus.Type = report.value match { - case Some(Success(x)) ⇒ (x \ "success").asOpt[Boolean] match { - case Some(true) ⇒ JobStatus.Success - case _ ⇒ JobStatus.Failure - } - case Some(Failure(_)) ⇒ JobStatus.Failure - case None ⇒ JobStatus.InProgress + case Some(Success(SuccessReport(_, _, _))) ⇒ JobStatus.Success + case Some(Success(FailureReport(_))) ⇒ JobStatus.Failure + case Some(Failure(_)) ⇒ JobStatus.Failure + case None ⇒ JobStatus.InProgress } } diff --git a/app/models/JsonFormat.scala b/app/models/JsonFormat.scala index b2536890c..9a81cd685 100644 --- a/app/models/JsonFormat.scala +++ b/app/models/JsonFormat.scala @@ -1,19 +1,11 @@ package models -import scala.annotation.implicitNotFound - -import play.api.libs.json.Json +import models.JobStatus.Type import play.api.libs.json.Json.toJsFieldJsValueWrapper -import play.api.libs.json.Writes -import scala.concurrent.Future -import play.api.libs.json.JsObject -import scala.util.Success -import scala.util.Failure -import play.api.libs.json.JsString -import play.api.libs.json.OWrites +import play.api.libs.json.{ JsString, Json, OWrites, Writes, _ } object JsonFormat { - implicit val analyzerWrites = Writes[Analyzer](analyzer ⇒ Json.obj( + implicit val analyzerWrites: Writes[Analyzer] = Writes[Analyzer](analyzer ⇒ Json.obj( "name" → analyzer.name, "version" → analyzer.version, "description" → analyzer.description, @@ -23,24 +15,65 @@ object JsonFormat { "license" → analyzer.license, "id" → analyzer.id)) - implicit val fileArtifactWrites = OWrites[FileArtifact](fileArtifact ⇒ Json.obj( + implicit val fileArtifactWrites: OWrites[FileArtifact] = OWrites[FileArtifact](fileArtifact ⇒ Json.obj( "attributes" → fileArtifact.attributes)) - implicit val dataArtifactWrites = Json.writes[DataArtifact] - implicit val dataActifactReads = Json.reads[DataArtifact] + implicit val dataArtifactWrites: OWrites[DataArtifact] = Json.writes[DataArtifact] + implicit val dataActifactReads: Reads[DataArtifact] = Json.reads[DataArtifact] - implicit val artifactWrites = OWrites[Artifact](artifact ⇒ artifact match { + val artifactWrites: OWrites[Artifact] = OWrites[Artifact] { case dataArtifact: DataArtifact ⇒ dataArtifactWrites.writes(dataArtifact) case fileArtifact: FileArtifact ⇒ fileArtifactWrites.writes(fileArtifact) - }) + } + + val artifactReads: Reads[Artifact] = Reads[Artifact] { json ⇒ + (json \ "data").asOpt[String] + .map { data ⇒ JsSuccess(DataArtifact(data, json.as[JsObject])) } + .getOrElse(JsError(__ \ "data", "data is missing")) + } + implicit val artifactFormat = Format(artifactReads, artifactWrites) + + implicit val jobStatusWrites: Writes[Type] = Writes[JobStatus.Type](jobStatus ⇒ JsString(jobStatus.toString)) + + val reportArtifactReads: Reads[Artifact] = + for { + tpe ← (__ \ "type").read[String] + value ← (__ \ "value").read[String] + } yield DataArtifact(value, Json.obj("dataType" → tpe)) + + val reportReads: Reads[Report] = Reads[Report] { json ⇒ + val success = (json \ "success").asOpt[Boolean].getOrElse(false) + JsSuccess { + if (success) + (for { + artifacts ← (json \ "artifacts").asOpt[Seq[Artifact]](Reads.seq(reportArtifactReads)) + full ← (json \ "full").asOpt[JsObject] + summary ← (json \ "summary").asOpt[JsObject] + } yield SuccessReport(artifacts, full, summary)) + .getOrElse(FailureReport(s"Invalid analyzer output format : $json")) + else + FailureReport((json \ "error").asOpt[String].getOrElse(json.toString)) + } + } - implicit val jobStatusWrites = Writes[JobStatus.Type](jobStatus ⇒ JsString(jobStatus.toString)) + val reportWrites: Writes[Report] = Writes[Report] { + case SuccessReport(artifacts, full, summary) ⇒ Json.obj( + "artifacts" → artifacts, + "full" → full, + "summary" → summary) + case FailureReport(message) ⇒ Json.obj("errorMessage" → message) + } - implicit val jobWrites = OWrites[Job](job ⇒ Json.obj( - "id" → job.id, - "analyzerId" → job.analyzerId, - "status" → job.status, - "date" → job.date, - "artifact" → job.artifact)) + implicit val reportFormat: Format[Report] = Format[Report](reportReads, reportWrites) + implicit val jobWrites: OWrites[Job] = OWrites[Job] { job ⇒ + val report = job.report.value.flatMap(_.toOption).map(Json.toJson(_)).getOrElse(JsNull) + Json.obj( + "id" → job.id, + "analyzerId" → job.analyzer.id, + "status" → job.status, + "date" → job.date, + "artifact" → job.artifact, + "report" → report) + } } diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala index a715abdfd..14ef92350 100644 --- a/app/models/MispModule.scala +++ b/app/models/MispModule.scala @@ -1,97 +1,14 @@ package models -import java.io._ - -import org.apache.commons.codec.binary.Base64InputStream -import play.api.Logger -import play.api.libs.json.{ JsObject, Json } -import services.MispSrv - -import scala.collection.JavaConverters._ -import scala.concurrent.{ ExecutionContext, Future } -import scala.sys.process._ -import scala.util.{ Failure, Success, Try } - case class MispModule( - mispSrv: MispSrv, - name: String, + name: String, version: String, description: String, - dataTypeList: Seq[String], author: String, - moduleName: String, - loaderCommand: String)(implicit val ec: ExecutionContext) extends Analyzer { - + dataTypeList: Seq[String], + inputAttributes: Seq[String], + config: Seq[String], + loaderCommand: String) extends Analyzer { val license = "AGPL-3.0" val url = "https://github.com/MISP/misp-modules" - - private def stringStream(string: String): InputStream = { - new ByteArrayInputStream(string.getBytes) - } - def analyze(artifact: Artifact): Future[JsObject] = { - val input = artifact match { - case DataArtifact(data, _) ⇒ - stringStream(Json.obj(mispSrv.dataType2mispType(artifact.dataType).head → data).toString) - case FileArtifact(data, _) ⇒ - new SequenceInputStream(Iterator( - stringStream("""{"attachment":""""), - new Base64InputStream(new FileInputStream(data), true), - stringStream("\"}")).asJavaEnumeration) - } - val output = (s"$loaderCommand --run $moduleName" #< input).!! - Future { - Json.parse(output).as[JsObject] - } - } -} - -object MispModule { - private[MispModule] lazy val logger = Logger(getClass) - def list(loaderCommand: String): Seq[String] = - Json.parse(s"$loaderCommand --list".!!).as[Seq[String]] - - def apply( - loaderCommand: String, - moduleName: String, - mispSrv: MispSrv)(implicit ec: ExecutionContext): Option[MispModule] = { - println(s"Loading MISP module $moduleName") - for { - moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) match { - case Success(s) ⇒ Some(s) - case Failure(f) ⇒ - f.printStackTrace() - None - } - name ← (moduleInfo \ "name").asOpt[String].orElse { - println("name not defined") - None - } - version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String].orElse { - println("version not defined") - None - } - description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String].orElse { - println("description not defined") - None - } - dataTypeList ← (moduleInfo \ "mispattributes" \ "input") - .asOpt[Seq[String]] - .map(_.map(mispSrv.mispType2dataType(_)).distinct) - .orElse { - println("input attributes not defined") - None - } - author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String].orElse { - println("author not defined") - None - } - mispModule ← Try(MispModule(mispSrv, name, version, description, dataTypeList, author, moduleName, loaderCommand)) match { - case Success(s) ⇒ Some(s) - case Failure(f) ⇒ - f.printStackTrace() - sys.error("Load module fails") - } - _ = println("Module load succeed") - } yield mispModule - } } \ No newline at end of file diff --git a/app/models/Report.scala b/app/models/Report.scala new file mode 100644 index 000000000..296466413 --- /dev/null +++ b/app/models/Report.scala @@ -0,0 +1,9 @@ +package models + +import play.api.libs.json.JsObject + +sealed abstract class Report(success: Boolean) + +case class SuccessReport(artifacts: Seq[Artifact], full: JsObject, summary: JsObject) extends Report(true) + +case class FailureReport(message: String) extends Report(false) \ No newline at end of file diff --git a/app/services/Analyzer.scala b/app/services/Analyzer.scala deleted file mode 100644 index b69c56cd6..000000000 --- a/app/services/Analyzer.scala +++ /dev/null @@ -1,107 +0,0 @@ -package services - -import java.io.File -import java.nio.file.{ Files, Path, Paths } -import javax.inject.{ Inject, Provider, Singleton } - -import akka.actor.ActorSystem -import models.{ Analyzer, ExternalAnalyzer, MispModule } -import play.api.libs.json.{ JsObject, JsValue, Json } -import play.api.{ Configuration, Logger } -import util.JsonConfig.configWrites - -import scala.collection.JavaConversions.iterableAsScalaIterable -import scala.concurrent.ExecutionContext -import scala.util.Try - -@Singleton -class AnalyzerSrv( - mispSrvProvider: Provider[MispSrv], - analyzerPath: Path, - analyzerConfig: JsObject, - mispModulesPath: Path, - mispModuleLoaderCommand: Option[String], - akkaSystem: ActorSystem) { - @Inject def this( - mispSrvProvider: Provider[MispSrv], - configuration: Configuration, - akkaSystem: ActorSystem) = - this( - mispSrvProvider, - Paths.get(configuration.getString("analyzer.path").getOrElse(".")), - configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), - Paths.get(configuration.getString("misp.modules.path").getOrElse(".")), - configuration.getString("misp.modules.loader"), - akkaSystem) - - private[AnalyzerSrv] lazy val logger = Logger(getClass) - lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") - lazy val mispSrv = mispSrvProvider.get - - private lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers - private lazy val mispModules: Seq[Analyzer] = getMispModules - def list: Seq[Analyzer] = externalAnalyzers ++ mispModules // ::: javaAnalyzers - def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) - def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) - - private[services] def getExternalAnalyzers: Seq[Analyzer] = { - val globalConfig = (analyzerConfig \ "global").asOpt[JsObject].getOrElse(JsObject(Nil)) - for { - analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).toSeq).getOrElse { - logger.warn(s"Analyzer directory ($analyzerPath) is not found") - Nil - } - if Files.isDirectory(analyzerDir) - infoFile ← Files.newDirectoryStream(analyzerDir, "*.json").toSeq - if Files.isReadable(infoFile) - info = readInfo(infoFile) - name ← (info \ "name").asOpt[String] orElse { - logger.warn(s"name is missing in $infoFile"); None - } - version ← (info \ "version").asOpt[String] orElse { - logger.warn(s"version is missing in $infoFile"); None - } - description ← (info \ "description").asOpt[String] orElse { - logger.warn(s"description is missing in $infoFile"); None - } - dataTypeList ← (info \ "dataTypeList").asOpt[Seq[String]] orElse { - logger.warn(s"dataTypeList is missing in $infoFile"); None - } - command ← (info \ "command").asOpt[String] orElse { - logger.warn(s"command is missing in $infoFile"); None - } - author ← (info \ "author").asOpt[String] orElse { - logger.warn(s"author is missing in $infoFile"); None - } - url ← (info \ "url").asOpt[String] orElse { - logger.warn(s"url is missing in $infoFile"); None - } - license ← (info \ "license").asOpt[String] orElse { - logger.warn(s"license is missing in $infoFile"); None - } - config = (info \ "config").asOpt[JsObject].getOrElse(JsObject(Nil)) - baseConfig = (info \ "baseConfig").asOpt[String].flatMap(c ⇒ (analyzerConfig \ c).asOpt[JsObject]).getOrElse(JsObject(Nil)) - absoluteCommand = analyzerPath.resolve(Paths.get(command.replaceAll("[\\/]", File.separator))) - _ = logger.info(s"Register analyzer $name $version (${(name + "_" + version).replaceAll("\\.", "_")})") - } yield ExternalAnalyzer(name, version, description, dataTypeList, author, url, license, absoluteCommand, globalConfig deepMerge baseConfig deepMerge config)(analyzeExecutionContext) - } - - private[services] def getMispModules: Seq[Analyzer] = { - for { - loaderCommand ← mispModuleLoaderCommand.toSeq - moduleName ← MispModule.list(loaderCommand) - - _ = println("MISP module loading ...") - mispModule ← MispModule(loaderCommand, moduleName, mispSrv)(analyzeExecutionContext) - _ = println("MISP module load success") - } yield mispModule - } - - private[services] def readInfo(file: Path): JsValue = { - val source = scala.io.Source.fromFile(file.toFile) - try { - Json.parse(source.mkString) - } - finally { source.close() } - } -} diff --git a/app/services/AnalyzerSrv.scala b/app/services/AnalyzerSrv.scala new file mode 100644 index 000000000..4637cf98b --- /dev/null +++ b/app/services/AnalyzerSrv.scala @@ -0,0 +1,37 @@ +package services + +import javax.inject.{ Inject, Singleton } + +import models._ +import play.api.Logger + +import scala.concurrent.Future + +@Singleton +class AnalyzerSrv @Inject() ( + jobSrv: JobSrv, + mispSrv: MispSrv, + externalAnalyzerSrv: ExternalAnalyzerSrv) { + + private[AnalyzerSrv] lazy val logger = Logger(getClass) + + def list: Seq[Analyzer] = externalAnalyzerSrv.list ++ mispSrv.list + + def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) + + def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) + + def analyze(analyzerId: String, artifact: Artifact): Future[Job] = { + get(analyzerId) + .map { analyzer ⇒ analyze(analyzer, artifact) } + .getOrElse(sys.error("analyzer not found")) + } + + def analyze(analyzer: Analyzer, artifact: Artifact): Future[Job] = { + val report = analyzer match { + case ea: ExternalAnalyzer ⇒ externalAnalyzerSrv.analyze(ea, artifact) + case mm: MispModule ⇒ mispSrv.analyze(mm, artifact) + } + jobSrv.create(analyzer, artifact, report) + } +} diff --git a/app/services/ExternalAnalyzerSrv.scala b/app/services/ExternalAnalyzerSrv.scala new file mode 100644 index 000000000..ae1927e69 --- /dev/null +++ b/app/services/ExternalAnalyzerSrv.scala @@ -0,0 +1,142 @@ +package services + +import java.io.{ BufferedReader, File, InputStreamReader } +import java.nio.file.{ Files, Path, Paths } +import javax.inject.{ Inject, Singleton } + +import akka.actor.ActorSystem +import com.fasterxml.jackson.core.JsonParseException +import com.fasterxml.jackson.databind.JsonMappingException +import models.JsonFormat._ +import models._ +import util.JsonConfig +import play.api.libs.json._ +import play.api.{ Configuration, Logger } + +import scala.collection.JavaConversions.iterableAsScalaIterable +import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process.{ BasicIO, Process, ProcessIO } +import scala.util.{ Failure, Try } + +@Singleton +class ExternalAnalyzerSrv( + analyzerPath: Path, + analyzerConfig: JsObject, + akkaSystem: ActorSystem) { + + @Inject() def this(configuration: Configuration, akkaSystem: ActorSystem) = + this( + Paths.get(configuration.getString("analyzer.path").getOrElse(".")), + JsonConfig.configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), + akkaSystem) + + private[ExternalAnalyzerSrv] lazy val analyzeExecutionContext: ExecutionContext = + akkaSystem.dispatchers.lookup("analyzer") + private[ExternalAnalyzerSrv] lazy val globalConfig: JsObject = + (analyzerConfig \ "global").asOpt[JsObject].getOrElse(JsObject(Nil)) + private[ExternalAnalyzerSrv] lazy val logger = + Logger(getClass) + + lazy val list: Seq[ExternalAnalyzer] = { + for { + analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).toSeq).getOrElse { + logger.warn(s"Analyzer directory ($analyzerPath) is not found") + Nil + } + if Files.isDirectory(analyzerDir) + infoFile ← Files.newDirectoryStream(analyzerDir, "*.json").toSeq + if Files.isReadable(infoFile) + analyzer ← Try(readInfo(infoFile).as[ExternalAnalyzer](reads)) + .recoverWith { + case error ⇒ + logger.warn(s"Load of analyzer $infoFile fails", error) + Failure(error) + } + .toOption + _ = logger.info(s"Register analyzer ${analyzer.name} ${analyzer.version} (${analyzer.id})") + } yield analyzer + } + + def get(analyzerId: String): Option[ExternalAnalyzer] = list.find(_.id == analyzerId) + + private val osexec = + if (System.getProperty("os.name").toLowerCase.contains("win")) + (c: String) ⇒ s"""cmd /c $c""" + else + (c: String) ⇒ s"""sh -c "./$c" """ + + def analyze(analyzer: ExternalAnalyzer, artifact: Artifact): Future[Report] = { + Future { + val input = artifact match { + case FileArtifact(file, attributes) ⇒ attributes + ("file" → JsString(file.getAbsoluteFile.toString)) + ("config" → analyzer.config) + case DataArtifact(data, attributes) ⇒ attributes + ("data" → JsString(data)) + ("config" → analyzer.config) + } + val output = new StringBuffer + val error = new StringBuffer + try { + logger.info(s"Execute ${osexec(analyzer.command.getFileName.toString)} in ${analyzer.command.getParent.toFile.getAbsoluteFile.getName}") + Process(osexec(analyzer.command.getFileName.toString), analyzer.command.getParent.toFile).run( + new ProcessIO( + { stdin ⇒ + try stdin.write(input.toString.getBytes("UTF-8")) + finally stdin.close() + }, { stdout ⇒ + val reader = new BufferedReader(new InputStreamReader(stdout, "UTF-8")) + try BasicIO.processLinesFully { line ⇒ + output.append(line).append(System.lineSeparator()) + () + }(reader.readLine) + finally reader.close() + }, { stderr ⇒ + val reader = new BufferedReader(new InputStreamReader(stderr, "UTF-8")) + try BasicIO.processLinesFully { line ⇒ + error.append(line).append(System.lineSeparator()) + () + }(reader.readLine) + finally reader.close() + })).exitValue + Json.parse(output.toString).as[Report] + } + catch { + case _: JsonMappingException ⇒ + error.append(output) + FailureReport(s"Error: Invalid output\n$error") + case _: JsonParseException ⇒ + error.append(output) + FailureReport(s"Error: Invalid output\n$error") + case t: Throwable ⇒ + FailureReport(t.getMessage + ":" + t.getStackTrace.mkString("", "\n\t", "\n")) + } + }(analyzeExecutionContext) + } + + private[ExternalAnalyzerSrv] def readInfo(file: Path): JsValue = { + val source = scala.io.Source.fromFile(file.toFile) + try Json.parse(source.mkString) + finally source.close() + } + + private[ExternalAnalyzerSrv] val reads: Reads[ExternalAnalyzer] = + for { + name ← (__ \ "name").read[String] + version ← (__ \ "version").read[String] + description ← (__ \ "description").read[String] + dataTypeList ← (__ \ "dataTypeList").read[Seq[String]] + author ← (__ \ "author").read[String] + url ← (__ \ "url").read[String] + license ← (__ \ "license").read[String] + command ← (__ \ "command").read[String] + absoluteCommand = analyzerPath.resolve(Paths.get(command.replaceAll("[\\/]", File.separator))) + config ← (__ \ "config").read[JsObject] + baseConfigKey ← (__ \ "baseConfig").read[String] + baseConfig = (analyzerConfig \ baseConfigKey).asOpt[JsObject].getOrElse(JsObject(Nil)) + } yield ExternalAnalyzer( + name, + version, + description, + dataTypeList, + author, + url, + license, + absoluteCommand, globalConfig deepMerge baseConfig deepMerge config) +} \ No newline at end of file diff --git a/app/services/Job.scala b/app/services/JobSrv.scala similarity index 73% rename from app/services/Job.scala rename to app/services/JobSrv.scala index c66a7b455..787ca7d6f 100644 --- a/app/services/Job.scala +++ b/app/services/JobSrv.scala @@ -6,8 +6,7 @@ import javax.inject.{ Inject, Named } import akka.actor.{ Actor, ActorRef, ActorSystem, actorRef2Scala } import akka.pattern.ask import akka.util.Timeout -import models.{ Analyzer, Artifact, Job, JobStatus } -import play.api.libs.json.{ JsString, JsValue } +import models._ import play.api.{ Configuration, Logger } import scala.concurrent.duration.Duration.Infinite @@ -16,11 +15,13 @@ import scala.concurrent.{ ExecutionContext, Future, Promise } import scala.util.Random class JobSrv @Inject() ( - analyzerSrv: AnalyzerSrv, + //analyzerSrv: AnalyzerSrv, @Named("JobActor") jobActor: ActorRef, implicit val ec: ExecutionContext, implicit val system: ActorSystem) { + import services.JobActor._ + implicit val timeout = Timeout(5.seconds) def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Future[(Int, Seq[Job])] = { @@ -36,17 +37,26 @@ class JobSrv @Inject() ( case _ ⇒ sys.error("TODO") } - def create(artifact: Artifact, analyzerId: String): Future[Job] = { - analyzerSrv.get(analyzerId) - .map { analyzer ⇒ create(artifact, analyzer) } - .getOrElse(Future.failed(new Exception("analyzer not found"))) + def create(analyzer: Analyzer, artifact: Artifact, report: Future[Report]): Future[Job] = { + (jobActor ? CreateJob(artifact, analyzer, report)) map { + case job: Job ⇒ job + case _ ⇒ sys.error("TODO") + } } - def create(artifact: Artifact, analyzer: Analyzer): Future[Job] = - (jobActor ? CreateJob(artifact, analyzer)) map { - case j: Job ⇒ j - case _ ⇒ sys.error("TODO") - } + // @deprecated + // def create(artifact: Artifact, analyzerId: String): Future[Job] = { + // analyzerSrv.get(analyzerId) + // .map { analyzer ⇒ create(artifact, analyzer) } + // .getOrElse(Future.failed(new Exception("analyzer not found"))) + // } + // + // @deprecated + // def create(artifact: Artifact, analyzer: Analyzer): Future[Job] = + // (jobActor ? CreateJob(artifact, analyzer)) map { + // case j: Job ⇒ j + // case _ ⇒ sys.error("TODO") + // } def remove(jobId: String): Future[Unit] = { (jobActor ? RemoveJob(jobId)).map { @@ -56,19 +66,20 @@ class JobSrv @Inject() ( } } - def waitReport(jobId: String, atMost: Duration): Future[(JobStatus.Type, JsValue)] = { - val statusResult = get(jobId) - .flatMap(_.report) - .map((JobStatus.Success, _)) - .recover { case error ⇒ (JobStatus.Failure, JsString(error.getMessage)) } - - atMost match { - case _: Infinite ⇒ statusResult - case duration: FiniteDuration ⇒ - val prom = Promise[(JobStatus.Type, JsValue)]() - val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.InProgress, JsString("Timeout"))); () } - statusResult.onComplete(_ ⇒ timeout.cancel()) - Future.firstCompletedOf(List(statusResult, prom.future)) + def waitReport(jobId: String, atMost: Duration): Future[Job] = { + get(jobId).flatMap { job ⇒ + val finishedJob = job.report.map(_ ⇒ job) + atMost match { + case _: Infinite ⇒ finishedJob + case duration: FiniteDuration ⇒ + val prom = Promise[Job]() + val timeout = system.scheduler.scheduleOnce(duration) { + prom.success(job) + () + } + finishedJob.onComplete(_ ⇒ timeout.cancel()) + Future.firstCompletedOf(List(finishedJob, prom.future)) + } } } } @@ -78,7 +89,7 @@ object JobActor { case class JobList(total: Int, jobs: Seq[Job]) case class GetJob(jobId: String) case object JobNotFound - case class CreateJob(artifact: Artifact, analyzer: Analyzer) + case class CreateJob(artifact: Artifact, analyzer: Analyzer, report: Future[Report]) case class RemoveJob(jobId: String) case object JobRemoved case object JobCleanup @@ -91,6 +102,7 @@ class JobActor( implicit val ec: ExecutionContext) extends Actor { import services.JobActor._ + @Inject def this( configuration: Configuration, analyzerSrv: AnalyzerSrv, @@ -122,7 +134,7 @@ class JobActor( val filteredJobs = jobs.filter(j ⇒ dataTypeFilter.fold(true)(j.artifact.dataTypeFilter) && dataFilter.fold(true)(j.artifact.dataFilter) && - analyzerFilter.fold(true)(j.analyzerId.contains)) + analyzerFilter.fold(true)(j.analyzer.id.contains)) sender ! JobList(filteredJobs.size, filteredJobs.slice(start, start + limit)) case GetJob(jobId) ⇒ sender ! jobs.find(_.id == jobId).getOrElse(JobNotFound) case RemoveJob(jobId) ⇒ @@ -132,9 +144,9 @@ class JobActor( context.become(jobState(j)) case None ⇒ sender ! JobNotFound } - case CreateJob(artifact, analyzer) ⇒ + case CreateJob(artifact, analyzer, report) ⇒ val jobId = Random.alphanumeric.take(16).mkString - val job = Job(jobId, analyzer.id, artifact, analyzer.analyze(artifact)) + val job = Job(jobId, analyzer, artifact, report) sender ! job context.become(jobState(job :: jobs)) case JobCleanup if jobLifeTime.isInstanceOf[FiniteDuration] ⇒ diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala index e11253e35..139eb749c 100644 --- a/app/services/MispSrv.scala +++ b/app/services/MispSrv.scala @@ -1,20 +1,77 @@ package services +import java.io.{ ByteArrayInputStream, FileInputStream, InputStream, SequenceInputStream } import javax.inject.Inject -import models.JsonFormat.dataActifactReads -import models.{ DataArtifact, FileArtifact } -import org.apache.commons.codec.binary.Base64 -import play.api.Logger -import play.api.libs.json.{ JsArray, JsObject, JsValue, Json } +import akka.actor.ActorSystem +import models.JsonFormat._ +import models._ +import org.apache.commons.codec.binary.{ Base64, Base64InputStream } +import play.api.libs.json.{ Json, _ } +import play.api.{ Configuration, Logger } +import scala.collection.JavaConverters._ import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process._ +import scala.util.{ Failure, Success, Try } + +class MispSrv( + loaderCommandOption: Option[String], + externalAnalyzerSrv: ExternalAnalyzerSrv, + jobSrv: JobSrv, + akkaSystem: ActorSystem) { + + @Inject() def this( + configuration: Configuration, + externalAnalyzerSrv: ExternalAnalyzerSrv, + jobSrv: JobSrv, + akkaSystem: ActorSystem) = this( + configuration.getString("misp.modules.loader"), + externalAnalyzerSrv, + jobSrv, + akkaSystem) -class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { private[MispSrv] lazy val logger = Logger(getClass) + private[MispSrv] lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") + + lazy val list: Seq[MispModule] = + loaderCommandOption.fold(Seq.empty[MispModule]) { loaderCommand ⇒ + Json.parse(s"$loaderCommand --list".!!) + .as[Seq[String]] + .map { moduleName ⇒ + moduleName → (for { + moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) + module ← Try(moduleInfo.as[MispModule](reads(loaderCommand))) + } yield module) + } + .flatMap { + case (moduleName, Failure(error)) ⇒ + logger.warn(s"Load MISP module $moduleName fails", error) + Nil + case (_, Success(module)) ⇒ + logger.info(s"Register MISP module ${module.name} ${module.version}") + Seq(module) + } + } + + def get(moduleName: String): Option[MispModule] = list.find(_.name == moduleName) - def moduleList: JsValue = { - JsArray(analyzerSrv.list.map { analyzer ⇒ + def moduleList: JsArray = { + val mispModules = list.map { module ⇒ + Json.obj( + "name" → module.name, + "type" → "cortex", + "mispattributes" → Json.obj( + "input" → module.inputAttributes, + "output" → Json.arr()), + "meta" → Json.obj( + "module-type" → Json.arr("cortex"), + "description" → module.description, + "author" → module.author, + "version" → module.version, + "config" → module.config)) + } + val externalAnalyzers = externalAnalyzerSrv.list.map { analyzer ⇒ Json.obj( "name" → analyzer.id, "type" → "cortex", @@ -27,58 +84,117 @@ class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { "author" → analyzer.author, "version" → analyzer.version, "config" → Json.arr())) - }) + } + JsArray(mispModules ++ externalAnalyzers) } def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = { - analyzerSrv.get(module).map { analyzer ⇒ - val artifact = mispType2dataType(mispType) match { - case "file" if mispType == "malware-sample" ⇒ ??? - case "file" ⇒ FileArtifact(Base64.decodeBase64(data), Json.obj( - "tlp" → 1, - "dataType" → "file")) - case dataType ⇒ DataArtifact(data, Json.obj( - "tlp" → 1, - "dataType" → dataType)) - } + loaderCommandOption + .flatMap { loaderCommand ⇒ + val artifact = toArtifact(mispType, data) + get(module) + .map { mispModule ⇒ + val mispReport = Future { + val input = Json.obj(mispType → data) + val output = (s"$loaderCommand --run $module" #< input.toString).!! + Json.parse(output).as[JsObject] + } + jobSrv.create(mispModule, artifact, mispReport.map(toReport)) + mispReport - analyzer - .analyze(artifact) - .map { output ⇒ - logger.info(s"analyzer output:\n$output") - val success = (output \ "success") - .asOpt[Boolean] - .getOrElse(false) - if (success) { - Json.obj( - "results" → ((output \ "artifacts") - .asOpt[Seq[JsObject]] - .getOrElse(Nil) - .map { artifact ⇒ - Json.obj( - "types" → dataType2mispType((artifact \ "type").as[String]), - "values" → Json.arr((artifact \ "value").as[String])) - } - :+ Json.obj( - "types" → Json.arr("cortex"), - "values" → Json.arr(output.toString)))) } - else { - val message = (output \ "error").asOpt[String].getOrElse(output.toString) - Json.obj( - "error" → message) + .orElse { + externalAnalyzerSrv + .get(module) + .map { analyzer ⇒ + externalAnalyzerSrv.analyze(analyzer, artifact) + .map { report ⇒ toMispOutput(report) } + } } + } + .getOrElse(Future.failed(new Exception(s"Module $module not found"))) + } + + def analyze(module: MispModule, artifact: Artifact): Future[Report] = { + def stringStream(string: String): InputStream = + new ByteArrayInputStream(string.getBytes) + + val input = artifact match { + case DataArtifact(data, _) ⇒ + stringStream(Json.obj(dataType2mispType(artifact.dataType).head → data).toString) + case FileArtifact(data, _) ⇒ + new SequenceInputStream(Iterator( + stringStream("""{"attachment":""""), + new Base64InputStream(new FileInputStream(data), true), + stringStream("\"}")).asJavaEnumeration) + } + + Future { + val output = (s"${module.loaderCommand} --run ${module.name}" #< input).!! + toReport(Json.parse(output).as[JsObject]) + }(analyzeExecutionContext) + } + + private def toArtifact(mispType: String, data: String): Artifact = { + mispType2dataType(mispType) match { + case "file" if mispType == "malware-sample" ⇒ ??? + case "file" ⇒ FileArtifact(Base64.decodeBase64(data), Json.obj( + "tlp" → 1, + "dataType" → "file")) + case dataType ⇒ DataArtifact(data, Json.obj( + "tlp" → 1, + "dataType" → dataType)) + } + } + + private def toReport(mispOutput: JsObject): Report = { + (mispOutput \ "results").asOpt[Seq[JsObject]] + .map { attributes ⇒ + val artifacts: Seq[Artifact] = for { + attribute ← attributes + tpe ← (attribute \ "types").asOpt[Seq[String]] + .orElse((attribute \ "types").asOpt[String].map(Seq(_))) + .getOrElse(Nil) + dataType = mispType2dataType(tpe) // TODO handle FileArtifact + value ← (attribute \ "values").asOpt[Seq[String]] + .orElse((attribute \ "values").asOpt[String].map(Seq(_))) + .getOrElse(Nil) + } yield DataArtifact(value, Json.obj("dataType" → dataType)) + SuccessReport(artifacts, Json.obj("artifacts" → Json.toJson(artifacts)), JsObject(Nil)) + } + .getOrElse { + val message = (mispOutput \ "error").asOpt[String].getOrElse(mispOutput.toString) + FailureReport(message) + } + } + + private def toMispOutput(report: Report): JsObject = { + report match { + case SuccessReport(artifacts, _, _) ⇒ + val attributes = artifacts.map { + case artifact: DataArtifact ⇒ + Json.obj( + "types" → dataType2mispType(artifact.dataType), + "values" → Json.arr(artifact.data)) + case artifact: FileArtifact ⇒ + ??? // TODO } + val cortexAttribute = Json.obj( + "types" → Seq("cortex"), + "values" → Json.arr(Json.toJson(report).toString)) + + Json.obj("results" → (attributes :+ cortexAttribute)) + case FailureReport(message) ⇒ + Json.obj("error" → message) } - .getOrElse(Future.failed(new Exception(s"Module $module not found"))) } - def mispType2dataType(mispType: String): String = typeLookup.getOrElse(mispType, { + private def mispType2dataType(mispType: String): String = typeLookup.getOrElse(mispType, { logger.warn(s"Misp type $mispType not recognized") "other" }) - def dataType2mispType(dataType: String): Seq[String] = { + private def dataType2mispType(dataType: String): Seq[String] = { val mispTypes = typeLookup.filter(_._2 == dataType) .keys .toSeq @@ -91,6 +207,17 @@ class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { else mispTypes } + private def reads(loaderCommand: String): Reads[MispModule] = + for { + name ← (__ \ "name").read[String] + version ← (__ \ "meta" \ "version").read[String] + description ← (__ \ "meta" \ "description").read[String] + author ← (__ \ "meta" \ "author").read[String] + config ← (__ \ "meta" \ "config").read[Seq[String]] + input ← (__ \ "mispattributes" \ "input").read[Seq[String]] + dataTypes = input.map(mispType2dataType) + } yield MispModule(name, version, description, author, dataTypes, input, config, loaderCommand) + private val typeLookup = Map( "md5" → "hash", "sha1" → "hash", diff --git a/app/util/JsonConfig.scala b/app/util/JsonConfig.scala index 0c91d548a..79e075e02 100644 --- a/app/util/JsonConfig.scala +++ b/app/util/JsonConfig.scala @@ -1,17 +1,15 @@ package util -import scala.BigDecimal -import scala.collection.JavaConversions.asScalaBuffer - +import com.typesafe.config.ConfigValueType.{ BOOLEAN, NULL, NUMBER, STRING } +import com.typesafe.config.{ ConfigList, ConfigObject, ConfigValue } import play.api.Configuration -import play.api.libs.json.{ JsArray, JsBoolean, JsNull, JsNumber, JsObject, JsString, OWrites, Writes } +import play.api.libs.json._ -import com.typesafe.config.{ ConfigList, ConfigObject, ConfigValue } -import com.typesafe.config.ConfigValueType.{ BOOLEAN, NULL, NUMBER, STRING } +import scala.collection.JavaConversions.asScalaBuffer object JsonConfig { implicit val configValueWrites: Writes[ConfigValue] = Writes((value: ConfigValue) ⇒ value match { - case v: ConfigObject ⇒ configWrites.writes(Configuration(v.toConfig())) + case v: ConfigObject ⇒ configWrites.writes(Configuration(v.toConfig)) case v: ConfigList ⇒ JsArray(v.toSeq.map(x ⇒ configValueWrites.writes(x))) case v if v.valueType == NUMBER ⇒ JsNumber(BigDecimal(v.unwrapped.asInstanceOf[java.lang.Number].toString)) case v if v.valueType == BOOLEAN ⇒ JsBoolean(v.unwrapped.asInstanceOf[Boolean]) @@ -19,7 +17,7 @@ object JsonConfig { case v if v.valueType == STRING ⇒ JsString(v.unwrapped.asInstanceOf[String]) }) - implicit val configWrites = OWrites { (cfg: Configuration) ⇒ + implicit def configWrites = OWrites { (cfg: Configuration) ⇒ JsObject(cfg.subKeys.map(key ⇒ key → configValueWrites.writes(cfg.underlying.getValue(key))).toSeq) } } \ No newline at end of file diff --git a/build.sbt b/build.sbt index b149c99fc..1668fa704 100644 --- a/build.sbt +++ b/build.sbt @@ -41,9 +41,8 @@ mappings in Universal ++= { } // Release // -import ReleaseTransformations._ - import Release._ +import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ bintrayOrganization := Some("cert-bdf") @@ -88,9 +87,9 @@ dockerUpdateLatest := true mappings in Universal += file("docker/entrypoint") -> "bin/entrypoint" mappings in Universal ~= { _.filterNot { - case (_, name) => name.startsWith("conf/") && name != "conf/keepme" + case (_, fileName) => fileName.startsWith("conf/") && name != "conf/keepme" }} -import com.typesafe.sbt.packager.docker.{ ExecCmd, Cmd } +import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd} dockerCommands := dockerCommands.value.map { case ExecCmd("ENTRYPOINT", _*) => ExecCmd("ENTRYPOINT", "bin/entrypoint") @@ -109,10 +108,10 @@ dockerCommands := dockerCommands.value.head +: dockerCommands.value.tail // Scalariform // -import scalariform.formatter.preferences._ -import com.typesafe.sbt.SbtScalariform import com.typesafe.sbt.SbtScalariform.ScalariformKeys +import scalariform.formatter.preferences._ + ScalariformKeys.preferences in ThisBuild := ScalariformKeys.preferences.value .setPreference(AlignParameters, false) // .setPreference(FirstParameterOnNewline, Force) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index fb0ed9e83..c265aaaf8 100755 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- diff --git a/project/Bintray.scala b/project/Bintray.scala index 7c3ad44e2..61fed98bf 100644 --- a/project/Bintray.scala +++ b/project/Bintray.scala @@ -1,18 +1,16 @@ import java.io.File -import scala.concurrent.duration.Duration -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global - -import sbt._ -import sbt.Keys._ - -import dispatch.{ Http, FunctionHandler } - -import bintry.Client import bintray.BintrayCredentials -import bintray.BintrayKeys.{ bintrayEnsureCredentials, bintrayOrganization, bintrayRepository, bintrayPackage } +import bintray.BintrayKeys.{bintrayEnsureCredentials, bintrayOrganization, bintrayPackage, bintrayRepository} +import bintry.Client import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport.Universal +import dispatch.{FunctionHandler, Http} +import sbt.Keys._ +import sbt._ + +import scala.concurrent.Await +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration object PublishToBinTray extends Plugin { val publishRelease = taskKey[Unit]("Publish binary in bintray") diff --git a/project/FrontEnd.scala b/project/FrontEnd.scala index f0bf55342..ac8b44d0c 100644 --- a/project/FrontEnd.scala +++ b/project/FrontEnd.scala @@ -1,5 +1,5 @@ -import sbt._ import sbt.Keys._ +import sbt._ object FrontEnd extends AutoPlugin { diff --git a/project/Release.scala b/project/Release.scala index 3dbeb2ee3..7a0184f08 100644 --- a/project/Release.scala +++ b/project/Release.scala @@ -1,10 +1,9 @@ -import sbt._ +import play.api.libs.json._ import sbt.Keys.baseDirectory -import sbt.{ Project, Extracted, State, IO, File, StateOps } +import sbt.{File, IO, Project, State, _} import sbtrelease.ReleasePlugin.autoImport._ import sbtrelease.ReleaseStateTransformations.readVersion -import sbtrelease.{ Vcs, Versions } -import play.api.libs.json._ +import sbtrelease.{Vcs, Versions} object Release { val releaseVersionUIFile = settingKey[File]("The json package file to write the version to")