From 96f44f04e3b0c30f44c832e6c4535c6f366ae6e2 Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 12:11:23 +0200 Subject: [PATCH 01/11] #21 create intermediate program to load misp-modules from Cortex --- contrib/misp-modules-loader.py | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 contrib/misp-modules-loader.py diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py new file mode 100644 index 000000000..9f2718467 --- /dev/null +++ b/contrib/misp-modules-loader.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import sys +import os +import getopt +import json + + + +# modules_path = "./misp-modules/misp_modules/modules/expansion" +# sys.path.append(modules_path) + +def run(argv): + try: + opts, args = getopt.getopt(argv, 'hp:i:r:', ["help","path=", "info=","run="]) + except getopt.GetoptError as err: + print(__file__ + " --info ") + print(__file__ + " --path --run ") + print(str(err)) + sys.exit(2) + + module = None + path = None + for opt,arg in opts: + + # TODO: check if module exist else exit + if opt in ('-h', '--help'): + print(__file__ + " --info ") + print(__file__ + " --run ") + sys.exit() + + elif opt in ('-p', '--path'): + path = arg + sys.path.append(path) + + elif opt in ('-r', '--run'): + module = arg + if path: + m = __import__(module) + + data = json.load(sys.stdin) + print(json.dumps(m.handler(json.dumps(data)))) + sys.exit(0) + + else: + print("add module path") + + elif opt in ('-i','--info'): + module = arg + print(module) + m = __import__(module) + print(({'name': module, 'mispattributes': m.mispattributes, + 'moduleinfo':m.moduleinfo})) + + + + +if __name__ == '__main__': + if len(sys.argv[1:]) > 0: + run(sys.argv[1:]) + else: + print(__file__ + " --info ") + print(__file__ + " --run ") + sys.exit(2) From fa812a86a42ac19c738934d19ee824a64cd1be72 Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 12:36:45 +0200 Subject: [PATCH 02/11] #21 manage module path --- contrib/misp-modules-loader.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index 9f2718467..e8afc72f8 100644 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -8,16 +8,18 @@ import json +def mod(path): + dirname = os.path.dirname(path) + name = os.path.basename(path).split('.')[0] + return [dirname, name] -# modules_path = "./misp-modules/misp_modules/modules/expansion" -# sys.path.append(modules_path) def run(argv): try: opts, args = getopt.getopt(argv, 'hp:i:r:', ["help","path=", "info=","run="]) except getopt.GetoptError as err: print(__file__ + " --info ") - print(__file__ + " --path --run ") + print(__file__ + " --run ") print(str(err)) sys.exit(2) @@ -31,14 +33,11 @@ def run(argv): print(__file__ + " --run ") sys.exit() - elif opt in ('-p', '--path'): - path = arg - sys.path.append(path) - elif opt in ('-r', '--run'): - module = arg + path = arg if path: - m = __import__(module) + sys.path.append(mod(path)[0]) + m = __import__(mod(path)[1]) data = json.load(sys.stdin) print(json.dumps(m.handler(json.dumps(data)))) From 5ad33a842b2aeaf01cfcbd6a2760b40f207f105f Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 12:39:46 +0200 Subject: [PATCH 03/11] #21 manage module path --- contrib/misp-modules-loader.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index e8afc72f8..0d868d447 100644 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -48,7 +48,14 @@ def run(argv): elif opt in ('-i','--info'): module = arg - print(module) + if path: + sys.path.append(mod(path)[0]) + m = __import__(mod(path)[1]) + + data = json.load(sys.stdin) + print(json.dumps(m.handler(json.dumps(data)))) + sys.exit(0) + m = __import__(module) print(({'name': module, 'mispattributes': m.mispattributes, 'moduleinfo':m.moduleinfo})) From c042bc6ff664445b08d94f7785f6f714aa0c321f Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 12:43:05 +0200 Subject: [PATCH 04/11] #21 manage module path --- contrib/misp-modules-loader.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index 0d868d447..48d3bb079 100644 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -16,7 +16,7 @@ def mod(path): def run(argv): try: - opts, args = getopt.getopt(argv, 'hp:i:r:', ["help","path=", "info=","run="]) + opts, args = getopt.getopt(argv, 'hp:i:r:', ["help", "info=","run="]) except getopt.GetoptError as err: print(__file__ + " --info ") print(__file__ + " --run ") @@ -47,17 +47,11 @@ def run(argv): print("add module path") elif opt in ('-i','--info'): - module = arg + path = arg if path: sys.path.append(mod(path)[0]) m = __import__(mod(path)[1]) - - data = json.load(sys.stdin) - print(json.dumps(m.handler(json.dumps(data)))) - sys.exit(0) - - m = __import__(module) - print(({'name': module, 'mispattributes': m.mispattributes, + print(({'name': module, 'mispattributes': m.mispattributes, 'moduleinfo':m.moduleinfo})) From 7196b821e522af11533b13c5cb512995b5e87c25 Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 14:48:45 +0200 Subject: [PATCH 05/11] #21 use misp_modules builtin functions to list and get information from modules (need to install misp_modules) --- contrib/misp-modules-loader.py | 40 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index 48d3bb079..d3bfdcc77 100644 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -7,16 +7,21 @@ import getopt import json +import misp_modules -def mod(path): - dirname = os.path.dirname(path) - name = os.path.basename(path).split('.')[0] - return [dirname, name] +""" +git clone https://github.com/MISP/misp-modules.git +cd misp_modules +pip3 -r REQUIREMENTS +pip3 install . +""" def run(argv): + + mhandlers, modules = misp_modules.load_package_modules() try: - opts, args = getopt.getopt(argv, 'hp:i:r:', ["help", "info=","run="]) + opts, args = getopt.getopt(argv, 'lh:i:r:', ["list", "help", "info=","run="]) except getopt.GetoptError as err: print(__file__ + " --info ") print(__file__ + " --run ") @@ -33,26 +38,21 @@ def run(argv): print(__file__ + " --run ") sys.exit() - elif opt in ('-r', '--run'): - path = arg - if path: - sys.path.append(mod(path)[0]) - m = __import__(mod(path)[1]) + elif opt in ('-l', '--list'): + print(modules) + sys.exit(0) + elif opt in ('-r', '--run'): + module = arg data = json.load(sys.stdin) - print(json.dumps(m.handler(json.dumps(data)))) + print(json.dumps(mhandlers[module].handler(json.dumps(data)))) sys.exit(0) - else: - print("add module path") - elif opt in ('-i','--info'): - path = arg - if path: - sys.path.append(mod(path)[0]) - m = __import__(mod(path)[1]) - print(({'name': module, 'mispattributes': m.mispattributes, - 'moduleinfo':m.moduleinfo})) + module = arg + + print(({'name': module, 'mispattributes': mhandlers[module].mispattributes, + 'moduleinfo':mhandlers[module].moduleinfo})) From 52bea41e2bf6f18fff7693e21a198dcd4742bf9d Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 15:04:07 +0200 Subject: [PATCH 06/11] #21 fix json issues --- contrib/misp-modules-loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index d3bfdcc77..8ffb19397 100644 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -39,7 +39,7 @@ def run(argv): sys.exit() elif opt in ('-l', '--list'): - print(modules) + print(json.dumps(modules)) sys.exit(0) elif opt in ('-r', '--run'): @@ -51,7 +51,7 @@ def run(argv): elif opt in ('-i','--info'): module = arg - print(({'name': module, 'mispattributes': mhandlers[module].mispattributes, + print(json.dumps({'name': module, 'mispattributes': mhandlers[module].mispattributes, 'moduleinfo':mhandlers[module].moduleinfo})) From fa9babda398d1249c764823fe60832558eb3fc76 Mon Sep 17 00:00:00 2001 From: Jerome Leonard Date: Wed, 3 May 2017 15:27:32 +0200 Subject: [PATCH 07/11] #21 return only expansion modules --- contrib/misp-modules-loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index 8ffb19397..fb0ed9e83 100644 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -39,8 +39,9 @@ def run(argv): sys.exit() elif opt in ('-l', '--list'): - print(json.dumps(modules)) - sys.exit(0) + modules = [m for m in modules if mhandlers['type:' + m ] == "expansion"] + print(json.dumps(modules)) + sys.exit(0) elif opt in ('-r', '--run'): module = arg From c32353434b2d99e187e4ed69a917c2026fc693cf Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 3 May 2017 12:16:30 +0200 Subject: [PATCH 08/11] #21 Add misp integration --- app/controllers/MispCtrl.scala | 29 +++++ app/models/Artifact.scala | 16 ++- app/models/MispModule.scala | 73 +++++++++++ app/services/Analyzer.scala | 34 +++-- app/services/Job.scala | 29 ++--- app/services/MispSrv.scala | 228 +++++++++++++++++++++++++++++++++ conf/routes | 4 + 7 files changed, 383 insertions(+), 30 deletions(-) create mode 100644 app/controllers/MispCtrl.scala create mode 100644 app/models/MispModule.scala create mode 100644 app/services/MispSrv.scala diff --git a/app/controllers/MispCtrl.scala b/app/controllers/MispCtrl.scala new file mode 100644 index 000000000..312708b65 --- /dev/null +++ b/app/controllers/MispCtrl.scala @@ -0,0 +1,29 @@ +package controllers + +import javax.inject.Inject + +import play.api.libs.json.{ JsObject, JsValue } +import play.api.mvc.{ Action, AnyContent, Controller } +import services.MispSrv + +import scala.concurrent.ExecutionContext + +class MispCtrl @Inject() (mispSrv: MispSrv, implicit val ec: ExecutionContext) extends Controller { + + def modules: Action[AnyContent] = Action { _ ⇒ + Ok(mispSrv.moduleList) + } + + def query: Action[JsValue] = Action.async(parse.json) { request ⇒ + val module = (request.body \ "module").asOpt[String].getOrElse(sys.error("module not present in request")) + val (mispType, dataJson) = request.body.as[JsObject].fields + .collectFirst { + case kv @ (k, _) if k != "module" ⇒ kv + } + .getOrElse(sys.error("invalid request")) + val data = dataJson.asOpt[String].getOrElse(sys.error("data has invalid type (expected string)")) + mispSrv.query(module, mispType, data) + .map(Ok(_)) + } +} + diff --git a/app/models/Artifact.scala b/app/models/Artifact.scala index c71a63439..b4a8f033e 100644 --- a/app/models/Artifact.scala +++ b/app/models/Artifact.scala @@ -1,23 +1,31 @@ package models -import play.api.libs.json.JsObject import java.io.File +import java.nio.file.Files + +import play.api.libs.json.JsObject abstract class Artifact(attributes: JsObject) { - def dataTypeFilter(filter: String): Boolean = (attributes \ "dataType").asOpt[String].fold(false)(_.toLowerCase.contains(filter.toLowerCase)) + def dataType: String = (attributes \ "dataType").asOpt[String].getOrElse("other") + def dataTypeFilter(filter: String): Boolean = dataType.toLowerCase.contains(filter.toLowerCase) def dataFilter(filter: String): Boolean = false } class FileArtifact(val data: File, val attributes: JsObject) extends Artifact(attributes) { - override def finalize { + override def finalize() { data.delete() } } object FileArtifact { - def apply(data: File, attributes: JsObject) = { + def apply(data: File, attributes: JsObject): FileArtifact = { val tempFile = File.createTempFile("cortex-", "-datafile") data.renameTo(tempFile) new FileArtifact(tempFile, attributes) } + def apply(data: Array[Byte], attributes: JsObject): FileArtifact = { + val tempFile = File.createTempFile("cortex-", "-datafile") + Files.write(tempFile.toPath, data) + new FileArtifact(tempFile, attributes) + } def unapply(fileArtifact: FileArtifact) = Some(fileArtifact.data → fileArtifact.attributes) } case class DataArtifact(data: String, attributes: JsObject) extends Artifact(attributes) { diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala new file mode 100644 index 000000000..4e01ec929 --- /dev/null +++ b/app/models/MispModule.scala @@ -0,0 +1,73 @@ +package models + +import java.io._ +import java.nio.file.Path +import java.nio.file.Files + +import org.apache.commons.codec.binary.{ Base64, Base64InputStream } +import play.api.libs.json.{ JsObject, JsValue, Json } +import services.MispSrv + +import scala.collection.JavaConverters._ +import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process.{ BasicIO, Process, ProcessIO } +import scala.sys.process._ + +case class MispModule( + name: String, + version: String, + description: String, + dataTypeList: Seq[String], + author: String, + modulePath: Path, + loaderCommand: String)(implicit val ec: ExecutionContext) extends Analyzer { + + val license = "AGPL-3.0" + val url = "https://github.com/MISP/misp-modules" + + private def stringStream(string: String): InputStream = { + new ByteArrayInputStream(string.getBytes) + } + def analyze(artifact: Artifact): Future[JsObject] = { + val input = artifact match { + case DataArtifact(data, _) ⇒ + stringStream(Json.obj(artifact.dataType → data).toString) + case FileArtifact(data, _) ⇒ + new SequenceInputStream(Iterator( + stringStream("""{"file":""""), + new Base64InputStream(new FileInputStream(data), true), + stringStream("\"}")).asJavaEnumeration) + } + val output = (s"$loaderCommand --run $modulePath" #< input).!! + Future { + Json.parse(output).as[JsObject] + } + } +} + +object MispModule { + def apply( + loaderCommand: String, + modulePath: Path, + mispSrv: MispSrv)(implicit ec: ExecutionContext): Option[MispModule] = { + val moduleInfo = Json.parse(s"$loaderCommand --info $modulePath".!!) + for { + name ← (moduleInfo \ "name").asOpt[String] + version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String] + description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String] + dataTypeList ← (moduleInfo \ "mispattributes" \ "input") + .asOpt[Seq[String]] + .map(_.map(mispSrv.mispType2dataType(_))) + author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String] + } yield MispModule(name, version, description, dataTypeList, author, modulePath, loaderCommand) + } +} + +//{'mispattributes': { +// 'input': ['ip-src', 'ip-dst', 'domain|ip'], +// 'output': ['hostname'] +// }, +// 'moduleinfo': { +// 'version': '0.1', +// 'author': 'Andreas Muehlemann', +// 'description': 'Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes', 'module-type': ['expansion', 'hover']}} \ No newline at end of file diff --git a/app/services/Analyzer.scala b/app/services/Analyzer.scala index c64eef8b2..d063747d9 100644 --- a/app/services/Analyzer.scala +++ b/app/services/Analyzer.scala @@ -2,37 +2,41 @@ package services import java.io.File import java.nio.file.{ Files, Path, Paths } - import javax.inject.Inject -import scala.collection.JavaConversions.iterableAsScalaIterable -import scala.concurrent.ExecutionContext - import akka.actor.ActorSystem - -import play.api.{ Configuration, Logger } +import models.{ Analyzer, ExternalAnalyzer, MispModule } import play.api.libs.json.{ JsObject, JsValue, Json } - -import models.{ Analyzer, ExternalAnalyzer } +import play.api.{ Configuration, Logger } import util.JsonConfig.configWrites + +import scala.collection.JavaConversions.iterableAsScalaIterable +import scala.concurrent.ExecutionContext import scala.util.Try class AnalyzerSrv( + mispSrv: MispSrv, analyzerPath: Path, analyzerConfig: JsObject, + mispModulesPath: Path, + mispModuleLoaderCommand: Option[String], akkaSystem: ActorSystem) { @Inject def this( + mispSrv: MispSrv, configuration: Configuration, akkaSystem: ActorSystem) = this( + mispSrv, Paths.get(configuration.getString("analyzer.path").getOrElse(".")), configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), + Paths.get(configuration.getString("misp.modules.path").getOrElse(".")), + configuration.getString("misp.modules.loader"), akkaSystem) lazy val log = Logger(getClass) lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") - lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers + private lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers def list: Seq[Analyzer] = externalAnalyzers // ::: javaAnalyzers def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) @@ -79,6 +83,18 @@ class AnalyzerSrv( } yield ExternalAnalyzer(name, version, description, dataTypeList, author, url, license, absoluteCommand, globalConfig deepMerge baseConfig deepMerge config)(analyzeExecutionContext) } + private[services] def getMispModules: Seq[Analyzer] = { + for { + moduleFile ← Try(Files.newDirectoryStream(mispModulesPath).toSeq).getOrElse { + log.warn(s"MISP modules directory ($mispModulesPath) is not found") + Nil + } + loaderCommand ← mispModuleLoaderCommand + if Files.isRegularFile(moduleFile) && moduleFile.toString.endsWith(".py") + mispModule ← MispModule(loaderCommand, moduleFile, mispSrv)(analyzeExecutionContext) + } yield mispModule + } + private[services] def readInfo(file: Path): JsValue = { val source = scala.io.Source.fromFile(file.toFile) try { diff --git a/app/services/Job.scala b/app/services/Job.scala index 1e8742da9..c66a7b455 100644 --- a/app/services/Job.scala +++ b/app/services/Job.scala @@ -1,31 +1,26 @@ package services import java.util.Date - import javax.inject.{ Inject, Named } -import scala.annotation.implicitNotFound -import scala.concurrent.{ ExecutionContext, Future, Promise } -import scala.concurrent.duration.{ DurationInt, FiniteDuration } -import scala.concurrent.duration.Duration -import scala.concurrent.duration.Duration.Infinite -import scala.util.Random - import akka.actor.{ Actor, ActorRef, ActorSystem, actorRef2Scala } import akka.pattern.ask import akka.util.Timeout - -import play.api.{ Configuration, Logger } +import models.{ Analyzer, Artifact, Job, JobStatus } import play.api.libs.json.{ JsString, JsValue } +import play.api.{ Configuration, Logger } -import models.{ Analyzer, Artifact, Job, JobStatus } +import scala.concurrent.duration.Duration.Infinite +import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration } +import scala.concurrent.{ ExecutionContext, Future, Promise } +import scala.util.Random class JobSrv @Inject() ( analyzerSrv: AnalyzerSrv, @Named("JobActor") jobActor: ActorRef, implicit val ec: ExecutionContext, implicit val system: ActorSystem) { - import JobActor._ + import services.JobActor._ implicit val timeout = Timeout(5.seconds) def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Future[(Int, Seq[Job])] = { @@ -71,8 +66,8 @@ class JobSrv @Inject() ( case _: Infinite ⇒ statusResult case duration: FiniteDuration ⇒ val prom = Promise[(JobStatus.Type, JsValue)]() - val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.Failure, JsString("Timeout"))); () } - statusResult onComplete { case _ ⇒ timeout.cancel() } + val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.InProgress, JsString("Timeout"))); () } + statusResult.onComplete(_ ⇒ timeout.cancel()) Future.firstCompletedOf(List(statusResult, prom.future)) } } @@ -95,7 +90,7 @@ class JobActor( analyzerSrv: AnalyzerSrv, implicit val ec: ExecutionContext) extends Actor { - import JobActor._ + import services.JobActor._ @Inject def this( configuration: Configuration, analyzerSrv: AnalyzerSrv, @@ -128,7 +123,7 @@ class JobActor( dataTypeFilter.fold(true)(j.artifact.dataTypeFilter) && dataFilter.fold(true)(j.artifact.dataFilter) && analyzerFilter.fold(true)(j.analyzerId.contains)) - sender ! JobList(filteredJobs.size, filteredJobs.drop(start).take(limit)) + sender ! JobList(filteredJobs.size, filteredJobs.slice(start, start + limit)) case GetJob(jobId) ⇒ sender ! jobs.find(_.id == jobId).getOrElse(JobNotFound) case RemoveJob(jobId) ⇒ removeJob(jobs, jobId) match { @@ -148,5 +143,5 @@ class JobActor( context.become(jobState(jobs.takeWhile(_.date after limitDate))) } - override def receive = jobState(Nil) + override def receive: Receive = jobState(Nil) } \ No newline at end of file diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala new file mode 100644 index 000000000..869c8c13a --- /dev/null +++ b/app/services/MispSrv.scala @@ -0,0 +1,228 @@ +package services + +import javax.inject.Inject + +import models.JsonFormat.dataActifactReads +import models.{ DataArtifact, FileArtifact } +import org.apache.commons.codec.binary.Base64 +import play.api.Logger +import play.api.libs.json.{ JsArray, JsObject, JsValue, Json } + +import scala.concurrent.{ ExecutionContext, Future } + +class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { + private[MispSrv] lazy val logger = Logger(getClass) + + def moduleList: JsValue = { + JsArray(analyzerSrv.list.map { analyzer ⇒ + Json.obj( + "name" → analyzer.id, + "type" → "expansion", + "mispattributes" → Json.obj( + "input" → analyzer.dataTypeList.flatMap(dataType2mispType).distinct, + "output" → Json.arr()), + "meta" → Json.obj( + "module-type" → Json.arr("expansion"), + "description" → analyzer.description, + "author" → analyzer.author, + "version" → analyzer.version, + "config" → Json.arr())) + }) + } + + def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = { + analyzerSrv.get(module).map { analyzer ⇒ + val artifact = mispType2dataType(mispType) match { + case "file" if mispType == "malware-sample" ⇒ ??? + case "file" ⇒ FileArtifact(Base64.decodeBase64(data), Json.obj( + "tlp" → 1, + "dataType" → "file")) + case dataType ⇒ DataArtifact(data, Json.obj( + "tlp" → 1, + "dataType" → dataType)) + } + + analyzer + .analyze(artifact) + .map { output ⇒ + logger.info(s"analyzer output:\n$output") + val success = (output \ "success") + .asOpt[Boolean] + .getOrElse(false) + if (success) { + Json.obj( + "results" → (output \ "artifacts") + .asOpt[Seq[JsObject]] + .getOrElse(Nil) + .map { artifact ⇒ + Json.obj( + "types" → dataType2mispType((artifact \ "type").as[String]), + "values" → Json.arr((artifact \ "value").as[String])) + }) + } + else { + val message = (output \ "error").asOpt[String].getOrElse(output.toString) + Json.obj( + "error" → message) + } + } + } + .getOrElse(Future.failed(new Exception(s"Module $module not found"))) + } + + def mispType2dataType(mispType: String): String = typeLookup.getOrElse(mispType, { + logger.warn(s"Misp type $mispType not recognized") + "other" + }) + + def dataType2mispType(dataType: String): Seq[String] = { + val mispTypes = typeLookup.filter(_._2 == dataType) + .keys + .toSeq + .distinct + + if (mispTypes.isEmpty) { + logger.warn(s"Data type $dataType not recognized") + Seq("other") + } + else mispTypes + } + + private val typeLookup = Map( + "md5" → "hash", + "sha1" → "hash", + "sha256" → "hash", + "filename" → "filename", + "pdb" → "other", + "filename|md5" → "other", + "filename|sha1" → "other", + "filename|sha256" → "other", + "ip-src" → "ip", + "ip-dst" → "ip", + "hostname" → "fqdn", + "domain" → "domain", + "domain|ip" → "other", + "email-src" → "mail", + "email-dst" → "mail", + "email-subject" → "mail_subject", + "email-attachment" → "other", + "float" → "other", + "url" → "url", + "http-method" → "other", + "user-agent" → "user-agent", + "regkey" → "registry", + "regkey|value" → "registry", + "AS" → "other", + "snort" → "other", + "pattern-in-file" → "other", + "pattern-in-traffic" → "other", + "pattern-in-memory" → "other", + "yara" → "other", + "sigma" → "other", + "vulnerability" → "other", + "attachment" → "file", + "malware-sample" → "file", + "link" → "other", + "comment" → "other", + "text" → "other", + "hex" → "other", + "other" → "other", + "named" → "other", + "mutex" → "other", + "target-user" → "other", + "target-email" → "mail", + "target-machine" → "hostname", + "target-org" → "other", + "target-location" → "other", + "target-external" → "other", + "btc" → "other", + "iban" → "other", + "bic" → "other", + "bank-account-nr" → "other", + "aba-rtn" → "other", + "bin" → "other", + "cc-number" → "other", + "prtn" → "other", + "threat-actor" → "other", + "campaign-name" → "other", + "campaign-id" → "other", + "malware-type" → "other", + "uri" → "uri_path", + "authentihash" → "other", + "ssdeep" → "hash", + "imphash" → "hash", + "pehash" → "hash", + "impfuzzy" → "hash", + "sha224" → "hash", + "sha384" → "hash", + "sha512" → "hash", + "sha512/224" → "hash", + "sha512/256" → "hash", + "tlsh" → "other", + "filename|authentihash" → "other", + "filename|ssdeep" → "other", + "filename|imphash" → "other", + "filename|impfuzzy" → "other", + "filename|pehash" → "other", + "filename|sha224" → "other", + "filename|sha384" → "other", + "filename|sha512" → "other", + "filename|sha512/224" → "other", + "filename|sha512/256" → "other", + "filename|tlsh" → "other", + "windows-scheduled-task" → "other", + "windows-service-name" → "other", + "windows-service-displayname" → "other", + "whois-registrant-email" → "mail", + "whois-registrant-phone" → "other", + "whois-registrant-name" → "other", + "whois-registrar" → "other", + "whois-creation-date" → "other", + "x509-fingerprint-sha1" → "other", + "dns-soa-email" → "other", + "size-in-bytes" → "other", + "counter" → "other", + "datetime" → "other", + "cpe" → "other", + "port" → "other", + "ip-dst|port" → "other", + "ip-src|port" → "other", + "hostname|port" → "other", + "email-dst-display-name" → "other", + "email-src-display-name" → "other", + "email-header" → "other", + "email-reply-to" → "other", + "email-x-mailer" → "other", + "email-mime-boundary" → "other", + "email-thread-index" → "other", + "email-message-id" → "other", + "github-username" → "other", + "github-repository" → "other", + "github-organisation" → "other", + "jabber-id" → "other", + "twitter-id" → "other", + "first-name" → "other", + "middle-name" → "other", + "last-name" → "other", + "date-of-birth" → "other", + "place-of-birth" → "other", + "gender" → "other", + "passport-number" → "other", + "passport-country" → "other", + "passport-expiration" → "other", + "redress-number" → "other", + "nationality" → "other", + "visa-number" → "other", + "issue-date-of-the-visa" → "other", + "primary-residence" → "other", + "country-of-residence" → "other", + "special-service-request" → "other", + "frequent-flyer-number" → "other", + "travel-details" → "other", + "payment-details" → "other", + "place-port-of-original-embarkation" → "other", + "place-port-of-clearance" → "other", + "place-port-of-onward-foreign-destination" → "other", + "passenger-name-record-locator-number" → "other", + "mobile-application-id" → "other") +} \ No newline at end of file diff --git a/conf/routes b/conf/routes index 00bf1381f..0b44c2cc4 100644 --- a/conf/routes +++ b/conf/routes @@ -2,6 +2,7 @@ # This file defines all application routes (Higher priority routes first) # ~~~~ + GET / controllers.Default.redirect(to = "/index.html") GET /api/analyzer controllers.AnalyzerCtrl.list GET /api/analyzer/:id controllers.AnalyzerCtrl.get(id) @@ -12,4 +13,7 @@ GET /api/job/:id controllers.JobCtrl.get(id) DELETE /api/job/:id controllers.JobCtrl.remove(id) GET /api/job/:id/report controllers.JobCtrl.report(id) GET /api/job/:id/waitreport controllers.JobCtrl.waitReport(id, atMost ?= "Inf") +GET /modules controllers.MispCtrl.modules +POST /query controllers.MispCtrl.query GET /*file controllers.AssetCtrl.get(file) + From f68b41a80bffc3eda736fb4f7c9698f371b18eee Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 3 May 2017 12:28:56 +0200 Subject: [PATCH 09/11] #21 Fix cyclic dependency, rename logger, make misp-modules-loader executable --- app/services/Analyzer.scala | 36 ++++++++++++++++++---------------- contrib/misp-modules-loader.py | 0 2 files changed, 19 insertions(+), 17 deletions(-) mode change 100644 => 100755 contrib/misp-modules-loader.py diff --git a/app/services/Analyzer.scala b/app/services/Analyzer.scala index d063747d9..6e9bb5518 100644 --- a/app/services/Analyzer.scala +++ b/app/services/Analyzer.scala @@ -2,7 +2,7 @@ package services import java.io.File import java.nio.file.{ Files, Path, Paths } -import javax.inject.Inject +import javax.inject.{ Inject, Provider } import akka.actor.ActorSystem import models.{ Analyzer, ExternalAnalyzer, MispModule } @@ -15,29 +15,31 @@ import scala.concurrent.ExecutionContext import scala.util.Try class AnalyzerSrv( - mispSrv: MispSrv, + mispSrvProvider: Provider[MispSrv], analyzerPath: Path, analyzerConfig: JsObject, mispModulesPath: Path, mispModuleLoaderCommand: Option[String], akkaSystem: ActorSystem) { @Inject def this( - mispSrv: MispSrv, + mispSrvProvider: Provider[MispSrv], configuration: Configuration, akkaSystem: ActorSystem) = this( - mispSrv, + mispSrvProvider, Paths.get(configuration.getString("analyzer.path").getOrElse(".")), configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), Paths.get(configuration.getString("misp.modules.path").getOrElse(".")), configuration.getString("misp.modules.loader"), akkaSystem) - lazy val log = Logger(getClass) + private[AnalyzerSrv] lazy val logger = Logger(getClass) lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") + lazy val mispSrv = mispSrvProvider.get private lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers - def list: Seq[Analyzer] = externalAnalyzers // ::: javaAnalyzers + private lazy val mispModules: Seq[Analyzer] = getMispModules + def list: Seq[Analyzer] = externalAnalyzers ++ mispModules // ::: javaAnalyzers def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) @@ -45,7 +47,7 @@ class AnalyzerSrv( val globalConfig = (analyzerConfig \ "global").asOpt[JsObject].getOrElse(JsObject(Nil)) for { analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).toSeq).getOrElse { - log.warn(s"Analyzer directory ($analyzerPath) is not found") + logger.warn(s"Analyzer directory ($analyzerPath) is not found") Nil } if Files.isDirectory(analyzerDir) @@ -53,40 +55,40 @@ class AnalyzerSrv( if Files.isReadable(infoFile) info = readInfo(infoFile) name ← (info \ "name").asOpt[String] orElse { - log.warn(s"name is missing in $infoFile"); None + logger.warn(s"name is missing in $infoFile"); None } version ← (info \ "version").asOpt[String] orElse { - log.warn(s"version is missing in $infoFile"); None + logger.warn(s"version is missing in $infoFile"); None } description ← (info \ "description").asOpt[String] orElse { - log.warn(s"description is missing in $infoFile"); None + logger.warn(s"description is missing in $infoFile"); None } dataTypeList ← (info \ "dataTypeList").asOpt[Seq[String]] orElse { - log.warn(s"dataTypeList is missing in $infoFile"); None + logger.warn(s"dataTypeList is missing in $infoFile"); None } command ← (info \ "command").asOpt[String] orElse { - log.warn(s"command is missing in $infoFile"); None + logger.warn(s"command is missing in $infoFile"); None } author ← (info \ "author").asOpt[String] orElse { - log.warn(s"author is missing in $infoFile"); None + logger.warn(s"author is missing in $infoFile"); None } url ← (info \ "url").asOpt[String] orElse { - log.warn(s"url is missing in $infoFile"); None + logger.warn(s"url is missing in $infoFile"); None } license ← (info \ "license").asOpt[String] orElse { - log.warn(s"license is missing in $infoFile"); None + logger.warn(s"license is missing in $infoFile"); None } config = (info \ "config").asOpt[JsObject].getOrElse(JsObject(Nil)) baseConfig = (info \ "baseConfig").asOpt[String].flatMap(c ⇒ (analyzerConfig \ c).asOpt[JsObject]).getOrElse(JsObject(Nil)) absoluteCommand = analyzerPath.resolve(Paths.get(command.replaceAll("[\\/]", File.separator))) - _ = log.info(s"Register analyzer $name $version (${(name + "_" + version).replaceAll("\\.", "_")})") + _ = logger.info(s"Register analyzer $name $version (${(name + "_" + version).replaceAll("\\.", "_")})") } yield ExternalAnalyzer(name, version, description, dataTypeList, author, url, license, absoluteCommand, globalConfig deepMerge baseConfig deepMerge config)(analyzeExecutionContext) } private[services] def getMispModules: Seq[Analyzer] = { for { moduleFile ← Try(Files.newDirectoryStream(mispModulesPath).toSeq).getOrElse { - log.warn(s"MISP modules directory ($mispModulesPath) is not found") + logger.warn(s"MISP modules directory ($mispModulesPath) is not found") Nil } loaderCommand ← mispModuleLoaderCommand diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py old mode 100644 new mode 100755 From a742623457678134ba658be86b185789f7adb1a1 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 3 May 2017 15:27:28 +0200 Subject: [PATCH 10/11] #Fix Json format for MISP modules --- app/controllers/StatusCtrl.scala | 25 ++++++++++ app/models/MispModule.scala | 80 +++++++++++++++++++++----------- app/services/Analyzer.scala | 16 +++---- app/services/MispSrv.scala | 11 +++-- conf/routes | 1 + 5 files changed, 93 insertions(+), 40 deletions(-) create mode 100644 app/controllers/StatusCtrl.scala diff --git a/app/controllers/StatusCtrl.scala b/app/controllers/StatusCtrl.scala new file mode 100644 index 000000000..7f0308283 --- /dev/null +++ b/app/controllers/StatusCtrl.scala @@ -0,0 +1,25 @@ +package controllers + +import javax.inject.{ Inject, Singleton } + +import play.api.Configuration +import play.api.libs.json.Json +import play.api.libs.json.Json.toJsFieldJsValueWrapper +import play.api.mvc.{ Action, Controller } + +@Singleton +class StatusCtrl @Inject() ( + configuration: Configuration) extends Controller { + + private[controllers] def getVersion(c: Class[_]) = Option(c.getPackage.getImplementationVersion).getOrElse("SNAPSHOT") + + def get = Action { _ ⇒ + Ok(Json.obj( + "versions" → Json.obj( + "Cortex" → getVersion(classOf[models.Artifact]), + "Play" → getVersion(classOf[Controller])), + "config" → Json.obj( + "authType" → "none", + "capabilities" → Json.arr()))) + } +} diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala index 4e01ec929..a715abdfd 100644 --- a/app/models/MispModule.scala +++ b/app/models/MispModule.scala @@ -1,25 +1,25 @@ package models import java.io._ -import java.nio.file.Path -import java.nio.file.Files -import org.apache.commons.codec.binary.{ Base64, Base64InputStream } -import play.api.libs.json.{ JsObject, JsValue, Json } +import org.apache.commons.codec.binary.Base64InputStream +import play.api.Logger +import play.api.libs.json.{ JsObject, Json } import services.MispSrv import scala.collection.JavaConverters._ import scala.concurrent.{ ExecutionContext, Future } -import scala.sys.process.{ BasicIO, Process, ProcessIO } import scala.sys.process._ +import scala.util.{ Failure, Success, Try } case class MispModule( - name: String, + mispSrv: MispSrv, + name: String, version: String, description: String, dataTypeList: Seq[String], author: String, - modulePath: Path, + moduleName: String, loaderCommand: String)(implicit val ec: ExecutionContext) extends Analyzer { val license = "AGPL-3.0" @@ -31,14 +31,14 @@ case class MispModule( def analyze(artifact: Artifact): Future[JsObject] = { val input = artifact match { case DataArtifact(data, _) ⇒ - stringStream(Json.obj(artifact.dataType → data).toString) + stringStream(Json.obj(mispSrv.dataType2mispType(artifact.dataType).head → data).toString) case FileArtifact(data, _) ⇒ new SequenceInputStream(Iterator( - stringStream("""{"file":""""), + stringStream("""{"attachment":""""), new Base64InputStream(new FileInputStream(data), true), stringStream("\"}")).asJavaEnumeration) } - val output = (s"$loaderCommand --run $modulePath" #< input).!! + val output = (s"$loaderCommand --run $moduleName" #< input).!! Future { Json.parse(output).as[JsObject] } @@ -46,28 +46,52 @@ case class MispModule( } object MispModule { + private[MispModule] lazy val logger = Logger(getClass) + def list(loaderCommand: String): Seq[String] = + Json.parse(s"$loaderCommand --list".!!).as[Seq[String]] + def apply( loaderCommand: String, - modulePath: Path, + moduleName: String, mispSrv: MispSrv)(implicit ec: ExecutionContext): Option[MispModule] = { - val moduleInfo = Json.parse(s"$loaderCommand --info $modulePath".!!) + println(s"Loading MISP module $moduleName") for { - name ← (moduleInfo \ "name").asOpt[String] - version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String] - description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String] + moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) match { + case Success(s) ⇒ Some(s) + case Failure(f) ⇒ + f.printStackTrace() + None + } + name ← (moduleInfo \ "name").asOpt[String].orElse { + println("name not defined") + None + } + version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String].orElse { + println("version not defined") + None + } + description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String].orElse { + println("description not defined") + None + } dataTypeList ← (moduleInfo \ "mispattributes" \ "input") .asOpt[Seq[String]] - .map(_.map(mispSrv.mispType2dataType(_))) - author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String] - } yield MispModule(name, version, description, dataTypeList, author, modulePath, loaderCommand) + .map(_.map(mispSrv.mispType2dataType(_)).distinct) + .orElse { + println("input attributes not defined") + None + } + author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String].orElse { + println("author not defined") + None + } + mispModule ← Try(MispModule(mispSrv, name, version, description, dataTypeList, author, moduleName, loaderCommand)) match { + case Success(s) ⇒ Some(s) + case Failure(f) ⇒ + f.printStackTrace() + sys.error("Load module fails") + } + _ = println("Module load succeed") + } yield mispModule } -} - -//{'mispattributes': { -// 'input': ['ip-src', 'ip-dst', 'domain|ip'], -// 'output': ['hostname'] -// }, -// 'moduleinfo': { -// 'version': '0.1', -// 'author': 'Andreas Muehlemann', -// 'description': 'Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes', 'module-type': ['expansion', 'hover']}} \ No newline at end of file +} \ No newline at end of file diff --git a/app/services/Analyzer.scala b/app/services/Analyzer.scala index 6e9bb5518..b69c56cd6 100644 --- a/app/services/Analyzer.scala +++ b/app/services/Analyzer.scala @@ -2,7 +2,7 @@ package services import java.io.File import java.nio.file.{ Files, Path, Paths } -import javax.inject.{ Inject, Provider } +import javax.inject.{ Inject, Provider, Singleton } import akka.actor.ActorSystem import models.{ Analyzer, ExternalAnalyzer, MispModule } @@ -14,6 +14,7 @@ import scala.collection.JavaConversions.iterableAsScalaIterable import scala.concurrent.ExecutionContext import scala.util.Try +@Singleton class AnalyzerSrv( mispSrvProvider: Provider[MispSrv], analyzerPath: Path, @@ -87,13 +88,12 @@ class AnalyzerSrv( private[services] def getMispModules: Seq[Analyzer] = { for { - moduleFile ← Try(Files.newDirectoryStream(mispModulesPath).toSeq).getOrElse { - logger.warn(s"MISP modules directory ($mispModulesPath) is not found") - Nil - } - loaderCommand ← mispModuleLoaderCommand - if Files.isRegularFile(moduleFile) && moduleFile.toString.endsWith(".py") - mispModule ← MispModule(loaderCommand, moduleFile, mispSrv)(analyzeExecutionContext) + loaderCommand ← mispModuleLoaderCommand.toSeq + moduleName ← MispModule.list(loaderCommand) + + _ = println("MISP module loading ...") + mispModule ← MispModule(loaderCommand, moduleName, mispSrv)(analyzeExecutionContext) + _ = println("MISP module load success") } yield mispModule } diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala index 869c8c13a..e11253e35 100644 --- a/app/services/MispSrv.scala +++ b/app/services/MispSrv.scala @@ -17,12 +17,12 @@ class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { JsArray(analyzerSrv.list.map { analyzer ⇒ Json.obj( "name" → analyzer.id, - "type" → "expansion", + "type" → "cortex", "mispattributes" → Json.obj( "input" → analyzer.dataTypeList.flatMap(dataType2mispType).distinct, "output" → Json.arr()), "meta" → Json.obj( - "module-type" → Json.arr("expansion"), + "module-type" → Json.arr("cortex"), "description" → analyzer.description, "author" → analyzer.author, "version" → analyzer.version, @@ -51,14 +51,17 @@ class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { .getOrElse(false) if (success) { Json.obj( - "results" → (output \ "artifacts") + "results" → ((output \ "artifacts") .asOpt[Seq[JsObject]] .getOrElse(Nil) .map { artifact ⇒ Json.obj( "types" → dataType2mispType((artifact \ "type").as[String]), "values" → Json.arr((artifact \ "value").as[String])) - }) + } + :+ Json.obj( + "types" → Json.arr("cortex"), + "values" → Json.arr(output.toString)))) } else { val message = (output \ "error").asOpt[String].getOrElse(output.toString) diff --git a/conf/routes b/conf/routes index 0b44c2cc4..5685860b8 100644 --- a/conf/routes +++ b/conf/routes @@ -4,6 +4,7 @@ GET / controllers.Default.redirect(to = "/index.html") +GET /api/status controllers.StatusCtrl.get GET /api/analyzer controllers.AnalyzerCtrl.list GET /api/analyzer/:id controllers.AnalyzerCtrl.get(id) POST /api/analyzer/:id/run controllers.AnalyzerCtrl.analyze(id) From f84342fdd7cc65b101b11f9403bd1573223122ac Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 5 May 2017 13:29:48 +0200 Subject: [PATCH 11/11] #21 Restructure classes --- app/Module.scala | 11 +- .../{Analyzer.scala => AnalyzerCtrl.scala} | 17 +- app/controllers/Asset.scala | 6 +- app/controllers/{Job.scala => JobCtrl.scala} | 30 +-- app/controllers/MispCtrl.scala | 2 + app/models/Analyzer.scala | 6 +- app/models/ExternalAnalyzer.scala | 84 +------ app/models/Job.scala | 20 +- app/models/JsonFormat.scala | 79 +++++-- app/models/MispModule.scala | 93 +------- app/models/Report.scala | 9 + app/services/Analyzer.scala | 107 --------- app/services/AnalyzerSrv.scala | 37 +++ app/services/ExternalAnalyzerSrv.scala | 142 ++++++++++++ app/services/{Job.scala => JobSrv.scala} | 70 +++--- app/services/MispSrv.scala | 219 ++++++++++++++---- app/util/JsonConfig.scala | 14 +- build.sbt | 11 +- contrib/misp-modules-loader.py | 2 +- project/Bintray.scala | 20 +- project/FrontEnd.scala | 2 +- project/Release.scala | 7 +- 22 files changed, 537 insertions(+), 451 deletions(-) rename app/controllers/{Analyzer.scala => AnalyzerCtrl.scala} (89%) rename app/controllers/{Job.scala => JobCtrl.scala} (54%) create mode 100644 app/models/Report.scala delete mode 100644 app/services/Analyzer.scala create mode 100644 app/services/AnalyzerSrv.scala create mode 100644 app/services/ExternalAnalyzerSrv.scala rename app/services/{Job.scala => JobSrv.scala} (73%) diff --git a/app/Module.scala b/app/Module.scala index 5a042f5ef..3dacb8fda 100644 --- a/app/Module.scala +++ b/app/Module.scala @@ -1,16 +1,13 @@ -import play.api.{ Configuration, Environment, Mode } -import play.api.libs.concurrent.AkkaGuiceSupport - import com.google.inject.AbstractModule - -import net.codingwell.scalaguice.ScalaModule - import controllers.{ AssetCtrl, AssetCtrlDev, AssetCtrlProd } +import net.codingwell.scalaguice.ScalaModule +import play.api.libs.concurrent.AkkaGuiceSupport +import play.api.{ Configuration, Environment, Mode } import services.JobActor class Module(environment: Environment, configuration: Configuration) extends AbstractModule with ScalaModule with AkkaGuiceSupport { - override def configure() = { + override def configure(): Unit = { bindActor[JobActor]("JobActor") if (environment.mode == Mode.Prod) diff --git a/app/controllers/Analyzer.scala b/app/controllers/AnalyzerCtrl.scala similarity index 89% rename from app/controllers/Analyzer.scala rename to app/controllers/AnalyzerCtrl.scala index 644571788..8464291bf 100644 --- a/app/controllers/Analyzer.scala +++ b/app/controllers/AnalyzerCtrl.scala @@ -2,16 +2,14 @@ package controllers import javax.inject.Inject -import scala.annotation.implicitNotFound -import scala.concurrent.{ ExecutionContext, Future } - +import models.JsonFormat.{ analyzerWrites, dataActifactReads, jobWrites } +import models.{ DataArtifact, FileArtifact } import play.api.libs.json.{ JsObject, JsString, Json } import play.api.mvc.{ Action, AnyContent, Controller, Request } - -import models.{ DataArtifact, FileArtifact } -import models.JsonFormat.{ analyzerWrites, dataActifactReads, jobWrites } import services.{ AnalyzerSrv, JobSrv } +import scala.concurrent.{ ExecutionContext, Future } + class AnalyzerCtrl @Inject() ( analyzerSrv: AnalyzerSrv, jobSrv: JobSrv, @@ -38,7 +36,7 @@ class AnalyzerCtrl @Inject() ( private[controllers] def readFileArtifact(request: Request[AnyContent]) = { for { parts ← request.body.asMultipartFormData - filePart ← parts.file("data").headOption + filePart ← parts.file("data") attrList ← parts.dataParts.get("_json") attrStr ← attrList.headOption attr ← Json.parse(attrStr).asOpt[JsObject] @@ -47,11 +45,12 @@ class AnalyzerCtrl @Inject() ( ("filename" → JsString(filePart.filename))) } - def analyze(analyzerId: String) = Action.async { request ⇒ + def analyze(analyzerId: String): Action[AnyContent] = Action.async { request ⇒ readDataArtifact(request) .orElse(readFileArtifact(request)) .map { artifact ⇒ - jobSrv.create(artifact, analyzerId) + analyzerSrv.analyze(analyzerId, artifact) + //jobSrv.create(artifact, analyzerId) .map(j ⇒ Ok(Json.toJson(j))) } .getOrElse(Future.successful(BadRequest("???"))) diff --git a/app/controllers/Asset.scala b/app/controllers/Asset.scala index 102cd11ab..6a37553ec 100644 --- a/app/controllers/Asset.scala +++ b/app/controllers/Asset.scala @@ -4,7 +4,7 @@ import javax.inject.{ Inject, Singleton } import play.api.Environment import play.api.http.HttpErrorHandler -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc.{ Action, AnyContent } trait AssetCtrl { def get(file: String): Action[AnyContent] @@ -12,12 +12,12 @@ trait AssetCtrl { @Singleton class AssetCtrlProd @Inject() (errorHandler: HttpErrorHandler) extends Assets(errorHandler) with AssetCtrl { - def get(file: String) = at("/ui", file) + def get(file: String): Action[AnyContent] = at("/ui", file) } @Singleton class AssetCtrlDev @Inject() (environment: Environment) extends ExternalAssets(environment) with AssetCtrl { - def get(file: String) = { + def get(file: String): Action[AnyContent] = { if (file.startsWith("bower_components/")) { at("ui", file) } diff --git a/app/controllers/Job.scala b/app/controllers/JobCtrl.scala similarity index 54% rename from app/controllers/Job.scala rename to app/controllers/JobCtrl.scala index 2a96ca0a1..af20f5395 100644 --- a/app/controllers/Job.scala +++ b/app/controllers/JobCtrl.scala @@ -6,40 +6,38 @@ import scala.annotation.implicitNotFound import scala.concurrent.ExecutionContext import scala.concurrent.duration.Duration import scala.util.{ Failure, Success } - import play.api.libs.json.{ JsString, Json } -import play.api.mvc.{ Action, Controller } - -import models.JsonFormat.{ jobStatusWrites, jobWrites } +import play.api.mvc.{ Action, AnyContent, Controller } +import models.JsonFormat._ import services.JobSrv class JobCtrl @Inject() ( jobSrv: JobSrv, implicit val ec: ExecutionContext) extends Controller { - def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int) = Action.async { request ⇒ + def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Action[AnyContent] = Action.async { request ⇒ jobSrv.list(dataTypeFilter, dataFilter, analyzerFilter, start, limit).map { case (total, jobs) ⇒ Ok(Json.toJson(jobs)).withHeaders("X-Total" → total.toString) } } - def get(jobId: String) = Action.async { request ⇒ + def get(jobId: String): Action[AnyContent] = Action.async { request ⇒ jobSrv.get(jobId).map { job ⇒ Ok(Json.toJson(job)) } } - def remove(jobId: String) = Action.async { request ⇒ + def remove(jobId: String): Action[AnyContent] = Action.async { request ⇒ jobSrv.remove(jobId).map(_ ⇒ Ok("")) } - def report(jobId: String) = Action.async { request ⇒ + def report(jobId: String): Action[AnyContent] = Action.async { request ⇒ jobSrv .get(jobId) .map { job ⇒ val report = job.report.value match { - case Some(Success(report)) ⇒ report - case Some(Failure(error)) ⇒ JsString(error.getMessage) - case None ⇒ JsString("Running") + case Some(Success(r)) ⇒ Json.toJson(r) + case Some(Failure(error)) ⇒ JsString(error.getMessage) + case None ⇒ JsString("Running") } Ok(jobWrites.writes(job) + ("status" → jobStatusWrites.writes(job.status)) + @@ -47,12 +45,8 @@ class JobCtrl @Inject() ( } } - def waitReport(jobId: String, atMost: String) = Action.async { request ⇒ - for { - job ← jobSrv.get(jobId) - (status, report) ← jobSrv.waitReport(jobId, Duration(atMost)) - } yield Ok(jobWrites.writes(job) + - ("status" → jobStatusWrites.writes(job.status)) + - ("report" → report)) + def waitReport(jobId: String, atMost: String): Action[AnyContent] = Action.async { request ⇒ + jobSrv.waitReport(jobId, Duration(atMost)) + .map { job ⇒ Ok(Json.toJson(job)) } } } \ No newline at end of file diff --git a/app/controllers/MispCtrl.scala b/app/controllers/MispCtrl.scala index 312708b65..8792921e5 100644 --- a/app/controllers/MispCtrl.scala +++ b/app/controllers/MispCtrl.scala @@ -2,6 +2,7 @@ package controllers import javax.inject.Inject +import play.api.Logger import play.api.libs.json.{ JsObject, JsValue } import play.api.mvc.{ Action, AnyContent, Controller } import services.MispSrv @@ -10,6 +11,7 @@ import scala.concurrent.ExecutionContext class MispCtrl @Inject() (mispSrv: MispSrv, implicit val ec: ExecutionContext) extends Controller { + private[MispCtrl] lazy val logger = Logger(getClass) def modules: Action[AnyContent] = Action { _ ⇒ Ok(mispSrv.moduleList) } diff --git a/app/models/Analyzer.scala b/app/models/Analyzer.scala index ce853ac97..d13381f3b 100644 --- a/app/models/Analyzer.scala +++ b/app/models/Analyzer.scala @@ -1,10 +1,6 @@ package models -import scala.concurrent.Future -import play.api.libs.json.JsObject - abstract class Analyzer { - def analyze(artifact: Artifact): Future[JsObject] val name: String val version: String val description: String @@ -12,5 +8,5 @@ abstract class Analyzer { val author: String val url: String val license: String - val id = (name + "_" + version).replaceAll("\\.", "_") + val id: String = (name + "_" + version).replaceAll("\\.", "_") } diff --git a/app/models/ExternalAnalyzer.scala b/app/models/ExternalAnalyzer.scala index 2ce882b5a..97e2556d5 100644 --- a/app/models/ExternalAnalyzer.scala +++ b/app/models/ExternalAnalyzer.scala @@ -1,80 +1,16 @@ package models -import java.io.{ BufferedReader, InputStreamReader } import java.nio.file.Path -import scala.concurrent.{ ExecutionContext, Future, blocking } -import scala.sys.process.{ BasicIO, Process, ProcessIO } - -import akka.stream.Materializer - -import play.api.Logger -import play.api.libs.json.{ JsObject, JsString, Json } - -import com.fasterxml.jackson.core.JsonParseException -import com.fasterxml.jackson.databind.JsonMappingException +import play.api.libs.json.JsObject case class ExternalAnalyzer( - name: String, - version: String, - description: String, - dataTypeList: Seq[String], - author: String, - url: String, - license: String, - command: Path, - config: JsObject)(implicit val ec: ExecutionContext) extends Analyzer { - - val log = Logger(getClass) - private val osexec = if (System.getProperty("os.name").toLowerCase.contains("win")) - (c: String) ⇒ s"""cmd /c $c""" - else - (c: String) ⇒ s"""sh -c "./$c" """ - - override def analyze(artifact: Artifact): Future[JsObject] = { - Future { - val input = artifact match { - case FileArtifact(file, attributes) ⇒ attributes + ("file" → JsString(file.getAbsoluteFile.toString)) + ("config" → config) - case DataArtifact(data, attributes) ⇒ attributes + ("data" → JsString(data)) + ("config" → config) - } - val output = new StringBuffer - val error = new StringBuffer - try { - log.info(s"Execute ${osexec(command.getFileName.toString)} in ${command.getParent.toFile.getAbsoluteFile.getName}") - val exitValue = Process(osexec(command.getFileName.toString), command.getParent.toFile).run( - new ProcessIO( - { stdin ⇒ - try stdin.write(input.toString.getBytes("UTF-8")) - finally stdin.close() - }, - { stdout ⇒ - val reader = new BufferedReader(new InputStreamReader(stdout, "UTF-8")) - try BasicIO.processLinesFully { line ⇒ - output.append(line).append(System.lineSeparator()) - () - }(reader.readLine) - finally reader.close() - }, - { stderr ⇒ - val reader = new BufferedReader(new InputStreamReader(stderr, "UTF-8")) - try BasicIO.processLinesFully { line ⇒ - error.append(line).append(System.lineSeparator()) - () - }(reader.readLine) - finally reader.close() - })).exitValue - Json.parse(output.toString).as[JsObject] - } - catch { - case _: JsonMappingException ⇒ - error.append(output) - JsObject(Seq("errorMessage" → JsString(s"Error: Invalid output\n$error"))) - case _: JsonParseException ⇒ - error.append(output) - JsObject(Seq("errorMessage" → JsString(s"Error: Invalid output\n$error"))) - case t: Throwable ⇒ - JsObject(Seq("errorMessage" → JsString(t.getMessage + ":" + t.getStackTrace().mkString("", "\n\t", "\n")))) - } - } - } -} + name: String, + version: String, + description: String, + dataTypeList: Seq[String], + author: String, + url: String, + license: String, + command: Path, + config: JsObject) extends Analyzer \ No newline at end of file diff --git a/app/models/Job.scala b/app/models/Job.scala index 8d4460f64..bdc90644d 100644 --- a/app/models/Job.scala +++ b/app/models/Job.scala @@ -1,24 +1,22 @@ package models -import play.api.libs.json.JsObject -import scala.concurrent.Future import java.util.Date -import scala.util.Success -import scala.util.Failure + +import scala.concurrent.Future +import scala.util.{ Failure, Success } object JobStatus extends Enumeration { type Type = Value val InProgress, Success, Failure = Value } -case class Job(id: String, analyzerId: String, artifact: Artifact, report: Future[JsObject]) { + +case class Job(id: String, analyzer: Analyzer, artifact: Artifact, report: Future[Report]) { val date: Date = new Date() def status: JobStatus.Type = report.value match { - case Some(Success(x)) ⇒ (x \ "success").asOpt[Boolean] match { - case Some(true) ⇒ JobStatus.Success - case _ ⇒ JobStatus.Failure - } - case Some(Failure(_)) ⇒ JobStatus.Failure - case None ⇒ JobStatus.InProgress + case Some(Success(SuccessReport(_, _, _))) ⇒ JobStatus.Success + case Some(Success(FailureReport(_))) ⇒ JobStatus.Failure + case Some(Failure(_)) ⇒ JobStatus.Failure + case None ⇒ JobStatus.InProgress } } diff --git a/app/models/JsonFormat.scala b/app/models/JsonFormat.scala index b2536890c..9a81cd685 100644 --- a/app/models/JsonFormat.scala +++ b/app/models/JsonFormat.scala @@ -1,19 +1,11 @@ package models -import scala.annotation.implicitNotFound - -import play.api.libs.json.Json +import models.JobStatus.Type import play.api.libs.json.Json.toJsFieldJsValueWrapper -import play.api.libs.json.Writes -import scala.concurrent.Future -import play.api.libs.json.JsObject -import scala.util.Success -import scala.util.Failure -import play.api.libs.json.JsString -import play.api.libs.json.OWrites +import play.api.libs.json.{ JsString, Json, OWrites, Writes, _ } object JsonFormat { - implicit val analyzerWrites = Writes[Analyzer](analyzer ⇒ Json.obj( + implicit val analyzerWrites: Writes[Analyzer] = Writes[Analyzer](analyzer ⇒ Json.obj( "name" → analyzer.name, "version" → analyzer.version, "description" → analyzer.description, @@ -23,24 +15,65 @@ object JsonFormat { "license" → analyzer.license, "id" → analyzer.id)) - implicit val fileArtifactWrites = OWrites[FileArtifact](fileArtifact ⇒ Json.obj( + implicit val fileArtifactWrites: OWrites[FileArtifact] = OWrites[FileArtifact](fileArtifact ⇒ Json.obj( "attributes" → fileArtifact.attributes)) - implicit val dataArtifactWrites = Json.writes[DataArtifact] - implicit val dataActifactReads = Json.reads[DataArtifact] + implicit val dataArtifactWrites: OWrites[DataArtifact] = Json.writes[DataArtifact] + implicit val dataActifactReads: Reads[DataArtifact] = Json.reads[DataArtifact] - implicit val artifactWrites = OWrites[Artifact](artifact ⇒ artifact match { + val artifactWrites: OWrites[Artifact] = OWrites[Artifact] { case dataArtifact: DataArtifact ⇒ dataArtifactWrites.writes(dataArtifact) case fileArtifact: FileArtifact ⇒ fileArtifactWrites.writes(fileArtifact) - }) + } + + val artifactReads: Reads[Artifact] = Reads[Artifact] { json ⇒ + (json \ "data").asOpt[String] + .map { data ⇒ JsSuccess(DataArtifact(data, json.as[JsObject])) } + .getOrElse(JsError(__ \ "data", "data is missing")) + } + implicit val artifactFormat = Format(artifactReads, artifactWrites) + + implicit val jobStatusWrites: Writes[Type] = Writes[JobStatus.Type](jobStatus ⇒ JsString(jobStatus.toString)) + + val reportArtifactReads: Reads[Artifact] = + for { + tpe ← (__ \ "type").read[String] + value ← (__ \ "value").read[String] + } yield DataArtifact(value, Json.obj("dataType" → tpe)) + + val reportReads: Reads[Report] = Reads[Report] { json ⇒ + val success = (json \ "success").asOpt[Boolean].getOrElse(false) + JsSuccess { + if (success) + (for { + artifacts ← (json \ "artifacts").asOpt[Seq[Artifact]](Reads.seq(reportArtifactReads)) + full ← (json \ "full").asOpt[JsObject] + summary ← (json \ "summary").asOpt[JsObject] + } yield SuccessReport(artifacts, full, summary)) + .getOrElse(FailureReport(s"Invalid analyzer output format : $json")) + else + FailureReport((json \ "error").asOpt[String].getOrElse(json.toString)) + } + } - implicit val jobStatusWrites = Writes[JobStatus.Type](jobStatus ⇒ JsString(jobStatus.toString)) + val reportWrites: Writes[Report] = Writes[Report] { + case SuccessReport(artifacts, full, summary) ⇒ Json.obj( + "artifacts" → artifacts, + "full" → full, + "summary" → summary) + case FailureReport(message) ⇒ Json.obj("errorMessage" → message) + } - implicit val jobWrites = OWrites[Job](job ⇒ Json.obj( - "id" → job.id, - "analyzerId" → job.analyzerId, - "status" → job.status, - "date" → job.date, - "artifact" → job.artifact)) + implicit val reportFormat: Format[Report] = Format[Report](reportReads, reportWrites) + implicit val jobWrites: OWrites[Job] = OWrites[Job] { job ⇒ + val report = job.report.value.flatMap(_.toOption).map(Json.toJson(_)).getOrElse(JsNull) + Json.obj( + "id" → job.id, + "analyzerId" → job.analyzer.id, + "status" → job.status, + "date" → job.date, + "artifact" → job.artifact, + "report" → report) + } } diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala index a715abdfd..14ef92350 100644 --- a/app/models/MispModule.scala +++ b/app/models/MispModule.scala @@ -1,97 +1,14 @@ package models -import java.io._ - -import org.apache.commons.codec.binary.Base64InputStream -import play.api.Logger -import play.api.libs.json.{ JsObject, Json } -import services.MispSrv - -import scala.collection.JavaConverters._ -import scala.concurrent.{ ExecutionContext, Future } -import scala.sys.process._ -import scala.util.{ Failure, Success, Try } - case class MispModule( - mispSrv: MispSrv, - name: String, + name: String, version: String, description: String, - dataTypeList: Seq[String], author: String, - moduleName: String, - loaderCommand: String)(implicit val ec: ExecutionContext) extends Analyzer { - + dataTypeList: Seq[String], + inputAttributes: Seq[String], + config: Seq[String], + loaderCommand: String) extends Analyzer { val license = "AGPL-3.0" val url = "https://github.com/MISP/misp-modules" - - private def stringStream(string: String): InputStream = { - new ByteArrayInputStream(string.getBytes) - } - def analyze(artifact: Artifact): Future[JsObject] = { - val input = artifact match { - case DataArtifact(data, _) ⇒ - stringStream(Json.obj(mispSrv.dataType2mispType(artifact.dataType).head → data).toString) - case FileArtifact(data, _) ⇒ - new SequenceInputStream(Iterator( - stringStream("""{"attachment":""""), - new Base64InputStream(new FileInputStream(data), true), - stringStream("\"}")).asJavaEnumeration) - } - val output = (s"$loaderCommand --run $moduleName" #< input).!! - Future { - Json.parse(output).as[JsObject] - } - } -} - -object MispModule { - private[MispModule] lazy val logger = Logger(getClass) - def list(loaderCommand: String): Seq[String] = - Json.parse(s"$loaderCommand --list".!!).as[Seq[String]] - - def apply( - loaderCommand: String, - moduleName: String, - mispSrv: MispSrv)(implicit ec: ExecutionContext): Option[MispModule] = { - println(s"Loading MISP module $moduleName") - for { - moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) match { - case Success(s) ⇒ Some(s) - case Failure(f) ⇒ - f.printStackTrace() - None - } - name ← (moduleInfo \ "name").asOpt[String].orElse { - println("name not defined") - None - } - version ← (moduleInfo \ "moduleinfo" \ "version").asOpt[String].orElse { - println("version not defined") - None - } - description ← (moduleInfo \ "moduleinfo" \ "description").asOpt[String].orElse { - println("description not defined") - None - } - dataTypeList ← (moduleInfo \ "mispattributes" \ "input") - .asOpt[Seq[String]] - .map(_.map(mispSrv.mispType2dataType(_)).distinct) - .orElse { - println("input attributes not defined") - None - } - author ← (moduleInfo \ "moduleinfo" \ "author").asOpt[String].orElse { - println("author not defined") - None - } - mispModule ← Try(MispModule(mispSrv, name, version, description, dataTypeList, author, moduleName, loaderCommand)) match { - case Success(s) ⇒ Some(s) - case Failure(f) ⇒ - f.printStackTrace() - sys.error("Load module fails") - } - _ = println("Module load succeed") - } yield mispModule - } } \ No newline at end of file diff --git a/app/models/Report.scala b/app/models/Report.scala new file mode 100644 index 000000000..296466413 --- /dev/null +++ b/app/models/Report.scala @@ -0,0 +1,9 @@ +package models + +import play.api.libs.json.JsObject + +sealed abstract class Report(success: Boolean) + +case class SuccessReport(artifacts: Seq[Artifact], full: JsObject, summary: JsObject) extends Report(true) + +case class FailureReport(message: String) extends Report(false) \ No newline at end of file diff --git a/app/services/Analyzer.scala b/app/services/Analyzer.scala deleted file mode 100644 index b69c56cd6..000000000 --- a/app/services/Analyzer.scala +++ /dev/null @@ -1,107 +0,0 @@ -package services - -import java.io.File -import java.nio.file.{ Files, Path, Paths } -import javax.inject.{ Inject, Provider, Singleton } - -import akka.actor.ActorSystem -import models.{ Analyzer, ExternalAnalyzer, MispModule } -import play.api.libs.json.{ JsObject, JsValue, Json } -import play.api.{ Configuration, Logger } -import util.JsonConfig.configWrites - -import scala.collection.JavaConversions.iterableAsScalaIterable -import scala.concurrent.ExecutionContext -import scala.util.Try - -@Singleton -class AnalyzerSrv( - mispSrvProvider: Provider[MispSrv], - analyzerPath: Path, - analyzerConfig: JsObject, - mispModulesPath: Path, - mispModuleLoaderCommand: Option[String], - akkaSystem: ActorSystem) { - @Inject def this( - mispSrvProvider: Provider[MispSrv], - configuration: Configuration, - akkaSystem: ActorSystem) = - this( - mispSrvProvider, - Paths.get(configuration.getString("analyzer.path").getOrElse(".")), - configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), - Paths.get(configuration.getString("misp.modules.path").getOrElse(".")), - configuration.getString("misp.modules.loader"), - akkaSystem) - - private[AnalyzerSrv] lazy val logger = Logger(getClass) - lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") - lazy val mispSrv = mispSrvProvider.get - - private lazy val externalAnalyzers: Seq[Analyzer] = getExternalAnalyzers - private lazy val mispModules: Seq[Analyzer] = getMispModules - def list: Seq[Analyzer] = externalAnalyzers ++ mispModules // ::: javaAnalyzers - def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) - def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) - - private[services] def getExternalAnalyzers: Seq[Analyzer] = { - val globalConfig = (analyzerConfig \ "global").asOpt[JsObject].getOrElse(JsObject(Nil)) - for { - analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).toSeq).getOrElse { - logger.warn(s"Analyzer directory ($analyzerPath) is not found") - Nil - } - if Files.isDirectory(analyzerDir) - infoFile ← Files.newDirectoryStream(analyzerDir, "*.json").toSeq - if Files.isReadable(infoFile) - info = readInfo(infoFile) - name ← (info \ "name").asOpt[String] orElse { - logger.warn(s"name is missing in $infoFile"); None - } - version ← (info \ "version").asOpt[String] orElse { - logger.warn(s"version is missing in $infoFile"); None - } - description ← (info \ "description").asOpt[String] orElse { - logger.warn(s"description is missing in $infoFile"); None - } - dataTypeList ← (info \ "dataTypeList").asOpt[Seq[String]] orElse { - logger.warn(s"dataTypeList is missing in $infoFile"); None - } - command ← (info \ "command").asOpt[String] orElse { - logger.warn(s"command is missing in $infoFile"); None - } - author ← (info \ "author").asOpt[String] orElse { - logger.warn(s"author is missing in $infoFile"); None - } - url ← (info \ "url").asOpt[String] orElse { - logger.warn(s"url is missing in $infoFile"); None - } - license ← (info \ "license").asOpt[String] orElse { - logger.warn(s"license is missing in $infoFile"); None - } - config = (info \ "config").asOpt[JsObject].getOrElse(JsObject(Nil)) - baseConfig = (info \ "baseConfig").asOpt[String].flatMap(c ⇒ (analyzerConfig \ c).asOpt[JsObject]).getOrElse(JsObject(Nil)) - absoluteCommand = analyzerPath.resolve(Paths.get(command.replaceAll("[\\/]", File.separator))) - _ = logger.info(s"Register analyzer $name $version (${(name + "_" + version).replaceAll("\\.", "_")})") - } yield ExternalAnalyzer(name, version, description, dataTypeList, author, url, license, absoluteCommand, globalConfig deepMerge baseConfig deepMerge config)(analyzeExecutionContext) - } - - private[services] def getMispModules: Seq[Analyzer] = { - for { - loaderCommand ← mispModuleLoaderCommand.toSeq - moduleName ← MispModule.list(loaderCommand) - - _ = println("MISP module loading ...") - mispModule ← MispModule(loaderCommand, moduleName, mispSrv)(analyzeExecutionContext) - _ = println("MISP module load success") - } yield mispModule - } - - private[services] def readInfo(file: Path): JsValue = { - val source = scala.io.Source.fromFile(file.toFile) - try { - Json.parse(source.mkString) - } - finally { source.close() } - } -} diff --git a/app/services/AnalyzerSrv.scala b/app/services/AnalyzerSrv.scala new file mode 100644 index 000000000..4637cf98b --- /dev/null +++ b/app/services/AnalyzerSrv.scala @@ -0,0 +1,37 @@ +package services + +import javax.inject.{ Inject, Singleton } + +import models._ +import play.api.Logger + +import scala.concurrent.Future + +@Singleton +class AnalyzerSrv @Inject() ( + jobSrv: JobSrv, + mispSrv: MispSrv, + externalAnalyzerSrv: ExternalAnalyzerSrv) { + + private[AnalyzerSrv] lazy val logger = Logger(getClass) + + def list: Seq[Analyzer] = externalAnalyzerSrv.list ++ mispSrv.list + + def get(analyzerId: String): Option[Analyzer] = list.find(_.id == analyzerId) + + def listForType(dataType: String): Seq[Analyzer] = list.filter(_.dataTypeList.contains(dataType)) + + def analyze(analyzerId: String, artifact: Artifact): Future[Job] = { + get(analyzerId) + .map { analyzer ⇒ analyze(analyzer, artifact) } + .getOrElse(sys.error("analyzer not found")) + } + + def analyze(analyzer: Analyzer, artifact: Artifact): Future[Job] = { + val report = analyzer match { + case ea: ExternalAnalyzer ⇒ externalAnalyzerSrv.analyze(ea, artifact) + case mm: MispModule ⇒ mispSrv.analyze(mm, artifact) + } + jobSrv.create(analyzer, artifact, report) + } +} diff --git a/app/services/ExternalAnalyzerSrv.scala b/app/services/ExternalAnalyzerSrv.scala new file mode 100644 index 000000000..ae1927e69 --- /dev/null +++ b/app/services/ExternalAnalyzerSrv.scala @@ -0,0 +1,142 @@ +package services + +import java.io.{ BufferedReader, File, InputStreamReader } +import java.nio.file.{ Files, Path, Paths } +import javax.inject.{ Inject, Singleton } + +import akka.actor.ActorSystem +import com.fasterxml.jackson.core.JsonParseException +import com.fasterxml.jackson.databind.JsonMappingException +import models.JsonFormat._ +import models._ +import util.JsonConfig +import play.api.libs.json._ +import play.api.{ Configuration, Logger } + +import scala.collection.JavaConversions.iterableAsScalaIterable +import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process.{ BasicIO, Process, ProcessIO } +import scala.util.{ Failure, Try } + +@Singleton +class ExternalAnalyzerSrv( + analyzerPath: Path, + analyzerConfig: JsObject, + akkaSystem: ActorSystem) { + + @Inject() def this(configuration: Configuration, akkaSystem: ActorSystem) = + this( + Paths.get(configuration.getString("analyzer.path").getOrElse(".")), + JsonConfig.configWrites.writes(configuration.getConfig("analyzer.config").getOrElse(Configuration.empty)), + akkaSystem) + + private[ExternalAnalyzerSrv] lazy val analyzeExecutionContext: ExecutionContext = + akkaSystem.dispatchers.lookup("analyzer") + private[ExternalAnalyzerSrv] lazy val globalConfig: JsObject = + (analyzerConfig \ "global").asOpt[JsObject].getOrElse(JsObject(Nil)) + private[ExternalAnalyzerSrv] lazy val logger = + Logger(getClass) + + lazy val list: Seq[ExternalAnalyzer] = { + for { + analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).toSeq).getOrElse { + logger.warn(s"Analyzer directory ($analyzerPath) is not found") + Nil + } + if Files.isDirectory(analyzerDir) + infoFile ← Files.newDirectoryStream(analyzerDir, "*.json").toSeq + if Files.isReadable(infoFile) + analyzer ← Try(readInfo(infoFile).as[ExternalAnalyzer](reads)) + .recoverWith { + case error ⇒ + logger.warn(s"Load of analyzer $infoFile fails", error) + Failure(error) + } + .toOption + _ = logger.info(s"Register analyzer ${analyzer.name} ${analyzer.version} (${analyzer.id})") + } yield analyzer + } + + def get(analyzerId: String): Option[ExternalAnalyzer] = list.find(_.id == analyzerId) + + private val osexec = + if (System.getProperty("os.name").toLowerCase.contains("win")) + (c: String) ⇒ s"""cmd /c $c""" + else + (c: String) ⇒ s"""sh -c "./$c" """ + + def analyze(analyzer: ExternalAnalyzer, artifact: Artifact): Future[Report] = { + Future { + val input = artifact match { + case FileArtifact(file, attributes) ⇒ attributes + ("file" → JsString(file.getAbsoluteFile.toString)) + ("config" → analyzer.config) + case DataArtifact(data, attributes) ⇒ attributes + ("data" → JsString(data)) + ("config" → analyzer.config) + } + val output = new StringBuffer + val error = new StringBuffer + try { + logger.info(s"Execute ${osexec(analyzer.command.getFileName.toString)} in ${analyzer.command.getParent.toFile.getAbsoluteFile.getName}") + Process(osexec(analyzer.command.getFileName.toString), analyzer.command.getParent.toFile).run( + new ProcessIO( + { stdin ⇒ + try stdin.write(input.toString.getBytes("UTF-8")) + finally stdin.close() + }, { stdout ⇒ + val reader = new BufferedReader(new InputStreamReader(stdout, "UTF-8")) + try BasicIO.processLinesFully { line ⇒ + output.append(line).append(System.lineSeparator()) + () + }(reader.readLine) + finally reader.close() + }, { stderr ⇒ + val reader = new BufferedReader(new InputStreamReader(stderr, "UTF-8")) + try BasicIO.processLinesFully { line ⇒ + error.append(line).append(System.lineSeparator()) + () + }(reader.readLine) + finally reader.close() + })).exitValue + Json.parse(output.toString).as[Report] + } + catch { + case _: JsonMappingException ⇒ + error.append(output) + FailureReport(s"Error: Invalid output\n$error") + case _: JsonParseException ⇒ + error.append(output) + FailureReport(s"Error: Invalid output\n$error") + case t: Throwable ⇒ + FailureReport(t.getMessage + ":" + t.getStackTrace.mkString("", "\n\t", "\n")) + } + }(analyzeExecutionContext) + } + + private[ExternalAnalyzerSrv] def readInfo(file: Path): JsValue = { + val source = scala.io.Source.fromFile(file.toFile) + try Json.parse(source.mkString) + finally source.close() + } + + private[ExternalAnalyzerSrv] val reads: Reads[ExternalAnalyzer] = + for { + name ← (__ \ "name").read[String] + version ← (__ \ "version").read[String] + description ← (__ \ "description").read[String] + dataTypeList ← (__ \ "dataTypeList").read[Seq[String]] + author ← (__ \ "author").read[String] + url ← (__ \ "url").read[String] + license ← (__ \ "license").read[String] + command ← (__ \ "command").read[String] + absoluteCommand = analyzerPath.resolve(Paths.get(command.replaceAll("[\\/]", File.separator))) + config ← (__ \ "config").read[JsObject] + baseConfigKey ← (__ \ "baseConfig").read[String] + baseConfig = (analyzerConfig \ baseConfigKey).asOpt[JsObject].getOrElse(JsObject(Nil)) + } yield ExternalAnalyzer( + name, + version, + description, + dataTypeList, + author, + url, + license, + absoluteCommand, globalConfig deepMerge baseConfig deepMerge config) +} \ No newline at end of file diff --git a/app/services/Job.scala b/app/services/JobSrv.scala similarity index 73% rename from app/services/Job.scala rename to app/services/JobSrv.scala index c66a7b455..787ca7d6f 100644 --- a/app/services/Job.scala +++ b/app/services/JobSrv.scala @@ -6,8 +6,7 @@ import javax.inject.{ Inject, Named } import akka.actor.{ Actor, ActorRef, ActorSystem, actorRef2Scala } import akka.pattern.ask import akka.util.Timeout -import models.{ Analyzer, Artifact, Job, JobStatus } -import play.api.libs.json.{ JsString, JsValue } +import models._ import play.api.{ Configuration, Logger } import scala.concurrent.duration.Duration.Infinite @@ -16,11 +15,13 @@ import scala.concurrent.{ ExecutionContext, Future, Promise } import scala.util.Random class JobSrv @Inject() ( - analyzerSrv: AnalyzerSrv, + //analyzerSrv: AnalyzerSrv, @Named("JobActor") jobActor: ActorRef, implicit val ec: ExecutionContext, implicit val system: ActorSystem) { + import services.JobActor._ + implicit val timeout = Timeout(5.seconds) def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], start: Int, limit: Int): Future[(Int, Seq[Job])] = { @@ -36,17 +37,26 @@ class JobSrv @Inject() ( case _ ⇒ sys.error("TODO") } - def create(artifact: Artifact, analyzerId: String): Future[Job] = { - analyzerSrv.get(analyzerId) - .map { analyzer ⇒ create(artifact, analyzer) } - .getOrElse(Future.failed(new Exception("analyzer not found"))) + def create(analyzer: Analyzer, artifact: Artifact, report: Future[Report]): Future[Job] = { + (jobActor ? CreateJob(artifact, analyzer, report)) map { + case job: Job ⇒ job + case _ ⇒ sys.error("TODO") + } } - def create(artifact: Artifact, analyzer: Analyzer): Future[Job] = - (jobActor ? CreateJob(artifact, analyzer)) map { - case j: Job ⇒ j - case _ ⇒ sys.error("TODO") - } + // @deprecated + // def create(artifact: Artifact, analyzerId: String): Future[Job] = { + // analyzerSrv.get(analyzerId) + // .map { analyzer ⇒ create(artifact, analyzer) } + // .getOrElse(Future.failed(new Exception("analyzer not found"))) + // } + // + // @deprecated + // def create(artifact: Artifact, analyzer: Analyzer): Future[Job] = + // (jobActor ? CreateJob(artifact, analyzer)) map { + // case j: Job ⇒ j + // case _ ⇒ sys.error("TODO") + // } def remove(jobId: String): Future[Unit] = { (jobActor ? RemoveJob(jobId)).map { @@ -56,19 +66,20 @@ class JobSrv @Inject() ( } } - def waitReport(jobId: String, atMost: Duration): Future[(JobStatus.Type, JsValue)] = { - val statusResult = get(jobId) - .flatMap(_.report) - .map((JobStatus.Success, _)) - .recover { case error ⇒ (JobStatus.Failure, JsString(error.getMessage)) } - - atMost match { - case _: Infinite ⇒ statusResult - case duration: FiniteDuration ⇒ - val prom = Promise[(JobStatus.Type, JsValue)]() - val timeout = system.scheduler.scheduleOnce(duration) { prom.success((JobStatus.InProgress, JsString("Timeout"))); () } - statusResult.onComplete(_ ⇒ timeout.cancel()) - Future.firstCompletedOf(List(statusResult, prom.future)) + def waitReport(jobId: String, atMost: Duration): Future[Job] = { + get(jobId).flatMap { job ⇒ + val finishedJob = job.report.map(_ ⇒ job) + atMost match { + case _: Infinite ⇒ finishedJob + case duration: FiniteDuration ⇒ + val prom = Promise[Job]() + val timeout = system.scheduler.scheduleOnce(duration) { + prom.success(job) + () + } + finishedJob.onComplete(_ ⇒ timeout.cancel()) + Future.firstCompletedOf(List(finishedJob, prom.future)) + } } } } @@ -78,7 +89,7 @@ object JobActor { case class JobList(total: Int, jobs: Seq[Job]) case class GetJob(jobId: String) case object JobNotFound - case class CreateJob(artifact: Artifact, analyzer: Analyzer) + case class CreateJob(artifact: Artifact, analyzer: Analyzer, report: Future[Report]) case class RemoveJob(jobId: String) case object JobRemoved case object JobCleanup @@ -91,6 +102,7 @@ class JobActor( implicit val ec: ExecutionContext) extends Actor { import services.JobActor._ + @Inject def this( configuration: Configuration, analyzerSrv: AnalyzerSrv, @@ -122,7 +134,7 @@ class JobActor( val filteredJobs = jobs.filter(j ⇒ dataTypeFilter.fold(true)(j.artifact.dataTypeFilter) && dataFilter.fold(true)(j.artifact.dataFilter) && - analyzerFilter.fold(true)(j.analyzerId.contains)) + analyzerFilter.fold(true)(j.analyzer.id.contains)) sender ! JobList(filteredJobs.size, filteredJobs.slice(start, start + limit)) case GetJob(jobId) ⇒ sender ! jobs.find(_.id == jobId).getOrElse(JobNotFound) case RemoveJob(jobId) ⇒ @@ -132,9 +144,9 @@ class JobActor( context.become(jobState(j)) case None ⇒ sender ! JobNotFound } - case CreateJob(artifact, analyzer) ⇒ + case CreateJob(artifact, analyzer, report) ⇒ val jobId = Random.alphanumeric.take(16).mkString - val job = Job(jobId, analyzer.id, artifact, analyzer.analyze(artifact)) + val job = Job(jobId, analyzer, artifact, report) sender ! job context.become(jobState(job :: jobs)) case JobCleanup if jobLifeTime.isInstanceOf[FiniteDuration] ⇒ diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala index e11253e35..139eb749c 100644 --- a/app/services/MispSrv.scala +++ b/app/services/MispSrv.scala @@ -1,20 +1,77 @@ package services +import java.io.{ ByteArrayInputStream, FileInputStream, InputStream, SequenceInputStream } import javax.inject.Inject -import models.JsonFormat.dataActifactReads -import models.{ DataArtifact, FileArtifact } -import org.apache.commons.codec.binary.Base64 -import play.api.Logger -import play.api.libs.json.{ JsArray, JsObject, JsValue, Json } +import akka.actor.ActorSystem +import models.JsonFormat._ +import models._ +import org.apache.commons.codec.binary.{ Base64, Base64InputStream } +import play.api.libs.json.{ Json, _ } +import play.api.{ Configuration, Logger } +import scala.collection.JavaConverters._ import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process._ +import scala.util.{ Failure, Success, Try } + +class MispSrv( + loaderCommandOption: Option[String], + externalAnalyzerSrv: ExternalAnalyzerSrv, + jobSrv: JobSrv, + akkaSystem: ActorSystem) { + + @Inject() def this( + configuration: Configuration, + externalAnalyzerSrv: ExternalAnalyzerSrv, + jobSrv: JobSrv, + akkaSystem: ActorSystem) = this( + configuration.getString("misp.modules.loader"), + externalAnalyzerSrv, + jobSrv, + akkaSystem) -class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { private[MispSrv] lazy val logger = Logger(getClass) + private[MispSrv] lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") + + lazy val list: Seq[MispModule] = + loaderCommandOption.fold(Seq.empty[MispModule]) { loaderCommand ⇒ + Json.parse(s"$loaderCommand --list".!!) + .as[Seq[String]] + .map { moduleName ⇒ + moduleName → (for { + moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) + module ← Try(moduleInfo.as[MispModule](reads(loaderCommand))) + } yield module) + } + .flatMap { + case (moduleName, Failure(error)) ⇒ + logger.warn(s"Load MISP module $moduleName fails", error) + Nil + case (_, Success(module)) ⇒ + logger.info(s"Register MISP module ${module.name} ${module.version}") + Seq(module) + } + } + + def get(moduleName: String): Option[MispModule] = list.find(_.name == moduleName) - def moduleList: JsValue = { - JsArray(analyzerSrv.list.map { analyzer ⇒ + def moduleList: JsArray = { + val mispModules = list.map { module ⇒ + Json.obj( + "name" → module.name, + "type" → "cortex", + "mispattributes" → Json.obj( + "input" → module.inputAttributes, + "output" → Json.arr()), + "meta" → Json.obj( + "module-type" → Json.arr("cortex"), + "description" → module.description, + "author" → module.author, + "version" → module.version, + "config" → module.config)) + } + val externalAnalyzers = externalAnalyzerSrv.list.map { analyzer ⇒ Json.obj( "name" → analyzer.id, "type" → "cortex", @@ -27,58 +84,117 @@ class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { "author" → analyzer.author, "version" → analyzer.version, "config" → Json.arr())) - }) + } + JsArray(mispModules ++ externalAnalyzers) } def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = { - analyzerSrv.get(module).map { analyzer ⇒ - val artifact = mispType2dataType(mispType) match { - case "file" if mispType == "malware-sample" ⇒ ??? - case "file" ⇒ FileArtifact(Base64.decodeBase64(data), Json.obj( - "tlp" → 1, - "dataType" → "file")) - case dataType ⇒ DataArtifact(data, Json.obj( - "tlp" → 1, - "dataType" → dataType)) - } + loaderCommandOption + .flatMap { loaderCommand ⇒ + val artifact = toArtifact(mispType, data) + get(module) + .map { mispModule ⇒ + val mispReport = Future { + val input = Json.obj(mispType → data) + val output = (s"$loaderCommand --run $module" #< input.toString).!! + Json.parse(output).as[JsObject] + } + jobSrv.create(mispModule, artifact, mispReport.map(toReport)) + mispReport - analyzer - .analyze(artifact) - .map { output ⇒ - logger.info(s"analyzer output:\n$output") - val success = (output \ "success") - .asOpt[Boolean] - .getOrElse(false) - if (success) { - Json.obj( - "results" → ((output \ "artifacts") - .asOpt[Seq[JsObject]] - .getOrElse(Nil) - .map { artifact ⇒ - Json.obj( - "types" → dataType2mispType((artifact \ "type").as[String]), - "values" → Json.arr((artifact \ "value").as[String])) - } - :+ Json.obj( - "types" → Json.arr("cortex"), - "values" → Json.arr(output.toString)))) } - else { - val message = (output \ "error").asOpt[String].getOrElse(output.toString) - Json.obj( - "error" → message) + .orElse { + externalAnalyzerSrv + .get(module) + .map { analyzer ⇒ + externalAnalyzerSrv.analyze(analyzer, artifact) + .map { report ⇒ toMispOutput(report) } + } } + } + .getOrElse(Future.failed(new Exception(s"Module $module not found"))) + } + + def analyze(module: MispModule, artifact: Artifact): Future[Report] = { + def stringStream(string: String): InputStream = + new ByteArrayInputStream(string.getBytes) + + val input = artifact match { + case DataArtifact(data, _) ⇒ + stringStream(Json.obj(dataType2mispType(artifact.dataType).head → data).toString) + case FileArtifact(data, _) ⇒ + new SequenceInputStream(Iterator( + stringStream("""{"attachment":""""), + new Base64InputStream(new FileInputStream(data), true), + stringStream("\"}")).asJavaEnumeration) + } + + Future { + val output = (s"${module.loaderCommand} --run ${module.name}" #< input).!! + toReport(Json.parse(output).as[JsObject]) + }(analyzeExecutionContext) + } + + private def toArtifact(mispType: String, data: String): Artifact = { + mispType2dataType(mispType) match { + case "file" if mispType == "malware-sample" ⇒ ??? + case "file" ⇒ FileArtifact(Base64.decodeBase64(data), Json.obj( + "tlp" → 1, + "dataType" → "file")) + case dataType ⇒ DataArtifact(data, Json.obj( + "tlp" → 1, + "dataType" → dataType)) + } + } + + private def toReport(mispOutput: JsObject): Report = { + (mispOutput \ "results").asOpt[Seq[JsObject]] + .map { attributes ⇒ + val artifacts: Seq[Artifact] = for { + attribute ← attributes + tpe ← (attribute \ "types").asOpt[Seq[String]] + .orElse((attribute \ "types").asOpt[String].map(Seq(_))) + .getOrElse(Nil) + dataType = mispType2dataType(tpe) // TODO handle FileArtifact + value ← (attribute \ "values").asOpt[Seq[String]] + .orElse((attribute \ "values").asOpt[String].map(Seq(_))) + .getOrElse(Nil) + } yield DataArtifact(value, Json.obj("dataType" → dataType)) + SuccessReport(artifacts, Json.obj("artifacts" → Json.toJson(artifacts)), JsObject(Nil)) + } + .getOrElse { + val message = (mispOutput \ "error").asOpt[String].getOrElse(mispOutput.toString) + FailureReport(message) + } + } + + private def toMispOutput(report: Report): JsObject = { + report match { + case SuccessReport(artifacts, _, _) ⇒ + val attributes = artifacts.map { + case artifact: DataArtifact ⇒ + Json.obj( + "types" → dataType2mispType(artifact.dataType), + "values" → Json.arr(artifact.data)) + case artifact: FileArtifact ⇒ + ??? // TODO } + val cortexAttribute = Json.obj( + "types" → Seq("cortex"), + "values" → Json.arr(Json.toJson(report).toString)) + + Json.obj("results" → (attributes :+ cortexAttribute)) + case FailureReport(message) ⇒ + Json.obj("error" → message) } - .getOrElse(Future.failed(new Exception(s"Module $module not found"))) } - def mispType2dataType(mispType: String): String = typeLookup.getOrElse(mispType, { + private def mispType2dataType(mispType: String): String = typeLookup.getOrElse(mispType, { logger.warn(s"Misp type $mispType not recognized") "other" }) - def dataType2mispType(dataType: String): Seq[String] = { + private def dataType2mispType(dataType: String): Seq[String] = { val mispTypes = typeLookup.filter(_._2 == dataType) .keys .toSeq @@ -91,6 +207,17 @@ class MispSrv @Inject() (analyzerSrv: AnalyzerSrv) { else mispTypes } + private def reads(loaderCommand: String): Reads[MispModule] = + for { + name ← (__ \ "name").read[String] + version ← (__ \ "meta" \ "version").read[String] + description ← (__ \ "meta" \ "description").read[String] + author ← (__ \ "meta" \ "author").read[String] + config ← (__ \ "meta" \ "config").read[Seq[String]] + input ← (__ \ "mispattributes" \ "input").read[Seq[String]] + dataTypes = input.map(mispType2dataType) + } yield MispModule(name, version, description, author, dataTypes, input, config, loaderCommand) + private val typeLookup = Map( "md5" → "hash", "sha1" → "hash", diff --git a/app/util/JsonConfig.scala b/app/util/JsonConfig.scala index 0c91d548a..79e075e02 100644 --- a/app/util/JsonConfig.scala +++ b/app/util/JsonConfig.scala @@ -1,17 +1,15 @@ package util -import scala.BigDecimal -import scala.collection.JavaConversions.asScalaBuffer - +import com.typesafe.config.ConfigValueType.{ BOOLEAN, NULL, NUMBER, STRING } +import com.typesafe.config.{ ConfigList, ConfigObject, ConfigValue } import play.api.Configuration -import play.api.libs.json.{ JsArray, JsBoolean, JsNull, JsNumber, JsObject, JsString, OWrites, Writes } +import play.api.libs.json._ -import com.typesafe.config.{ ConfigList, ConfigObject, ConfigValue } -import com.typesafe.config.ConfigValueType.{ BOOLEAN, NULL, NUMBER, STRING } +import scala.collection.JavaConversions.asScalaBuffer object JsonConfig { implicit val configValueWrites: Writes[ConfigValue] = Writes((value: ConfigValue) ⇒ value match { - case v: ConfigObject ⇒ configWrites.writes(Configuration(v.toConfig())) + case v: ConfigObject ⇒ configWrites.writes(Configuration(v.toConfig)) case v: ConfigList ⇒ JsArray(v.toSeq.map(x ⇒ configValueWrites.writes(x))) case v if v.valueType == NUMBER ⇒ JsNumber(BigDecimal(v.unwrapped.asInstanceOf[java.lang.Number].toString)) case v if v.valueType == BOOLEAN ⇒ JsBoolean(v.unwrapped.asInstanceOf[Boolean]) @@ -19,7 +17,7 @@ object JsonConfig { case v if v.valueType == STRING ⇒ JsString(v.unwrapped.asInstanceOf[String]) }) - implicit val configWrites = OWrites { (cfg: Configuration) ⇒ + implicit def configWrites = OWrites { (cfg: Configuration) ⇒ JsObject(cfg.subKeys.map(key ⇒ key → configValueWrites.writes(cfg.underlying.getValue(key))).toSeq) } } \ No newline at end of file diff --git a/build.sbt b/build.sbt index b149c99fc..1668fa704 100644 --- a/build.sbt +++ b/build.sbt @@ -41,9 +41,8 @@ mappings in Universal ++= { } // Release // -import ReleaseTransformations._ - import Release._ +import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ bintrayOrganization := Some("cert-bdf") @@ -88,9 +87,9 @@ dockerUpdateLatest := true mappings in Universal += file("docker/entrypoint") -> "bin/entrypoint" mappings in Universal ~= { _.filterNot { - case (_, name) => name.startsWith("conf/") && name != "conf/keepme" + case (_, fileName) => fileName.startsWith("conf/") && name != "conf/keepme" }} -import com.typesafe.sbt.packager.docker.{ ExecCmd, Cmd } +import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd} dockerCommands := dockerCommands.value.map { case ExecCmd("ENTRYPOINT", _*) => ExecCmd("ENTRYPOINT", "bin/entrypoint") @@ -109,10 +108,10 @@ dockerCommands := dockerCommands.value.head +: dockerCommands.value.tail // Scalariform // -import scalariform.formatter.preferences._ -import com.typesafe.sbt.SbtScalariform import com.typesafe.sbt.SbtScalariform.ScalariformKeys +import scalariform.formatter.preferences._ + ScalariformKeys.preferences in ThisBuild := ScalariformKeys.preferences.value .setPreference(AlignParameters, false) // .setPreference(FirstParameterOnNewline, Force) diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index fb0ed9e83..c265aaaf8 100755 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- diff --git a/project/Bintray.scala b/project/Bintray.scala index 7c3ad44e2..61fed98bf 100644 --- a/project/Bintray.scala +++ b/project/Bintray.scala @@ -1,18 +1,16 @@ import java.io.File -import scala.concurrent.duration.Duration -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global - -import sbt._ -import sbt.Keys._ - -import dispatch.{ Http, FunctionHandler } - -import bintry.Client import bintray.BintrayCredentials -import bintray.BintrayKeys.{ bintrayEnsureCredentials, bintrayOrganization, bintrayRepository, bintrayPackage } +import bintray.BintrayKeys.{bintrayEnsureCredentials, bintrayOrganization, bintrayPackage, bintrayRepository} +import bintry.Client import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport.Universal +import dispatch.{FunctionHandler, Http} +import sbt.Keys._ +import sbt._ + +import scala.concurrent.Await +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration object PublishToBinTray extends Plugin { val publishRelease = taskKey[Unit]("Publish binary in bintray") diff --git a/project/FrontEnd.scala b/project/FrontEnd.scala index f0bf55342..ac8b44d0c 100644 --- a/project/FrontEnd.scala +++ b/project/FrontEnd.scala @@ -1,5 +1,5 @@ -import sbt._ import sbt.Keys._ +import sbt._ object FrontEnd extends AutoPlugin { diff --git a/project/Release.scala b/project/Release.scala index 3dbeb2ee3..7a0184f08 100644 --- a/project/Release.scala +++ b/project/Release.scala @@ -1,10 +1,9 @@ -import sbt._ +import play.api.libs.json._ import sbt.Keys.baseDirectory -import sbt.{ Project, Extracted, State, IO, File, StateOps } +import sbt.{File, IO, Project, State, _} import sbtrelease.ReleasePlugin.autoImport._ import sbtrelease.ReleaseStateTransformations.readVersion -import sbtrelease.{ Vcs, Versions } -import play.api.libs.json._ +import sbtrelease.{Vcs, Versions} object Release { val releaseVersionUIFile = settingKey[File]("The json package file to write the version to")