diff --git a/app/models/JsonFormat.scala b/app/models/JsonFormat.scala index 561544d3b..4c08752ff 100644 --- a/app/models/JsonFormat.scala +++ b/app/models/JsonFormat.scala @@ -18,7 +18,8 @@ object JsonFormat { implicit val fileArtifactWrites: OWrites[FileArtifact] = OWrites[FileArtifact](fileArtifact ⇒ Json.obj( "attributes" → fileArtifact.attributes)) - implicit val dataArtifactWrites: OWrites[DataArtifact] = Json.writes[DataArtifact] + implicit val dataArtifactWrites: OWrites[DataArtifact] = OWrites[DataArtifact](artifact ⇒ + artifact.attributes + ("data" → JsString(artifact.data))) implicit val dataActifactReads: Reads[DataArtifact] = Json.reads[DataArtifact] val artifactWrites: OWrites[Artifact] = OWrites[Artifact] { diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala index c5c08f065..3d01ed72c 100644 --- a/app/models/MispModule.scala +++ b/app/models/MispModule.scala @@ -1,5 +1,7 @@ package models +import play.api.libs.json.JsObject + case class MispModule( name: String, version: String, @@ -7,7 +9,7 @@ case class MispModule( author: String, dataTypeList: Seq[String], inputAttributes: Seq[String], - config: Seq[String], + config: JsObject, loaderCommand: String) extends Analyzer { val license = "AGPL-3.0" val url = "https://github.com/MISP/misp-modules" diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala index c75ee7c23..ff232bb35 100644 --- a/app/services/MispSrv.scala +++ b/app/services/MispSrv.scala @@ -1,12 +1,13 @@ package services import java.io.{ ByteArrayInputStream, FileInputStream, InputStream, SequenceInputStream } -import javax.inject.Inject +import javax.inject.{ Inject, Singleton } import akka.actor.ActorSystem import models.JsonFormat._ import models._ import org.apache.commons.codec.binary.{ Base64, Base64InputStream } +import util.JsonConfig import play.api.libs.json.{ Json, _ } import play.api.{ Configuration, Logger } @@ -15,8 +16,11 @@ import scala.concurrent.{ ExecutionContext, Future } import scala.sys.process._ import scala.util.{ Failure, Success, Try } +@Singleton class MispSrv( - loaderCommandOption: Option[String], + mispModulesEnabled: Boolean, + loaderCommand: String, + mispModuleConfig: JsObject, externalAnalyzerSrv: ExternalAnalyzerSrv, jobSrv: JobSrv, akkaSystem: ActorSystem) { @@ -26,7 +30,9 @@ class MispSrv( externalAnalyzerSrv: ExternalAnalyzerSrv, jobSrv: JobSrv, akkaSystem: ActorSystem) = this( - configuration.getString("misp.modules.loader"), + configuration.getBoolean("misp.modules.enabled").getOrElse(false), + configuration.getString("misp.modules.loader").get, + JsonConfig.configWrites.writes(configuration.getConfig("misp.modules.config").getOrElse(Configuration.empty)), externalAnalyzerSrv, jobSrv, akkaSystem) @@ -34,25 +40,27 @@ class MispSrv( private[MispSrv] lazy val logger = Logger(getClass) private[MispSrv] lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") - lazy val list: Seq[MispModule] = - loaderCommandOption.fold(Seq.empty[MispModule]) { loaderCommand ⇒ - Json.parse(s"$loaderCommand --list".!!) - .as[Seq[String]] - .map { moduleName ⇒ - moduleName → (for { - moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) - module ← Try(moduleInfo.as[MispModule](reads(loaderCommand))) - } yield module) - } - .flatMap { - case (moduleName, Failure(error)) ⇒ - logger.warn(s"Load MISP module $moduleName fails", error) - Nil - case (_, Success(module)) ⇒ - logger.info(s"Register MISP module ${module.name} ${module.version}") - Seq(module) - } - } + logger.info(s"MISP modules is ${if (mispModulesEnabled) "enabled" else "disabled"}, loader is $loaderCommand") + + lazy val list: Seq[MispModule] = if (mispModulesEnabled) { + Json.parse(s"$loaderCommand --list".!!) + .as[Seq[String]] + .map { moduleName ⇒ + moduleName → (for { + moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!)) + module ← Try(moduleInfo.as[MispModule](reads(loaderCommand, mispModuleConfig))) + } yield module) + } + .flatMap { + case (moduleName, Failure(error)) ⇒ + logger.warn(s"Load MISP module $moduleName fails", error) + Nil + case (_, Success(module)) ⇒ + logger.info(s"Register MISP module ${module.name} ${module.version}") + Seq(module) + } + } + else Nil def get(moduleName: String): Option[MispModule] = list.find(_.name == moduleName) @@ -89,30 +97,31 @@ class MispSrv( } def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = { - loaderCommandOption - .flatMap { loaderCommand ⇒ - val artifact = toArtifact(mispType, data) - get(module) - .map { mispModule ⇒ - val mispReport = Future { - val input = Json.obj(mispType → data) - val output = (s"$loaderCommand --run $module" #< input.toString).!! - Json.parse(output).as[JsObject] - } - jobSrv.create(mispModule, artifact, mispReport.map(toReport)) - mispReport - + val artifact = toArtifact(mispType, data) + val mispModule = if (mispModulesEnabled) { + get(module) + .map { mispModule ⇒ + val mispReport = Future { + val input = Json.obj("config" → mispModule.config, mispType → data) + val output = (s"$loaderCommand --run $module" #< input.toString).!! + Json.parse(output).as[JsObject] } - .orElse { - externalAnalyzerSrv - .get(module) - .map { analyzer ⇒ - externalAnalyzerSrv.analyze(analyzer, artifact) - .map { report ⇒ toMispOutput(report) } - } + jobSrv.create(mispModule, artifact, mispReport.map(toReport)) + mispReport + + } + } + else None + mispModule + .orElse { + externalAnalyzerSrv + .get(module) + .map { analyzer ⇒ + externalAnalyzerSrv.analyze(analyzer, artifact) + .map { report ⇒ toMispOutput(report) } } } - .getOrElse(Future.failed(new Exception(s"Module $module not found"))) + .getOrElse(Future.failed(new Exception(s"Module $module not found"))) // TODO add appropriate exception } def analyze(module: MispModule, artifact: Artifact): Future[Report] = { @@ -121,10 +130,13 @@ class MispSrv( val input = artifact match { case DataArtifact(data, _) ⇒ - stringStream(Json.obj(dataType2mispType(artifact.dataType).head → data).toString) + val mispType = dataType2mispType(artifact.dataType) + .filter(module.inputAttributes.contains) + .head + stringStream((Json.obj("config" → module.config) + (mispType → JsString(data))).toString) case FileArtifact(data, _) ⇒ new SequenceInputStream(Iterator( - stringStream("""{"attachment":""""), + stringStream(Json.obj("config" → module.config).toString.replaceFirst("}$", ""","attachment":"""")), new Base64InputStream(new FileInputStream(data), true), stringStream("\"}")).asJavaEnumeration) } @@ -207,15 +219,26 @@ class MispSrv( else mispTypes } - private def reads(loaderCommand: String): Reads[MispModule] = + private def reads(loaderCommand: String, mispModuleConfig: JsObject): Reads[MispModule] = for { name ← (__ \ "name").read[String] - version ← (__ \ "meta" \ "version").read[String] - description ← (__ \ "meta" \ "description").read[String] - author ← (__ \ "meta" \ "author").read[String] - config ← (__ \ "meta" \ "config").read[Seq[String]] + version ← (__ \ "moduleinfo" \ "version").read[String] + description ← (__ \ "moduleinfo" \ "description").read[String] + author ← (__ \ "moduleinfo" \ "author").read[String] + config = (mispModuleConfig \ name).asOpt[JsObject].getOrElse(JsObject(Nil)) + requiredConfig ← (__ \ "config").read[Set[String]] + missingConfig = requiredConfig -- config.keys + _ ← if (missingConfig.nonEmpty) { + val message = s"MISP module $name is disabled because the following configuration " + + s"item${if (missingConfig.size > 1) "s are" else " is"} missing: ${missingConfig.mkString(", ")}" + logger.warn(message) + Reads[Unit](_ ⇒ JsError(message)) + } + else { + Reads[Unit](_ ⇒ JsSuccess(())) + } input ← (__ \ "mispattributes" \ "input").read[Seq[String]] - dataTypes = input.map(mispType2dataType) + dataTypes = input.map(mispType2dataType).distinct } yield MispModule(name, version, description, author, dataTypes, input, config, loaderCommand) private val typeLookup = Map( diff --git a/build.sbt b/build.sbt index 75bf94c82..2c82422c4 100644 --- a/build.sbt +++ b/build.sbt @@ -43,14 +43,21 @@ mappings in Universal ~= { file("package/cortex.service") -> "package/cortex.service", file("package/cortex.conf") -> "package/cortex.conf", file("package/cortex") -> "package/cortex", - file("package/logback.xml") -> "conf/logback.xml" + file("package/logback.xml") -> "conf/logback.xml", + file("contrib/misp-modules-loader.py") -> "contrib/misp-modules-loader.py" ) } // Package // -maintainer := "Thomas Franco val mappings = pm.mappings.filterNot { @@ -62,7 +69,7 @@ linuxPackageMappings ~= { _.map { pm => file("package/cortex.conf") -> "/etc/init/cortex.conf", file("package/cortex") -> "/etc/init.d/cortex", file("conf/application.sample") -> "/etc/cortex/application.conf", - file("conf/logback.xml") -> "/etc/cortex/logback.xml" + file("package/logback.xml") -> "/etc/cortex/logback.xml" ).withConfig() } @@ -125,7 +132,11 @@ dockerCommands ~= { dc => "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git && " + "cd /opt && " + "git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " + - "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt)"), + "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt) && " + + "apt-get install -y --no-install-recommends python3-setuptools python3-dev zlib1g-dev libxslt1-dev libxml2-dev libpq5 libjpeg-dev && git clone https://github.com/MISP/misp-modules.git && " + + "easy_install3 pip && " + + "(cd misp-modules && pip3 install -I -r REQUIREMENTS && pip3 install -I .) && " + + "rm -rf misp_modules /var/lib/apt/lists/*"), Cmd("ADD", "var", "/var"), Cmd("ADD", "etc", "/etc"), ExecCmd("RUN", "chown", "-R", "daemon:daemon", "/var/log/cortex")) ++ diff --git a/conf/application.sample b/conf/application.sample index bd499d47b..149437e4a 100644 --- a/conf/application.sample +++ b/conf/application.sample @@ -7,12 +7,6 @@ analyzer { path = "path/to/Cortex-Analyzers/analyzers" config { - global { - proxy { - #http="http://PROXY_ADDRESS:PORT", - #https="http://PROXY_ADDRESS:PORT" - } - } CIRCLPassiveDNS { #user= "..." #password= "..." @@ -79,4 +73,81 @@ analyzer { # Max number of threads available for analyze parallelism-max = 4 } -} \ No newline at end of file +} + +misp.modules { + enabled = true + + config { + shodan { + #apikey = "" + } + eupi { + #apikey = "" + #url = "" + } + passivetotal { + #username = "" + #api_key = "" + } + dns { + #nameserver = "" + } + whois { + #server = "" + #port = "" + } + sourcecache { + #archivepath = "" + } + geoip_country { + } + circl_passivessl { + #username = "" + #password = "" + } + iprep { + #apikey = "" + } + countrycode { + } + cve { + } + virustotal { + #apikey = "" + #event_limit = "" + } + ipasn { + #host = "" + #port = "" + #db = "" + } + circl_passivedns { + #username = "" + #password = "" + } + vmray_submit { + #apikey = "" + #url = "" + #shareable = "" + #do_not_reanalyze = "" + #do_not_include_vmrayjobids = "" + } + wiki { + } + domaintools { + #username = "" + #api_key = "" + } + reversedns { + #nameserver = "" + } + threatminer { + } + asn_history { + #host = "" + #port = "" + #db = "" + } + } +} diff --git a/conf/reference.conf b/conf/reference.conf index 8484a2429..229d9624d 100644 --- a/conf/reference.conf +++ b/conf/reference.conf @@ -1,6 +1,12 @@ # handler for errors (transform exception to related http status code play.http.errorHandler = services.ErrorHandler +# MISP modules loader location +misp.modules { + enabled = false + loader = ${play.server.dir}/"contrib/misp-modules-loader.py" +} + analyzer { # Directory that holds analyzers path = analyzers @@ -8,7 +14,7 @@ analyzer { config { dummy = dummy } - + fork-join-executor { # Min number of threads available for analyze parallelism-min = 2 diff --git a/contrib/misp-modules-loader.py b/contrib/misp-modules-loader.py index c265aaaf8..be5d1fea6 100755 --- a/contrib/misp-modules-loader.py +++ b/contrib/misp-modules-loader.py @@ -17,51 +17,61 @@ """ -def run(argv): +def usage(): + print(__file__ + " --list") + print(__file__ + " --info ") + print(__file__ + " --run ") + +def run(argv): mhandlers, modules = misp_modules.load_package_modules() try: - opts, args = getopt.getopt(argv, 'lh:i:r:', ["list", "help", "info=","run="]) + opts, args = getopt.getopt(argv, 'lh:i:r:', ["list", "help", "info=", "run="]) except getopt.GetoptError as err: - print(__file__ + " --info ") - print(__file__ + " --run ") + usage() print(str(err)) sys.exit(2) - module = None - path = None - for opt,arg in opts: + for opt, arg in opts: # TODO: check if module exist else exit if opt in ('-h', '--help'): - print(__file__ + " --info ") - print(__file__ + " --run ") + usage() sys.exit() elif opt in ('-l', '--list'): - modules = [m for m in modules if mhandlers['type:' + m ] == "expansion"] + modules = [m for m in modules if mhandlers['type:' + m] == "expansion"] print(json.dumps(modules)) sys.exit(0) elif opt in ('-r', '--run'): - module = arg + module_name = arg + try: data = json.load(sys.stdin) - print(json.dumps(mhandlers[module].handler(json.dumps(data)))) - sys.exit(0) - - elif opt in ('-i','--info'): - module = arg - - print(json.dumps({'name': module, 'mispattributes': mhandlers[module].mispattributes, - 'moduleinfo':mhandlers[module].moduleinfo})) + print(json.dumps(mhandlers[module_name].handler(json.dumps(data)))) + except: + error = {'error': sys.exc_info()[1].args[0]} + print(json.dumps(error)) + sys.exit(0) + elif opt in ('-i', '--info'): + module_name = arg + try: + config = mhandlers[module_name].moduleconfig + except AttributeError: + config = [] + print(json.dumps({ + 'name': module_name, + 'mispattributes': mhandlers[module_name].mispattributes, + 'moduleinfo': mhandlers[module_name].moduleinfo, + 'config': config + })) if __name__ == '__main__': if len(sys.argv[1:]) > 0: run(sys.argv[1:]) else: - print(__file__ + " --info ") - print(__file__ + " --run ") + usage() sys.exit(2) diff --git a/docs/README.md b/docs/README.md index f8eb06819..6a6eb8c27 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,4 +28,5 @@ Once you have installed Cortex, you need to [install the analyzers](installation - [How to create an analyzer](api/how-to-create-an-analyzer.md) ## Other +- [MISP integration](misp.md) - [FAQ](FAQ.md) diff --git a/docs/installation/docker-guide.md b/docs/installation/docker-guide.md index 13696c543..1cbd71fa8 100644 --- a/docs/installation/docker-guide.md +++ b/docs/installation/docker-guide.md @@ -46,5 +46,6 @@ Docker image accepts more options: - --no-config-secret : do not add random secret to configuration - --secret : secret to secure sessions - --analyzer-path : where analyzers are located + - --no-misp-modules : disabled MISP modules diff --git a/docs/misp.md b/docs/misp.md new file mode 100644 index 000000000..cee49840d --- /dev/null +++ b/docs/misp.md @@ -0,0 +1,56 @@ +# MISP integration + +## Invoke MISP modules in Cortex + +Since version 1.1.1, Cortex can analyze observable using +[MISP expansion modules](https://github.com/MISP/misp-modules#expansion-modules). + +Follow [MISP documentation](https://github.com/MISP/misp-modules#how-to-install-and-start-misp-modules) to install MISP +modules. MISP modules service doesn't need to be started. Modules must be present in the same host than Cortex. +``` +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev +cd /usr/local/src/ +sudo git clone https://github.com/MISP/misp-modules.git +cd misp-modules +sudo pip3 install -I -r REQUIREMENTS +sudo pip3 install -I . +``` + +Integration with MISP modules can then be enabled by adding the line `misp.modules.enabled = true` in +Cortex `application.conf`. + +Most MISP modules require configuration. Settings must be placed in misp.modules.config key. If required some +configuration is missing, MISP module is not loaded. + + +``` +misp.modules { + enabled = true + + config { + shodan { + apikey = "" + } + dns { + nameserver = "127.0.0.1" + } + } +} +``` +Cortex uses Python wrapper to run MISP modules. It is located in `contrib` folder. Cortex should be able to locate it +automatically. You can force its location in configuraton under settings: +``` +misp.modules.loader = /path/to/misp-modules-loader.py" +``` + +## Invoke Cortex in MISP + +Cortex can be connected to a MISP instance. Under `Server settings` of MISP `Administration` menu, go to `Plugin +settings` and in Cortex section: + - set `Plugin.Cortex_services_enable` to `true` + - set `Plugin.Cortex_services_url` to `http://127.0.0.1` (replace 127.0.0.1 by Cortex IP address) + - set `Plugin.Plugin.Cortex_services_port` to `9000` (replace 9000 by Cortex port) + +Then Cortex analyzer list should appear in Cortex section. They must be enabled before being available to MISP users. + + \ No newline at end of file diff --git a/package/docker/entrypoint b/package/docker/entrypoint index 08f2fd167..fd2e5a721 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -4,6 +4,7 @@ CONFIG_SECRET=1 CONFIG=1 CONFIG_FILE=/etc/cortex/application.conf ANALYZER_PATH=/opt/Cortex-Analyzers/analyzers +MISP_MODULE=1 function usage { cat <<- _EOF_ @@ -12,6 +13,7 @@ function usage { --no-config-secret | do not add random secret to configuration --secret | secret to secure sessions --analyzer-path | where analyzers are located + --no-misp-modules | disable MISP modules _EOF_ exit 1 } @@ -24,6 +26,7 @@ do "--no-config-secret") CONFIG_SECRET=0;; "--secret") shift; SECRET=$1;; "--analyzer-path") shift; ANALYZER_PATH=$1;; + "--no-misp-modules") shift; MISP_MODULE=0;; "--") STOP=1;; *) usage esac @@ -45,6 +48,11 @@ then echo analyzer.path=\"$ANALYZER_PATH\" >> $CONFIG_FILE + if test $MISP_MODULE = 1 + then + echo 'misp.modules.enabled = true' >> $CONFIG_FILE + fi + echo 'include file("/etc/cortex/application.conf")' >> $CONFIG_FILE fi