diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0e1971f..3a1a27d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,28 @@ # Change Log -## [2.1.3](https://github.com/TheHive-Project/Cortex/tree/2.1.3) +## [3.0.0-RC1](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC1) (2019-04-05) +[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.3...3.0.0-RC1) + +**Implemented enhancements:** + +- Remove size limitations [\#178](https://github.com/TheHive-Project/Cortex/issues/178) +- Collapse job error messages by default in job history [\#171](https://github.com/TheHive-Project/Cortex/issues/171) +- Update Copyright with year 2019 [\#168](https://github.com/TheHive-Project/Cortex/issues/168) + +**Fixed bugs:** + +- SSO: Authentication module not found [\#181](https://github.com/TheHive-Project/Cortex/issues/181) +- Akka Dispatcher Blocked [\#170](https://github.com/TheHive-Project/Cortex/issues/170) + +**Closed issues:** + +- Use files to communicate with analyzer/responder [\#176](https://github.com/TheHive-Project/Cortex/issues/176) +- Provide analyzers and responders packaged with docker [\#175](https://github.com/TheHive-Project/Cortex/issues/175) +- Single sign-on support for Cortex [\#165](https://github.com/TheHive-Project/Cortex/issues/165) +- File extraction [\#120](https://github.com/TheHive-Project/Cortex/issues/120) + +## [2.1.3](https://github.com/TheHive-Project/Cortex/tree/2.1.3) (2018-12-20) [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.2...2.1.3) **Implemented enhancements:** diff --git a/app/org/thp/cortex/Module.scala b/app/org/thp/cortex/Module.scala index ae2d1a0e7..f025e9d6b 100644 --- a/app/org/thp/cortex/Module.scala +++ b/app/org/thp/cortex/Module.scala @@ -1,20 +1,23 @@ package org.thp.cortex + import com.google.inject.AbstractModule import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder } import play.api.libs.concurrent.AkkaGuiceSupport import play.api.{ Configuration, Environment, Logger, Mode } - import scala.collection.JavaConverters._ + import com.google.inject.name.Names import org.reflections.Reflections import org.reflections.scanners.SubTypesScanner import org.reflections.util.ConfigurationBuilder import org.thp.cortex.models.{ AuditedModel, Migration } -import org.thp.cortex.services.{ AuditActor, CortexAuthSrv, UserSrv } +import org.thp.cortex.services._ + import org.elastic4play.models.BaseModelDef import org.elastic4play.services.auth.MultiAuthSrv import org.elastic4play.services.{ AuthSrv, MigrationOperations } import org.thp.cortex.controllers.{ AssetCtrl, AssetCtrlDev, AssetCtrlProd } +import services.mappers.{ MultiUserMapperSrv, UserMapper } class Module(environment: Environment, configuration: Configuration) extends AbstractModule with ScalaModule with AkkaGuiceSupport { @@ -50,9 +53,18 @@ class Module(environment: Environment, configuration: Configuration) extends Abs .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers) || c.isMemberClass) .filterNot(c ⇒ c == classOf[MultiAuthSrv] || c == classOf[CortexAuthSrv]) .foreach { authSrvClass ⇒ + logger.info(s"Loading authentication module $authSrvClass") authBindings.addBinding.to(authSrvClass) } + val ssoMapperBindings = ScalaMultibinder.newSetBinder[UserMapper](binder) + reflectionClasses + .getSubTypesOf(classOf[UserMapper]) + .asScala + .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers) || c.isMemberClass) + .filterNot(c ⇒ c == classOf[MultiUserMapperSrv]) + .foreach(mapperCls ⇒ ssoMapperBindings.addBinding.to(mapperCls)) + if (environment.mode == Mode.Prod) bind[AssetCtrl].to[AssetCtrlProd] else @@ -60,6 +72,7 @@ class Module(environment: Environment, configuration: Configuration) extends Abs bind[org.elastic4play.services.UserSrv].to[UserSrv] bind[Int].annotatedWith(Names.named("databaseVersion")).toInstance(models.modelVersion) + bind[UserMapper].to[MultiUserMapperSrv] bind[AuthSrv].to[CortexAuthSrv] bind[MigrationOperations].to[Migration] diff --git a/app/org/thp/cortex/controllers/AnalyzerCtrl.scala b/app/org/thp/cortex/controllers/AnalyzerCtrl.scala index bb636f65a..a6ce328b8 100644 --- a/app/org/thp/cortex/controllers/AnalyzerCtrl.scala +++ b/app/org/thp/cortex/controllers/AnalyzerCtrl.scala @@ -1,18 +1,16 @@ package org.thp.cortex.controllers -import javax.inject.{ Inject, Singleton } import scala.concurrent.{ ExecutionContext, Future } -import play.api.libs.json.{ JsNumber, JsObject, JsString, Json } +import play.api.libs.json.{ JsObject, JsString, Json } import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } import akka.stream.Materializer -import akka.stream.scaladsl.Sink -import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition } +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models.{ Roles, Worker } import org.thp.cortex.services.{ UserSrv, WorkerSrv } import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } -import org.elastic4play.models.JsonFormat.baseModelEntityWrites import org.elastic4play.services.JsonFormat.queryReads import org.elastic4play.services.{ QueryDSL, QueryDef } @@ -33,75 +31,42 @@ class AnalyzerCtrl @Inject() ( val sort = request.body.getStrings("sort").getOrElse(Nil) val isAdmin = request.roles.contains(Roles.orgAdmin) val (analyzers, analyzerTotal) = workerSrv.findAnalyzersForUser(request.userId, query, range, sort) - val enrichedAnalyzers = analyzers.mapAsync(2)(analyzerJson(isAdmin)) - renderer.toOutput(OK, enrichedAnalyzers, analyzerTotal) + renderer.toOutput(OK, analyzers.map(analyzerJson(isAdmin)), analyzerTotal) } def get(analyzerId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ val isAdmin = request.roles.contains(Roles.orgAdmin) workerSrv.getForUser(request.userId, analyzerId) - .flatMap(analyzerJson(isAdmin)) - .map(renderer.toOutput(OK, _)) - } - - private val emptyAnalyzerDefinitionJson = Json.obj( - "version" → "0.0", - "description" → "unknown", - "dataTypeList" → Nil, - "author" → "unknown", - "url" → "unknown", - "license" → "unknown") - - private def analyzerJson(analyzer: Worker, analyzerDefinition: Option[WorkerDefinition]) = { - analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad ⇒ - Json.obj( - "maxTlp" → (analyzer.config \ "max_tlp").asOpt[JsNumber], - "maxPap" → (analyzer.config \ "max_pap").asOpt[JsNumber], - "version" → ad.version, - "description" → ad.description, - "author" → ad.author, - "url" → ad.url, - "license" → ad.license, - "baseConfig" → ad.baseConfiguration) - } + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId())) // For compatibility reason + .map(a ⇒ renderer.toOutput(OK, analyzerJson(isAdmin)(a))) } - private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): Future[JsObject] = { - workerSrv.getDefinition(analyzer.workerDefinitionId()) - .map(analyzerDefinition ⇒ analyzerJson(analyzer, Some(analyzerDefinition))) - .recover { case _ ⇒ analyzerJson(analyzer, None) } - .map { - case a if isAdmin ⇒ a + ("configuration" → Json.parse(analyzer.configuration())) - case a ⇒ a - } + private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): JsObject = { + if (isAdmin) + analyzer.toJson + ("configuration" → Json.parse(analyzer.configuration())) + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId())) + else + analyzer.toJson + ("analyzerDefinitionId" → JsString(analyzer.workerDefinitionId())) } def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ import org.elastic4play.services.QueryDSL._ - workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) - ._1 - .mapAsyncUnordered(2) { analyzer ⇒ - workerSrv.getDefinition(analyzer.workerDefinitionId()) - .map(ad ⇒ analyzerJson(analyzer, Some(ad))) - } - .runWith(Sink.seq) - .map(analyzers ⇒ renderer.toOutput(OK, analyzers)) + val (responderList, responderCount) = workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) + renderer.toOutput(OK, responderList.map(analyzerJson(isAdmin = false)), responderCount) } def create(analyzerDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ for { organizationId ← userSrv.getOrganizationId(request.userId) - workerDefinition ← workerSrv.getDefinition(analyzerDefinitionId) + workerDefinition ← Future.fromTry(workerSrv.getDefinition(analyzerDefinitionId)) analyzer ← workerSrv.create(organizationId, workerDefinition, request.body) - } yield renderer.toOutput(CREATED, analyzerJson(analyzer, Some(workerDefinition))) + } yield renderer.toOutput(CREATED, analyzerJson(isAdmin = false)(analyzer)) } - def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒ + def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { _ ⇒ val (analyzers, analyzerTotal) = workerSrv.listAnalyzerDefinitions renderer.toOutput(OK, analyzers, analyzerTotal) } - def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒ + def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { _ ⇒ workerSrv.rescan() NoContent } @@ -117,7 +82,6 @@ class AnalyzerCtrl @Inject() ( for { analyzer ← workerSrv.getForUser(request.userId, analyzerId) updatedAnalyzer ← workerSrv.update(analyzer, request.body) - updatedAnalyzerJson ← analyzerJson(isAdmin = true)(updatedAnalyzer) - } yield renderer.toOutput(OK, updatedAnalyzerJson) + } yield renderer.toOutput(OK, analyzerJson(isAdmin = true)(updatedAnalyzer)) } } \ No newline at end of file diff --git a/app/org/thp/cortex/controllers/AuthenticationCtrl.scala b/app/org/thp/cortex/controllers/AuthenticationCtrl.scala index accca52d8..6eb37af00 100644 --- a/app/org/thp/cortex/controllers/AuthenticationCtrl.scala +++ b/app/org/thp/cortex/controllers/AuthenticationCtrl.scala @@ -1,18 +1,18 @@ package org.thp.cortex.controllers -import javax.inject.{ Inject, Singleton } - import scala.concurrent.{ ExecutionContext, Future } import play.api.mvc._ +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models.UserStatus import org.thp.cortex.services.UserSrv import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.database.DBIndex import org.elastic4play.services.AuthSrv -import org.elastic4play.{ MissingAttributeError, Timed } import org.elastic4play.services.JsonFormat.authContextWrites +import org.elastic4play.{ AuthorizationError, MissingAttributeError, OAuth2Redirect, Timed } @Singleton class AuthenticationCtrl @Inject() ( @@ -38,6 +38,27 @@ class AuthenticationCtrl @Inject() ( } } + @Timed + def ssoLogin: Action[AnyContent] = Action.async { implicit request ⇒ + dbIndex.getIndexStatus.flatMap { + case false ⇒ Future.successful(Results.Status(520)) + case _ ⇒ + (for { + authContext ← authSrv.authenticate() + user ← userSrv.get(authContext.userId) + } yield { + if (user.status() == UserStatus.Ok) + authenticated.setSessingUser(Ok, authContext) + else + throw AuthorizationError("Your account is locked") + }) recover { + // A bit of a hack with the status code, so that Angular doesn't reject the origin + case OAuth2Redirect(redirectUrl, qp) ⇒ Redirect(redirectUrl, qp, status = OK) + case e ⇒ throw e + } + } + } + @Timed def logout = Action { Ok.withNewSession diff --git a/app/org/thp/cortex/controllers/JobCtrl.scala b/app/org/thp/cortex/controllers/JobCtrl.scala index feab4b916..7e49eeb22 100644 --- a/app/org/thp/cortex/controllers/JobCtrl.scala +++ b/app/org/thp/cortex/controllers/JobCtrl.scala @@ -1,31 +1,31 @@ package org.thp.cortex.controllers -import javax.inject.{ Inject, Named, Singleton } +import scala.concurrent.duration.{ Duration, FiniteDuration } +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.http.Status +import play.api.libs.json.{ JsObject, JsString, JsValue, Json } +import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } import akka.actor.{ ActorRef, ActorSystem } import akka.pattern.ask import akka.stream.Materializer import akka.stream.scaladsl.Sink import akka.util.Timeout +import javax.inject.{ Inject, Named, Singleton } +import org.thp.cortex.models.{ Job, JobStatus, Roles } +import org.thp.cortex.services.AuditActor.{ JobEnded, Register } +import org.thp.cortex.services.JobSrv + import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites import org.elastic4play.services.JsonFormat.queryReads import org.elastic4play.services.{ QueryDSL, QueryDef } import org.elastic4play.utils.RichFuture -import org.thp.cortex.models.{ Job, JobStatus, Roles } -import org.thp.cortex.services.AuditActor.{ JobEnded, Register } -import org.thp.cortex.services.{ JobSrv, UserSrv } -import play.api.http.Status -import play.api.libs.json.{ JsObject, JsString, JsValue, Json } -import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } - -import scala.concurrent.duration.{ Duration, FiniteDuration } -import scala.concurrent.{ ExecutionContext, Future } @Singleton class JobCtrl @Inject() ( jobSrv: JobSrv, - userSrv: UserSrv, @Named("audit") auditActor: ActorRef, fieldsBodyParser: FieldsBodyParser, authenticated: Authenticated, diff --git a/app/org/thp/cortex/controllers/ResponderCtrl.scala b/app/org/thp/cortex/controllers/ResponderCtrl.scala index 3a13f452e..9e5246445 100644 --- a/app/org/thp/cortex/controllers/ResponderCtrl.scala +++ b/app/org/thp/cortex/controllers/ResponderCtrl.scala @@ -6,7 +6,6 @@ import play.api.libs.json.{ JsNumber, JsObject, Json } import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } import akka.stream.Materializer -import akka.stream.scaladsl.Sink import javax.inject.{ Inject, Singleton } import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition } import org.thp.cortex.services.{ UserSrv, WorkerSrv } @@ -33,15 +32,13 @@ class ResponderCtrl @Inject() ( val sort = request.body.getStrings("sort").getOrElse(Nil) val isAdmin = request.roles.contains(Roles.orgAdmin) val (responders, responderTotal) = workerSrv.findRespondersForUser(request.userId, query, range, sort) - val enrichedResponders = responders.mapAsync(2)(responderJson(isAdmin)) - renderer.toOutput(OK, enrichedResponders, responderTotal) + renderer.toOutput(OK, responders.map(responderJson(isAdmin)), responderTotal) } def get(responderId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ val isAdmin = request.roles.contains(Roles.orgAdmin) workerSrv.getForUser(request.userId, responderId) - .flatMap(responderJson(isAdmin)) - .map(renderer.toOutput(OK, _)) + .map(responder ⇒ renderer.toOutput(OK, responderJson(isAdmin)(responder))) } private val emptyResponderDefinitionJson = Json.obj( @@ -66,42 +63,33 @@ class ResponderCtrl @Inject() ( } } - private def responderJson(isAdmin: Boolean)(responder: Worker): Future[JsObject] = { - workerSrv.getDefinition(responder.workerDefinitionId()) - .map(responderDefinition ⇒ responderJson(responder, Some(responderDefinition))) - .recover { case _ ⇒ responderJson(responder, None) } - .map { - case a if isAdmin ⇒ a + ("configuration" → Json.parse(responder.configuration())) - case a ⇒ a - } + private def responderJson(isAdmin: Boolean)(responder: Worker): JsObject = { + if (isAdmin) + responder.toJson + ("configuration" → Json.parse(responder.configuration())) + else + responder.toJson } def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ import org.elastic4play.services.QueryDSL._ - workerSrv.findRespondersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) - ._1 - .mapAsyncUnordered(2) { responder ⇒ - workerSrv.getDefinition(responder.workerDefinitionId()) - .map(ad ⇒ responderJson(responder, Some(ad))) - } - .runWith(Sink.seq) - .map(responders ⇒ renderer.toOutput(OK, responders)) + val (responderList, responderCount) = workerSrv.findRespondersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) + renderer.toOutput(OK, responderList.map(responderJson(false)), responderCount) } def create(responderDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ for { organizationId ← userSrv.getOrganizationId(request.userId) - workerDefinition ← workerSrv.getDefinition(responderDefinitionId) + workerDefinition ← Future.fromTry(workerSrv.getDefinition(responderDefinitionId)) responder ← workerSrv.create(organizationId, workerDefinition, request.body) } yield renderer.toOutput(CREATED, responderJson(responder, Some(workerDefinition))) } - def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒ + def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { _ ⇒ val (responders, responderTotal) = workerSrv.listResponderDefinitions renderer.toOutput(OK, responders, responderTotal) } - def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒ + def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { _ ⇒ workerSrv.rescan() NoContent } @@ -117,7 +105,6 @@ class ResponderCtrl @Inject() ( for { responder ← workerSrv.getForUser(request.userId, responderId) updatedResponder ← workerSrv.update(responder, request.body) - updatedResponderJson ← responderJson(isAdmin = true)(updatedResponder) - } yield renderer.toOutput(OK, updatedResponderJson) + } yield renderer.toOutput(OK, responderJson(isAdmin = true)(updatedResponder)) } } \ No newline at end of file diff --git a/app/org/thp/cortex/controllers/StatusCtrl.scala b/app/org/thp/cortex/controllers/StatusCtrl.scala index 0c0e25be1..165a1b492 100644 --- a/app/org/thp/cortex/controllers/StatusCtrl.scala +++ b/app/org/thp/cortex/controllers/StatusCtrl.scala @@ -1,12 +1,11 @@ package org.thp.cortex.controllers import javax.inject.{ Inject, Singleton } - import scala.concurrent.ExecutionContext import play.api.Configuration import play.api.http.Status -import play.api.libs.json.{ JsString, Json } +import play.api.libs.json.{ JsBoolean, JsString, Json } import play.api.libs.json.Json.toJsFieldJsValueWrapper import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } @@ -42,7 +41,8 @@ class StatusCtrl @Inject() ( case multiAuthSrv: MultiAuthSrv ⇒ multiAuthSrv.authProviders.map { a ⇒ JsString(a.name) } case _ ⇒ JsString(authSrv.name) }), - "capabilities" → authSrv.capabilities.map(c ⇒ JsString(c.toString))))) + "capabilities" → authSrv.capabilities.map(c ⇒ JsString(c.toString)), + "ssoAutoLogin" → JsBoolean(configuration.getOptional[Boolean]("auth.sso.autologin").getOrElse(false))))) } } diff --git a/app/org/thp/cortex/models/Artifact.scala b/app/org/thp/cortex/models/Artifact.scala index f199ddabf..b8a7b108d 100644 --- a/app/org/thp/cortex/models/Artifact.scala +++ b/app/org/thp/cortex/models/Artifact.scala @@ -8,7 +8,7 @@ import org.elastic4play.models.{ AttributeDef, EntityDef, AttributeFormat ⇒ F, trait ArtifactAttributes { _: AttributeDef ⇒ val dataType = attribute("dataType", F.stringFmt, "Type of the artifact", O.readonly) - val data = optionalAttribute("data", F.stringFmt, "Content of the artifact", O.readonly) + val data = optionalAttribute("data", F.rawFmt, "Content of the artifact", O.readonly) val attachment = optionalAttribute("attachment", F.attachmentFmt, "Artifact file content", O.readonly) val tlp = attribute("tlp", TlpAttributeFormat, "TLP level", 2L) val tags = multiAttribute("tags", F.stringFmt, "Artifact tags") diff --git a/app/org/thp/cortex/models/BaseConfig.scala b/app/org/thp/cortex/models/BaseConfig.scala index 0c8fd98cf..389ea2c25 100644 --- a/app/org/thp/cortex/models/BaseConfig.scala +++ b/app/org/thp/cortex/models/BaseConfig.scala @@ -1,5 +1,8 @@ package org.thp.cortex.models +import scala.concurrent.duration.Duration + +import play.api.Configuration import play.api.libs.json._ import org.elastic4play.utils.Collection.distinctBy @@ -7,6 +10,7 @@ import org.elastic4play.utils.Collection.distinctBy case class BaseConfig(name: String, workerNames: Seq[String], items: Seq[ConfigurationDefinitionItem], config: Option[WorkerConfig]) { def +(other: BaseConfig) = BaseConfig(name, workerNames ++ other.workerNames, distinctBy(items ++ other.items)(_.name), config.orElse(other.config)) } + object BaseConfig { implicit val writes: Writes[BaseConfig] = Writes[BaseConfig] { baseConfig ⇒ Json.obj( @@ -15,15 +19,18 @@ object BaseConfig { "configurationItems" → baseConfig.items, "config" → baseConfig.config.fold(JsObject.empty)(_.jsonConfig)) } - def global(tpe: WorkerType.Type) = { + def global(tpe: WorkerType.Type, configuration: Configuration): BaseConfig = { val typedItems = tpe match { case WorkerType.responder ⇒ Nil case WorkerType.analyzer ⇒ Seq( - ConfigurationDefinitionItem("auto_extract_artifacts", "extract artifacts from full report automatically", WorkerConfigItemType.boolean, multi = false, required = false, Some(JsFalse))) + ConfigurationDefinitionItem("auto_extract_artifacts", "extract artifacts from full report automatically", WorkerConfigItemType.boolean, multi = false, required = false, Some(JsFalse)), + ConfigurationDefinitionItem("jobCache", "maximum time, in minutes, previous result is used if similar job is requested", WorkerConfigItemType.number, multi = false, required = false, configuration.getOptional[Duration]("cache.job").map(d ⇒ JsNumber(d.toMinutes)))) } BaseConfig("global", Nil, typedItems ++ Seq( ConfigurationDefinitionItem("proxy_http", "url of http proxy", WorkerConfigItemType.string, multi = false, required = false, None), - ConfigurationDefinitionItem("proxy_https", "url of https proxy", WorkerConfigItemType.string, multi = false, required = false, None)), + ConfigurationDefinitionItem("proxy_https", "url of https proxy", WorkerConfigItemType.string, multi = false, required = false, None), + ConfigurationDefinitionItem("cacerts", "certificate authorities", WorkerConfigItemType.text, multi = false, required = false, None), + ConfigurationDefinitionItem("jobTimeout", "maximum allowed job execution time (in minutes)", WorkerConfigItemType.number, multi = false, required = false, configuration.getOptional[Duration]("job.timeout").map(d ⇒ JsNumber(d.toMinutes)))), None) } val tlp = BaseConfig("tlp", Nil, Seq( diff --git a/app/org/thp/cortex/models/Job.scala b/app/org/thp/cortex/models/Job.scala index 9462202bd..183949676 100644 --- a/app/org/thp/cortex/models/Job.scala +++ b/app/org/thp/cortex/models/Job.scala @@ -25,14 +25,14 @@ trait JobAttributes { val startDate = optionalAttribute("startDate", F.dateFmt, "Analysis start date") val endDate = optionalAttribute("endDate", F.dateFmt, "Analysis end date") val dataType = attribute("dataType", F.stringFmt, "Type of the artifact", O.readonly) - val data = optionalAttribute("data", F.stringFmt, "Content of the artifact", O.readonly) + val data = optionalAttribute("data", F.rawFmt, "Content of the artifact", O.readonly) val attachment = optionalAttribute("attachment", F.attachmentFmt, "Artifact file content", O.readonly) val tlp = attribute("tlp", TlpAttributeFormat, "TLP level", 2L) val pap = attribute("pap", TlpAttributeFormat, "PAP level", 2L) val message = optionalAttribute("message", F.textFmt, "Message associated to the analysis") val errorMessage = optionalAttribute("message", F.textFmt, "Message returned by the worker when it fails") - val parameters = attribute("parameters", F.stringFmt, "Parameters for this job", "{}") - val input = optionalAttribute("input", F.textFmt, "Data sent to worker") + val parameters = attribute("parameters", F.rawFmt, "Parameters for this job", "{}") + val input = optionalAttribute("input", F.rawFmt, "Data sent to worker") val fromCache = optionalAttribute("fromCache", F.booleanFmt, "Indicates if cache is used", O.form) val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly) val lbel = optionalAttribute("label", F.stringFmt, "Label of the job") diff --git a/app/org/thp/cortex/models/Migration.scala b/app/org/thp/cortex/models/Migration.scala index ac363dc61..17f16d408 100644 --- a/app/org/thp/cortex/models/Migration.scala +++ b/app/org/thp/cortex/models/Migration.scala @@ -4,9 +4,10 @@ import javax.inject.{ Inject, Singleton } import scala.concurrent.{ ExecutionContext, Future } import scala.util.Success -import play.api.libs.json.{ JsString, Json } +import play.api.Logger +import play.api.libs.json.{ JsNull, JsString, JsValue, Json } -import org.thp.cortex.services.{ OrganizationSrv, UserSrv } +import org.thp.cortex.services.{ OrganizationSrv, UserSrv, WorkerSrv } import org.elastic4play.controllers.Fields import org.elastic4play.services.Operation._ @@ -17,8 +18,10 @@ import org.elastic4play.utils.Hasher class Migration @Inject() ( userSrv: UserSrv, organizationSrv: OrganizationSrv, + workerSrv: WorkerSrv, implicit val ec: ExecutionContext) extends MigrationOperations { + lazy val logger = Logger(getClass) def beginMigration(version: Int): Future[Unit] = Future.successful(()) def endMigration(version: Int): Future[Unit] = { @@ -27,7 +30,7 @@ class Migration @Inject() ( "name" → "cortex", "description" → "Default organization", "status" → "Active"))) - .transform { case _ ⇒ Success(()) } // ignore errors (already exist) + .transform(_ ⇒ Success(())) // ignore errors (already exist) } } @@ -61,6 +64,24 @@ class Migration @Inject() ( renameEntity("analyzerConfig", "workerConfig"), addAttribute("workerConfig", "type" → JsString(WorkerType.analyzer.toString))) - case _ ⇒ Nil + case DatabaseState(2) ⇒ + Seq(mapEntity("worker") { worker ⇒ + val definitionId = (worker \ "workerDefinitionId").asOpt[String] + definitionId + .flatMap(workerSrv.getDefinition(_).toOption) + .fold { + logger.warn(s"no definition found for worker ${definitionId.getOrElse(worker)}. You should probably have to disable and re-enable it") + worker + } { definition ⇒ + worker + + ("version" -> JsString(definition.version)) + + ("author" -> JsString(definition.author)) + + ("url" -> JsString(definition.url)) + + ("license" -> JsString(definition.license)) + + ("command" -> definition.command.fold[JsValue](JsNull)(c ⇒ JsString(c.toString))) + + ("dockerImage" -> definition.dockerImage.fold[JsValue](JsNull)(JsString.apply)) + + ("baseConfig" -> definition.baseConfiguration.fold[JsValue](JsNull)(JsString.apply)) + } + }) } } diff --git a/app/org/thp/cortex/models/Report.scala b/app/org/thp/cortex/models/Report.scala index 15e03c0a3..eff2f1f26 100644 --- a/app/org/thp/cortex/models/Report.scala +++ b/app/org/thp/cortex/models/Report.scala @@ -7,9 +7,9 @@ import play.api.libs.json.JsObject import org.elastic4play.models.{ AttributeDef, EntityDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, ChildModelDef } trait ReportAttributes { _: AttributeDef ⇒ - val full = attribute("full", F.textFmt, "Full content of the report", O.readonly) - val summary = attribute("summary", F.textFmt, "Summary of the report", O.readonly) - val operations = attribute("operations", F.textFmt, "Update operations applied at the end of the job", "[]", O.unaudited) + val full = attribute("full", F.rawFmt, "Full content of the report", O.readonly) + val summary = attribute("summary", F.rawFmt, "Summary of the report", O.readonly) + val operations = attribute("operations", F.rawFmt, "Update operations applied at the end of the job", "[]", O.unaudited) } @Singleton diff --git a/app/org/thp/cortex/models/User.scala b/app/org/thp/cortex/models/User.scala index 81710057d..6a374c968 100644 --- a/app/org/thp/cortex/models/User.scala +++ b/app/org/thp/cortex/models/User.scala @@ -21,8 +21,8 @@ trait UserAttributes { _: AttributeDef ⇒ val roles = multiAttribute("roles", RoleAttributeFormat, "Comma separated role list (READ, WRITE and ADMIN)") val status = attribute("status", F.enumFmt(UserStatus), "Status of the user", UserStatus.Ok) val password = optionalAttribute("password", F.stringFmt, "Password", O.sensitive, O.unaudited) - val avatar = optionalAttribute("avatar", F.stringFmt, "Base64 representation of user avatar image", O.unaudited) - val preferences = attribute("preferences", F.stringFmt, "User preferences", "{}", O.sensitive, O.unaudited) + val avatar = optionalAttribute("avatar", F.rawFmt, "Base64 representation of user avatar image", O.unaudited) + val preferences = attribute("preferences", F.rawFmt, "User preferences", "{}", O.sensitive, O.unaudited) val organization = attribute("organization", F.stringFmt, "User organization") } diff --git a/app/org/thp/cortex/models/Worker.scala b/app/org/thp/cortex/models/Worker.scala index 5caf6d679..715a8aa07 100644 --- a/app/org/thp/cortex/models/Worker.scala +++ b/app/org/thp/cortex/models/Worker.scala @@ -28,13 +28,21 @@ object WorkerType extends Enumeration with HiveEnumeration { trait WorkerAttributes { _: AttributeDef ⇒ val workerId = attribute("_id", F.stringFmt, "Worker id", O.model) val name = attribute("name", F.stringFmt, "Worker name") + val vers = attribute("version", F.stringFmt, "Worker version", O.readonly) val workerDefinitionId = attribute("workerDefinitionId", F.stringFmt, "Worker definition id", O.readonly) - val description = attribute("description", F.textFmt, "Worker description") + val description = attribute("description", F.textFmt, "Worker description", O.readonly) + val author = attribute("author", F.textFmt, "Worker author", O.readonly) + val url = attribute("url", F.textFmt, "Worker url", O.readonly) + val license = attribute("license", F.textFmt, "Worker license", O.readonly) + val command = optionalAttribute("command", F.textFmt, "Worker command", O.readonly) + val dockerImage = optionalAttribute("dockerImage", F.textFmt, "Worker docker image", O.readonly) val dataTypeList = multiAttribute("dataTypeList", F.stringFmt, "List of data type this worker can manage") - val configuration = attribute("configuration", F.textFmt, "Configuration of the worker", O.sensitive) + val configuration = attribute("configuration", F.rawFmt, "Configuration of the worker", O.sensitive) + val baseConfig = attribute("baseConfig", F.stringFmt, "Base configuration key", O.readonly) val rate = optionalAttribute("rate", F.numberFmt, "Number ") val rateUnit = optionalAttribute("rateUnit", F.enumFmt(RateUnit), "") val jobCache = optionalAttribute("jobCache", F.numberFmt, "") + val jobTimeout = optionalAttribute("jobTimeout", F.numberFmt, "") val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly) } diff --git a/app/org/thp/cortex/models/WorkerConfig.scala b/app/org/thp/cortex/models/WorkerConfig.scala index 75dd67336..84d5a26f4 100644 --- a/app/org/thp/cortex/models/WorkerConfig.scala +++ b/app/org/thp/cortex/models/WorkerConfig.scala @@ -10,7 +10,7 @@ import org.thp.cortex.models.JsonFormat.workerTypeFormat trait WorkerConfigAttributes { _: AttributeDef ⇒ val name = attribute("name", F.stringFmt, "Worker name") - val config = attribute("config", F.textFmt, "Configuration of worker", O.sensitive) + val config = attribute("config", F.rawFmt, "Configuration of worker", O.sensitive) val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly) } diff --git a/app/org/thp/cortex/models/WorkerDefinition.scala b/app/org/thp/cortex/models/WorkerDefinition.scala index afced08eb..e209cfe11 100644 --- a/app/org/thp/cortex/models/WorkerDefinition.scala +++ b/app/org/thp/cortex/models/WorkerDefinition.scala @@ -1,8 +1,6 @@ package org.thp.cortex.models -import java.nio.file.Path - -import scala.util.{ Failure, Success, Try } +import java.nio.file.{ Path, Paths } import play.api.Logger import play.api.libs.functional.syntax._ @@ -19,7 +17,7 @@ import org.elastic4play.{ AttributeError, InvalidFormatAttributeError, MissingAt object WorkerConfigItemType extends Enumeration with HiveEnumeration { type Type = Value - val string, number, boolean = Value + val text, string, number, boolean = Value implicit val reads: Format[WorkerConfigItemType.Type] = enumFormat(this) } @@ -37,11 +35,11 @@ case class ConfigurationDefinitionItem( private def check(v: JsValue): JsValue Or Every[AttributeError] = { import WorkerConfigItemType._ v match { - case _: JsString if `type` == string ⇒ Good(v) - case _: JsNumber if `type` == number ⇒ Good(v) - case _: JsBoolean if `type` == boolean ⇒ Good(v) - case JsNull if !isRequired ⇒ Good(v) - case _ ⇒ Bad(One(InvalidFormatAttributeError(s"$name[]", `type`.toString, JsonInputValue(v)))) + case _: JsString if `type` == string || `type` == text ⇒ Good(v) + case _: JsNumber if `type` == number ⇒ Good(v) + case _: JsBoolean if `type` == boolean ⇒ Good(v) + case JsNull if !isRequired ⇒ Good(v) + case _ ⇒ Bad(One(InvalidFormatAttributeError(s"$name[]", `type`.toString, JsonInputValue(v)))) } } @@ -80,8 +78,8 @@ case class WorkerDefinition( author: String, url: String, license: String, - baseDirectory: Path, - command: String, + dockerImage: Option[String], + command: Option[Path], baseConfiguration: Option[String], configurationItems: Seq[ConfigurationDefinitionItem], configuration: JsObject, @@ -94,28 +92,7 @@ case class WorkerDefinition( object WorkerDefinition { lazy val logger = Logger(getClass) - def fromPath(definitionFile: Path, workerType: WorkerType.Type): Try[WorkerDefinition] = { - readJsonFile(definitionFile) - .recoverWith { - case error ⇒ - logger.warn(s"Load of worker $definitionFile fails", error) - Failure(error) - } - .map(_.validate(WorkerDefinition.reads(definitionFile.getParent.getParent, workerType))) - .flatMap { - case JsSuccess(workerDefinition, _) ⇒ Success(workerDefinition) - case JsError(errors) ⇒ sys.error(s"Json description file $definitionFile is invalid: $errors") - } - } - - private def readJsonFile(file: Path): Try[JsObject] = { - val source = scala.io.Source.fromFile(file.toFile) - val json = Try(Json.parse(source.mkString).as[JsObject]) - source.close() - json - } - - def reads(path: Path, workerType: WorkerType.Type): Reads[WorkerDefinition] = ( + def singleReads(workerType: WorkerType.Type): Reads[WorkerDefinition] = ( (JsPath \ "name").read[String] and (JsPath \ "version").read[String] and (JsPath \ "description").read[String] and @@ -123,12 +100,18 @@ object WorkerDefinition { (JsPath \ "author").read[String] and (JsPath \ "url").read[String] and (JsPath \ "license").read[String] and - Reads.pure(path) and - (JsPath \ "command").read[String] and + (JsPath \ "dockerImage").readNullable[String] and + (JsPath \ "command").readNullable[String].map(_.map(Paths.get(_))) and (JsPath \ "baseConfig").readNullable[String] and (JsPath \ "configurationItems").read[Seq[ConfigurationDefinitionItem]].orElse(Reads.pure(Nil)) and (JsPath \ "config").read[JsObject].orElse(Reads.pure(JsObject.empty)) and Reads.pure(workerType))(WorkerDefinition.apply _) + + def reads(workerType: WorkerType.Type): Reads[List[WorkerDefinition]] = { + val reads = singleReads(workerType) + reads.map(List(_)) orElse Reads.list(reads) + } + implicit val writes: Writes[WorkerDefinition] = Writes[WorkerDefinition] { workerDefinition ⇒ Json.obj( "id" → workerDefinition.id, @@ -140,6 +123,8 @@ object WorkerDefinition { "url" → workerDefinition.url, "license" → workerDefinition.license, "baseConfig" → workerDefinition.baseConfiguration, - "configurationItems" → workerDefinition.configurationItems) + "configurationItems" → workerDefinition.configurationItems, + "dockerImage" → workerDefinition.dockerImage, + "command" → workerDefinition.command.map(_.getFileName.toString)) } } diff --git a/app/org/thp/cortex/models/package.scala b/app/org/thp/cortex/models/package.scala index 4ee41c710..bafbee1ba 100644 --- a/app/org/thp/cortex/models/package.scala +++ b/app/org/thp/cortex/models/package.scala @@ -1,5 +1,5 @@ package org.thp.cortex package object models { - val modelVersion = 2 + val modelVersion = 3 } diff --git a/app/org/thp/cortex/services/AnalyzerConfigSrv.scala b/app/org/thp/cortex/services/AnalyzerConfigSrv.scala index fbc1d2fd4..ca81b8385 100644 --- a/app/org/thp/cortex/services/AnalyzerConfigSrv.scala +++ b/app/org/thp/cortex/services/AnalyzerConfigSrv.scala @@ -2,6 +2,8 @@ package org.thp.cortex.services import scala.concurrent.{ ExecutionContext, Future } +import play.api.Configuration + import akka.stream.Materializer import javax.inject.{ Inject, Singleton } import org.thp.cortex.models.{ BaseConfig, WorkerConfigModel, WorkerType } @@ -10,6 +12,7 @@ import org.elastic4play.services.{ CreateSrv, FindSrv, UpdateSrv } @Singleton class AnalyzerConfigSrv @Inject() ( + val configuration: Configuration, val workerConfigModel: WorkerConfigModel, val userSrv: UserSrv, val organizationSrv: OrganizationSrv, diff --git a/app/org/thp/cortex/services/DockerJobRunnerSrv.scala b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala new file mode 100644 index 000000000..248e812db --- /dev/null +++ b/app/org/thp/cortex/services/DockerJobRunnerSrv.scala @@ -0,0 +1,103 @@ +package org.thp.cortex.services + +import java.nio.charset.StandardCharsets +import java.nio.file._ + +import scala.concurrent.duration.FiniteDuration +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Try + +import play.api.libs.json.Json +import play.api.{ Configuration, Logger } + +import akka.actor.ActorSystem +import com.spotify.docker.client.DockerClient.LogsParam +import com.spotify.docker.client.messages.HostConfig.Bind +import com.spotify.docker.client.messages.{ ContainerConfig, HostConfig } +import com.spotify.docker.client.{ DefaultDockerClient, DockerClient } +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models._ + +import org.elastic4play.utils.RichFuture + +@Singleton +class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val system: ActorSystem) { + + @Inject() + def this(config: Configuration, system: ActorSystem) = this( + new DefaultDockerClient.Builder() + .apiVersion(config.getOptional[String]("docker.version").orNull) + .connectionPoolSize(config.getOptional[Int]("docker.connectionPoolSize").getOrElse(100)) + .connectTimeoutMillis(config.getOptional[Long]("docker.connectTimeoutMillis").getOrElse(5000)) + //.dockerCertificates() + .readTimeoutMillis(config.getOptional[Long]("docker.readTimeoutMillis").getOrElse(30000)) + //.registryAuthSupplier() + .uri(config.getOptional[String]("docker.uri").getOrElse("unix:///var/run/docker.sock")) + .useProxy(config.getOptional[Boolean]("docker.useProxy").getOrElse(false)) + .build(), + config.getOptional[Boolean]("docker.autoUpdate").getOrElse(true), + system: ActorSystem) + + lazy val logger = Logger(getClass) + + lazy val isAvailable: Boolean = + Try { + logger.info(s"Docker is available:\n${client.info()}") + true + } + .getOrElse { + logger.info(s"Docker is not available") + false + } + + def run(jobDirectory: Path, dockerImage: String, job: Job, timeout: Option[FiniteDuration])(implicit ec: ExecutionContext): Future[Unit] = { + import scala.collection.JavaConverters._ + if (autoUpdate) client.pull(dockerImage) + // ContainerConfig.builder().addVolume() + val hostConfig = HostConfig.builder() + .appendBinds(Bind.from(jobDirectory.toAbsolutePath.toString) + .to("/job") + .readOnly(false) + .build()) + .build() + val cacertsFile = jobDirectory.resolve("input").resolve("cacerts") + val containerConfigBuilder = ContainerConfig + .builder() + .hostConfig(hostConfig) + .image(dockerImage) + .cmd("/job") + + val containerConfig = if (Files.exists(cacertsFile)) containerConfigBuilder.env(s"REQUESTS_CA_BUNDLE=/job/input/cacerts").build() + else containerConfigBuilder.build() + val containerCreation = client.createContainer(containerConfig) + // Option(containerCreation.warnings()).flatMap(_.asScala).foreach(logger.warn) + logger.info(s"Execute container ${containerCreation.id()}\n" + + s" timeout: ${timeout.fold("none")(_.toString)}\n" + + s" image : $dockerImage\n" + + s" volume : ${jobDirectory.toAbsolutePath}:/job" + + Option(containerConfig.env()).fold("")(_.asScala.map("\n env : " + _).mkString)) + + val execution = Future { + client.startContainer(containerCreation.id()) + client.waitContainer(containerCreation.id()) + () + } + .andThen { + case r ⇒ + if (!Files.exists(jobDirectory.resolve("output").resolve("output.json"))) { + val message = r.fold(e ⇒ s"Docker creation error: ${e.getMessage}\n", _ ⇒ "") + + Try(client.logs(containerCreation.id(), LogsParam.stdout(), LogsParam.stderr()).readFully()) + .recover { case e ⇒ s"Container logs can't be read (${e.getMessage}" } + val report = Json.obj( + "success" -> false, + "errorMessage" -> message) + Files.write(jobDirectory.resolve("output").resolve("output.json"), report.toString.getBytes(StandardCharsets.UTF_8)) + } + } + timeout.fold(execution)(t ⇒ execution.withTimeout(t, client.stopContainer(containerCreation.id(), 3))) + .andThen { + case _ ⇒ client.removeContainer(containerCreation.id(), DockerClient.RemoveContainerParam.forceKill()) + } + } + +} diff --git a/app/org/thp/cortex/services/JobRunnerSrv.scala b/app/org/thp/cortex/services/JobRunnerSrv.scala new file mode 100644 index 000000000..76af9cc0d --- /dev/null +++ b/app/org/thp/cortex/services/JobRunnerSrv.scala @@ -0,0 +1,257 @@ +package org.thp.cortex.services + +import java.io.IOException +import java.nio.charset.StandardCharsets +import java.nio.file._ +import java.nio.file.attribute.BasicFileAttributes +import java.util.Date + +import scala.concurrent.duration.DurationLong +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Failure + +import play.api.libs.json._ +import play.api.{ Configuration, Logger } + +import akka.actor.ActorSystem +import akka.stream.Materializer +import akka.stream.scaladsl.FileIO +import javax.inject.Inject +import org.thp.cortex.models._ + +import org.elastic4play.BadRequestError +import org.elastic4play.controllers.{ Fields, FileInputValue } +import org.elastic4play.database.ModifyConfig +import org.elastic4play.services.{ AttachmentSrv, AuthContext, CreateSrv, UpdateSrv } + +class JobRunnerSrv @Inject() ( + config: Configuration, + reportModel: ReportModel, + artifactModel: ArtifactModel, + processJobRunnerSrv: ProcessJobRunnerSrv, + dockerJobRunnerSrv: DockerJobRunnerSrv, + workerSrv: WorkerSrv, + createSrv: CreateSrv, + updateSrv: UpdateSrv, + attachmentSrv: AttachmentSrv, + akkaSystem: ActorSystem, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) { + + val logger = Logger(getClass) + lazy val analyzerExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") + lazy val responderExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("responder") + + private val runners: Seq[String] = config + .getOptional[Seq[String]]("job.runners") + .getOrElse(Seq("docker", "process")) + .map(_.toLowerCase) + .collect { + case "docker" if dockerJobRunnerSrv.isAvailable ⇒ "docker" + case "process" ⇒ + Seq("", "2", "3").foreach { pythonVersion ⇒ + val cortexUtilsVersion = processJobRunnerSrv.checkCortexUtilsVersion(pythonVersion) + cortexUtilsVersion.fold(logger.warn(s"The package cortexutils for python$pythonVersion hasn't been found")) { + case (major, minor, patch) if major >= 2 ⇒ logger.info(s"The package cortexutils for python$pythonVersion has valid version: $major.$minor.$patch") + case (major, minor, patch) ⇒ logger.error(s"The package cortexutils for python$pythonVersion has invalid version: $major.$minor.$patch. Cortex 2 requires cortexutils >= 2.0") + } + } + "process" + } + + lazy val processRunnerIsEnable: Boolean = runners.contains("process") + lazy val dockerRunnerIsEnable: Boolean = runners.contains("docker") + + private object deleteVisitor extends SimpleFileVisitor[Path] { + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + Files.delete(file) + FileVisitResult.CONTINUE + } + + override def postVisitDirectory(dir: Path, e: IOException): FileVisitResult = { + Files.delete(dir) + FileVisitResult.CONTINUE + } + } + + private def delete(directory: Path): Unit = try { + if (Files.exists(directory)) + Files.walkFileTree(directory, deleteVisitor) + () + } + catch { + case t: Throwable ⇒ logger.warn(s"Fail to remove temporary files ($directory) : $t") + } + + private def prepareJobFolder(worker: Worker, job: Job): Future[Path] = { + val jobFolder = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), s"cortex-job-${job.id}-") + val inputJobFolder = Files.createDirectories(jobFolder.resolve("input")) + Files.createDirectories(jobFolder.resolve("output")) + + job.attachment() + .map { attachment ⇒ + val attachmentFile = Files.createTempFile(inputJobFolder, "attachment", "") + attachmentSrv.source(attachment.id).runWith(FileIO.toPath(attachmentFile)) + .flatMap { + case ioresult if ioresult.status.isSuccess ⇒ Future.successful(Some(attachmentFile)) + case ioresult ⇒ Future.failed(ioresult.getError) + } + } + .getOrElse(Future.successful(None)) + .map { + case Some(file) ⇒ + Json.obj( + "file" → file.getFileName.toString, + "filename" → job.attachment().get.name, + "contentType" → job.attachment().get.contentType) + case None if job.data().nonEmpty && job.tpe() == WorkerType.responder ⇒ + Json.obj("data" → Json.parse(job.data().get)) + case None if job.data().nonEmpty && job.tpe() == WorkerType.analyzer ⇒ + Json.obj("data" → job.data().get) + } + .map { artifact ⇒ + val proxy_http = (worker.config \ "proxy_http").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" → Json.obj("http" → proxy)) } + val proxy_https = (worker.config \ "proxy_https").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" → Json.obj("https" → proxy)) } + val config = workerSrv.getDefinition(worker.workerDefinitionId()) + .fold(_ ⇒ JsObject.empty, _.configuration) + .deepMerge(worker.config) + .deepMerge(proxy_http) + .deepMerge(proxy_https) + (worker.config \ "cacerts").asOpt[String].foreach { cacerts ⇒ + val cacertsFile = jobFolder.resolve("input").resolve("cacerts") + Files.write(cacertsFile, cacerts.getBytes) + } + artifact + + ("dataType" → JsString(job.dataType())) + + ("tlp" → JsNumber(job.tlp())) + + ("pap" → JsNumber(job.pap())) + + ("message" → JsString(job.message().getOrElse(""))) + + ("parameters" → job.params) + + ("config" -> config) + } + .map { input ⇒ + Files.write(inputJobFolder.resolve("input.json"), input.toString.getBytes(StandardCharsets.UTF_8)) + jobFolder + } + .recoverWith { + case error ⇒ + delete(jobFolder) + Future.failed(error) + } + } + + private def extractReport(jobFolder: Path, job: Job)(implicit authContext: AuthContext) = { + val outputFile = jobFolder.resolve("output").resolve("output.json") + if (Files.exists(outputFile)) { + val is = Files.newInputStream(outputFile) + val report = Json.parse(is) + is.close() + + val success = (report \ "success").asOpt[Boolean].getOrElse(false) + if (success) { + val fullReport = (report \ "full").as[JsObject].toString + val summaryReport = (report \ "summary").asOpt[JsObject].getOrElse(JsObject.empty).toString + val artifacts = (report \ "artifacts").asOpt[Seq[JsObject]].getOrElse(Nil) + val operations = (report \ "operations").asOpt[Seq[JsObject]].getOrElse(Nil) + val reportFields = Fields.empty + .set("full", fullReport) + .set("summary", summaryReport) + .set("operations", JsArray(operations).toString) + createSrv[ReportModel, Report, Job](reportModel, job, reportFields) + .flatMap { report ⇒ + Future.sequence { + for { + artifact ← artifacts + dataType ← (artifact \ "dataType").asOpt[String] + fields ← dataType match { + case "file" ⇒ + for { + name ← (artifact \ "filename").asOpt[String] + file ← (artifact \ "file").asOpt[String] + path = jobFolder.resolve("output").resolve(file) + if Files.exists(path) && path.getParent == jobFolder.resolve("output") + contentType = (artifact \ "contentType").asOpt[String].getOrElse("application/octet-stream") + fiv = FileInputValue(name, path, contentType) + } yield Fields(artifact - "filename" - "file" - "contentType").set("attachment", fiv) + case _ ⇒ Some(Fields(artifact)) + } + } yield createSrv[ArtifactModel, Artifact, Report](artifactModel, report, fields) + } + } + .transformWith { + case Failure(e) ⇒ endJob(job, JobStatus.Failure, Some(s"Report creation failure: $e")) + case _ ⇒ endJob(job, JobStatus.Success) + } + } + else { + endJob(job, JobStatus.Failure, + (report \ "errorMessage").asOpt[String], + (report \ "input").asOpt[JsValue].map(_.toString)) + } + } + else { + endJob(job, JobStatus.Failure, Some(s"no output")) + } + } + + def run(worker: Worker, job: Job)(implicit authContext: AuthContext): Future[Job] = { + prepareJobFolder(worker, job).flatMap { jobFolder ⇒ + val executionContext = worker.tpe() match { + case WorkerType.analyzer ⇒ analyzerExecutionContext + case WorkerType.responder ⇒ responderExecutionContext + } + val finishedJob = for { + _ ← startJob(job) + j ← runners + .foldLeft[Option[Future[Unit]]](None) { + case (None, "docker") ⇒ + worker.dockerImage() + .map(dockerImage ⇒ dockerJobRunnerSrv.run(jobFolder, dockerImage, job, worker.jobTimeout().map(_.minutes))(executionContext)) + .orElse { + logger.warn(s"worker ${worker.id} can't be run with docker (doesn't have image)") + None + } + case (None, "process") ⇒ + + worker.command() + .map(command ⇒ processJobRunnerSrv.run(jobFolder, command, job, worker.jobTimeout().map(_.minutes))(executionContext)) + .orElse { + logger.warn(s"worker ${worker.id} can't be run with process (doesn't have image)") + None + } + case (j: Some[_], _) ⇒ j + case (None, runner) ⇒ + logger.warn(s"Unknown job runner: $runner") + None + + } + .getOrElse(Future.failed(BadRequestError("Worker cannot be run"))) + } yield j + finishedJob + .transformWith { r ⇒ + r.fold( + error ⇒ endJob(job, JobStatus.Failure, Option(error.getMessage), Some(readFile(jobFolder.resolve("input").resolve("input.json")))), + _ ⇒ extractReport(jobFolder, job)) + } + .andThen { case _ ⇒ delete(jobFolder) } + } + } + + private def readFile(input: Path): String = new String(Files.readAllBytes(input), StandardCharsets.UTF_8) + + private def startJob(job: Job)(implicit authContext: AuthContext): Future[Job] = { + val fields = Fields.empty + .set("status", JobStatus.InProgress.toString) + .set("startDate", Json.toJson(new Date)) + updateSrv(job, fields, ModifyConfig(retryOnConflict = 0)) + } + + private def endJob(job: Job, status: JobStatus.Type, errorMessage: Option[String] = None, input: Option[String] = None)(implicit authContext: AuthContext): Future[Job] = { + val fields = Fields.empty + .set("status", status.toString) + .set("endDate", Json.toJson(new Date)) + .set("input", input.map(JsString.apply)) + .set("message", errorMessage.map(JsString.apply)) + updateSrv(job, fields, ModifyConfig.default) + } +} \ No newline at end of file diff --git a/app/org/thp/cortex/services/JobSrv.scala b/app/org/thp/cortex/services/JobSrv.scala index 829771955..fe568c2c5 100644 --- a/app/org/thp/cortex/services/JobSrv.scala +++ b/app/org/thp/cortex/services/JobSrv.scala @@ -1,29 +1,25 @@ package org.thp.cortex.services -import java.io.{ ByteArrayOutputStream, InputStream } -import java.nio.file.{ Files, Paths } import java.util.Date -import javax.inject.{ Inject, Singleton } +import scala.concurrent.duration._ +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.{ Failure, Success } + +import play.api.libs.json._ +import play.api.{ Configuration, Logger } + import akka.NotUsed -import akka.actor.ActorSystem import akka.stream.Materializer -import akka.stream.scaladsl.{ FileIO, Sink, Source } +import akka.stream.scaladsl.{ Sink, Source } +import javax.inject.{ Inject, Singleton } +import org.scalactic.Accumulation._ +import org.scalactic.{ Bad, Good, One, Or } +import org.thp.cortex.models._ import org.elastic4play._ import org.elastic4play.controllers._ -import org.elastic4play.database.ModifyConfig import org.elastic4play.services._ -import org.scalactic.Accumulation._ -import org.scalactic.{ Bad, Good, One, Or } -import org.thp.cortex.models._ -import play.api.libs.json._ -import play.api.{ Configuration, Logger } -import scala.concurrent.duration._ -import scala.concurrent.{ ExecutionContext, Future } -import scala.sys.process.{ Process, ProcessIO } -import scala.util.control.NonFatal -import scala.util.{ Failure, Success, Try } @Singleton class JobSrv( @@ -33,13 +29,11 @@ class JobSrv( artifactModel: ArtifactModel, workerSrv: WorkerSrv, userSrv: UserSrv, - getSrv: GetSrv, + jobRunnerSrv: JobRunnerSrv, createSrv: CreateSrv, - updateSrv: UpdateSrv, findSrv: FindSrv, deleteSrv: DeleteSrv, attachmentSrv: AttachmentSrv, - akkaSystem: ActorSystem, implicit val ec: ExecutionContext, implicit val mat: Materializer) { @@ -50,13 +44,11 @@ class JobSrv( artifactModel: ArtifactModel, workerSrv: WorkerSrv, userSrv: UserSrv, - getSrv: GetSrv, + jobRunnerSrv: JobRunnerSrv, createSrv: CreateSrv, - updateSrv: UpdateSrv, findSrv: FindSrv, deleteSrv: DeleteSrv, attachmentSrv: AttachmentSrv, - akkaSystem: ActorSystem, ec: ExecutionContext, mat: Materializer) = this( configuration.getOptional[Duration]("cache.job").getOrElse(Duration.Zero), @@ -65,23 +57,15 @@ class JobSrv( artifactModel, workerSrv, userSrv, - getSrv, + jobRunnerSrv, createSrv, - updateSrv, findSrv, deleteSrv, attachmentSrv, - akkaSystem, - ec, mat) + ec, + mat) private lazy val logger = Logger(getClass) - private lazy val analyzerExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") - private lazy val responderExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("responder") - private val osexec = - if (System.getProperty("os.name").toLowerCase.contains("win")) - (c: String) ⇒ s"""cmd /c $c""" - else - (c: String) ⇒ c runPreviousJobs() @@ -93,14 +77,11 @@ class JobSrv( .runForeach { job ⇒ (for { worker ← workerSrv.get(job.workerId()) - workerDefinition ← workerSrv.getDefinition(job.workerId()) - updatedJob ← run(workerDefinition, worker, job) + updatedJob ← jobRunnerSrv.run(worker, job) } yield updatedJob) .onComplete { case Success(j) ⇒ logger.info(s"Job ${job.id} has finished with status ${j.status()}") - case Failure(e) ⇒ - endJob(job, JobStatus.Failure, Some(e.getMessage), None) - logger.error(s"Job ${job.id} has failed", e) + case Failure(e) ⇒ logger.error(s"Job ${job.id} has failed", e) } } } @@ -285,15 +266,13 @@ class JobSrv( case Left(data) ⇒ fields.set("data", data) case Right(attachment) ⇒ fields.set("attachment", AttachmentInputValue(attachment)) } - workerSrv.getDefinition(worker.workerDefinitionId()).flatMap { workerDefinition ⇒ - createSrv[JobModel, Job](jobModel, fieldWithData).andThen { - case Success(job) ⇒ - run(workerDefinition, worker, job) - .onComplete { - case Success(j) ⇒ logger.info(s"Job ${job.id} has finished with status ${j.status()}") - case Failure(e) ⇒ logger.error(s"Job ${job.id} has failed", e) - } - } + createSrv[JobModel, Job](jobModel, fieldWithData).andThen { + case Success(job) ⇒ + jobRunnerSrv.run(worker, job) + .onComplete { + case Success(j) ⇒ logger.info(s"Job ${job.id} has finished with status ${j.status()}") + case Failure(e) ⇒ logger.error(s"Job ${job.id} has failed", e) + } } case false ⇒ Future.failed(RateLimitExceeded(worker)) @@ -327,7 +306,7 @@ class JobSrv( } else { import org.elastic4play.services.QueryDSL._ - logger.info(s"Looking for similar job (worker=${worker.id}, dataType=$dataType, data=$dataAttachment, tlp=$tlp, parameters=$parameters") + logger.info(s"Looking for similar job in the last ${cache.toMinutes} minutes (worker=${worker.id}, dataType=$dataType, data=$dataAttachment, tlp=$tlp, parameters=$parameters)") val now = new Date().getTime find(and( "workerId" ~= worker.id, @@ -344,73 +323,6 @@ class JobSrv( } } - private def fixArtifact(artifact: Fields): Fields = { - def rename(oldName: String, newName: String): Fields ⇒ Fields = fields ⇒ - fields.getValue(oldName).fold(fields)(v ⇒ fields.unset(oldName).set(newName, v)) - - rename("value", "data").andThen( - rename("type", "dataType"))(artifact) - } - - def run(workerDefinition: WorkerDefinition, worker: Worker, job: Job)(implicit authContext: AuthContext): Future[Job] = { - val executionContext = workerDefinition.tpe match { - case WorkerType.analyzer ⇒ analyzerExecutionContext - case WorkerType.responder ⇒ responderExecutionContext - } - buildInput(workerDefinition, worker, job) - .flatMap { input ⇒ - startJob(job) - var output = "" - var error = "" - try { - logger.info(s"Execute ${osexec(workerDefinition.command)} in ${workerDefinition.baseDirectory}") - Process(osexec(workerDefinition.command), workerDefinition.baseDirectory.toFile).run( - new ProcessIO( - { stdin ⇒ Try(stdin.write(input.toString.getBytes("UTF-8"))); stdin.close() }, - { stdout ⇒ output = readStream(stdout) }, - { stderr ⇒ error = readStream(stderr) })) - .exitValue() - val report = Json.parse(output).as[JsObject] - val success = (report \ "success").asOpt[Boolean].getOrElse(false) - if (success) { - val fullReport = (report \ "full").as[JsObject].toString - val summaryReport = (report \ "summary").asOpt[JsObject].getOrElse(JsObject.empty).toString - val artifacts = (report \ "artifacts").asOpt[Seq[JsObject]].getOrElse(Nil) - val operations = (report \ "operations").asOpt[Seq[JsObject]].getOrElse(Nil) - val reportFields = Fields.empty - .set("full", fullReport) - .set("summary", summaryReport) - .set("operations", JsArray(operations).toString) - createSrv[ReportModel, Report, Job](reportModel, job, reportFields) - .flatMap { report ⇒ - Future.traverse(artifacts) { artifact ⇒ - createSrv[ArtifactModel, Artifact, Report](artifactModel, report, fixArtifact(Fields(artifact))) - } - } - .transformWith { - case Failure(e) ⇒ endJob(job, JobStatus.Failure, Some(s"Report creation failure: $e")) - case _ ⇒ endJob(job, JobStatus.Success) - } - } - else { - endJob(job, JobStatus.Failure, - (report \ "errorMessage").asOpt[String], - (report \ "input").asOpt[JsValue].map(_.toString)) - } - } - catch { - case NonFatal(_) ⇒ - val errorMessage = (error + output).take(8192) - endJob(job, JobStatus.Failure, Some(s"Invalid output\n$errorMessage")) - } - finally { - (input \ "file").asOpt[String].foreach { filename ⇒ - Files.deleteIfExists(Paths.get(filename)) - } - } - }(executionContext) - } - def getReport(jobId: String)(implicit authContext: AuthContext): Future[Report] = getForUser(authContext.userId, jobId).flatMap(getReport) def getReport(job: Job): Future[Report] = { @@ -419,73 +331,4 @@ class JobSrv( .runWith(Sink.headOption) .map(_.getOrElse(throw NotFoundError(s"Job ${job.id} has no report"))) } - - private def buildInput(workerDefinition: WorkerDefinition, worker: Worker, job: Job): Future[JsObject] = { - job.attachment() - .map { attachment ⇒ - val tempFile = Files.createTempFile(s"cortex-job-${job.id}-", "") - attachmentSrv.source(attachment.id).runWith(FileIO.toPath(tempFile)) - .flatMap { - case ioresult if ioresult.status.isSuccess ⇒ Future.successful(Some(tempFile)) - case ioresult ⇒ Future.failed(ioresult.getError) - } - } - .getOrElse(Future.successful(None)) - .map { - case Some(file) ⇒ - Json.obj( - "file" → file.toString, - "filename" → job.attachment().get.name, - "contentType" → job.attachment().get.contentType) - case None if job.data().nonEmpty && job.tpe() == WorkerType.responder ⇒ - Json.obj("data" → Json.parse(job.data().get)) - case None if job.data().nonEmpty && job.tpe() == WorkerType.analyzer ⇒ - Json.obj("data" → job.data().get) - } - .map { artifact ⇒ - (BaseConfig.global(worker.tpe()).items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items ++ workerDefinition.configurationItems) - .validatedBy(_.read(worker.config)) - .map(cfg ⇒ Json.obj("config" → JsObject(cfg).deepMerge(workerDefinition.configuration))) - .map { cfg ⇒ - val proxy_http = (cfg \ "config" \ "proxy_http").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" → Json.obj("http" → proxy)) } - val proxy_https = (cfg \ "config" \ "proxy_https").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" → Json.obj("https" → proxy)) } - cfg.deepMerge(Json.obj("config" → proxy_http.deepMerge(proxy_https))) - } - .map(_ deepMerge artifact + - ("dataType" → JsString(job.dataType())) + - ("tlp" → JsNumber(job.tlp())) + - ("pap" → JsNumber(job.pap())) + - ("message" → JsString(job.message().getOrElse(""))) + - ("parameters" → job.params)) - .badMap(e ⇒ AttributeCheckingError("job", e.toSeq)) - .toTry - } - .flatMap(Future.fromTry) - } - - // - private def startJob(job: Job)(implicit authContext: AuthContext): Future[Job] = { - val fields = Fields.empty - .set("status", JobStatus.InProgress.toString) - .set("startDate", Json.toJson(new Date)) - updateSrv(job, fields, ModifyConfig(retryOnConflict = 0)) - } - - private def endJob(job: Job, status: JobStatus.Type, errorMessage: Option[String] = None, input: Option[String] = None)(implicit authContext: AuthContext): Future[Job] = { - val fields = Fields.empty - .set("status", status.toString) - .set("endDate", Json.toJson(new Date)) - .set("input", input.map(JsString.apply)) - .set("message", errorMessage.map(JsString.apply)) - updateSrv(job, fields, ModifyConfig.default) - } - - private def readStream(stream: InputStream): String = { - val out = new ByteArrayOutputStream() - val buffer = Array.ofDim[Byte](4096) - Stream.continually(stream.read(buffer)) - .takeWhile(_ != -1) - .foreach(out.write(buffer, 0, _)) - out.toString("UTF-8") - } } \ No newline at end of file diff --git a/app/org/thp/cortex/services/KeyAuthSrv.scala b/app/org/thp/cortex/services/KeyAuthSrv.scala index ffada3a95..3114e17ec 100644 --- a/app/org/thp/cortex/services/KeyAuthSrv.scala +++ b/app/org/thp/cortex/services/KeyAuthSrv.scala @@ -39,7 +39,7 @@ class KeyAuthSrv @Inject() ( .filter(_.key().contains(key)) .runWith(Sink.headOption) .flatMap { - case Some(user) ⇒ userSrv.getFromUser(request, user) + case Some(user) ⇒ userSrv.getFromUser(request, user, name) case None ⇒ Future.failed(AuthenticationError("Authentication failure")) } } diff --git a/app/org/thp/cortex/services/LocalAuthSrv.scala b/app/org/thp/cortex/services/LocalAuthSrv.scala index 2d0cbd04f..9a7c4d9bc 100644 --- a/app/org/thp/cortex/services/LocalAuthSrv.scala +++ b/app/org/thp/cortex/services/LocalAuthSrv.scala @@ -35,7 +35,7 @@ class LocalAuthSrv @Inject() ( override def authenticate(username: String, password: String)(implicit request: RequestHeader): Future[AuthContext] = { userSrv.get(username).flatMap { user ⇒ - if (doAuthenticate(user, password)) userSrv.getFromUser(request, user) + if (doAuthenticate(user, password)) userSrv.getFromUser(request, user, name) else Future.failed(AuthenticationError("Authentication failure")) } } diff --git a/app/org/thp/cortex/services/MispSrv.scala b/app/org/thp/cortex/services/MispSrv.scala index 3883c10d6..b0b170a06 100644 --- a/app/org/thp/cortex/services/MispSrv.scala +++ b/app/org/thp/cortex/services/MispSrv.scala @@ -35,25 +35,19 @@ class MispSrv @Inject() ( val (analyzers, analyzerCount) = workerSrv.findAnalyzersForUser(authContext.userId, QueryDSL.any, Some("all"), Nil) val mispAnalyzers = analyzers - .mapAsyncUnordered(1) { analyzer ⇒ - workerSrv.getDefinition(analyzer.workerDefinitionId()) - .map(ad ⇒ Some(analyzer → ad)) - .recover { case _ ⇒ None } - } - .collect { - case Some((analyzer, analyzerDefinition)) ⇒ - Json.obj( - "name" → analyzer.name(), - "type" → "cortex", - "mispattributes" → Json.obj( - "input" → analyzer.dataTypeList().flatMap(dataType2mispType).distinct, - "output" → Json.arr()), - "meta" → Json.obj( - "module-type" → Json.arr("cortex"), - "description" → analyzer.description(), - "author" → analyzerDefinition.author, - "version" → analyzerDefinition.version, - "config" → Json.arr())) + .map { analyzer ⇒ + Json.obj( + "name" → analyzer.name(), + "type" → "cortex", + "mispattributes" → Json.obj( + "input" → analyzer.dataTypeList().flatMap(dataType2mispType).distinct, + "output" → Json.arr()), + "meta" → Json.obj( + "module-type" → Json.arr("cortex"), + "description" → analyzer.description(), + "author" → analyzer.author(), + "version" → analyzer.vers(), + "config" → Json.arr())) } mispAnalyzers → analyzerCount } diff --git a/app/org/thp/cortex/services/OAuth2Srv.scala b/app/org/thp/cortex/services/OAuth2Srv.scala new file mode 100644 index 000000000..a8f7be1c1 --- /dev/null +++ b/app/org/thp/cortex/services/OAuth2Srv.scala @@ -0,0 +1,163 @@ +package org.thp.cortex.services + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.http.Status +import play.api.libs.json.{ JsObject, JsValue } +import play.api.libs.ws.WSClient +import play.api.mvc.RequestHeader +import play.api.{ Configuration, Logger } + +import akka.stream.Materializer +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.services.mappers.UserMapper + +import org.elastic4play.services.{ AuthContext, AuthSrv } +import org.elastic4play.{ AuthenticationError, AuthorizationError, OAuth2Redirect } + +case class OAuth2Config( + clientId: String, + clientSecret: String, + redirectUri: String, + responseType: String, + grantType: String, + authorizationUrl: String, + tokenUrl: String, + userUrl: String, + scope: String, + autocreate: Boolean) + +object OAuth2Config { + def apply(configuration: Configuration): Option[OAuth2Config] = { + for { + clientId ← configuration.getOptional[String]("auth.oauth2.clientId") + clientSecret ← configuration.getOptional[String]("auth.oauth2.clientSecret") + redirectUri ← configuration.getOptional[String]("auth.oauth2.redirectUri") + responseType ← configuration.getOptional[String]("auth.oauth2.responseType") + grantType ← configuration.getOptional[String]("auth.oauth2.grantType") + authorizationUrl ← configuration.getOptional[String]("auth.oauth2.authorizationUrl") + userUrl ← configuration.getOptional[String]("auth.oauth2.userUrl") + tokenUrl ← configuration.getOptional[String]("auth.oauth2.tokenUrl") + scope ← configuration.getOptional[String]("auth.oauth2.scope") + autocreate = configuration.getOptional[Boolean]("auth.sso.autocreate").getOrElse(false) + } yield OAuth2Config(clientId, clientSecret, redirectUri, responseType, grantType, authorizationUrl, tokenUrl, userUrl, scope, autocreate) + } +} + +@Singleton +class OAuth2Srv( + ws: WSClient, + userSrv: UserSrv, + ssoMapper: UserMapper, + oauth2Config: Option[OAuth2Config], + implicit val ec: ExecutionContext, + implicit val mat: Materializer) + extends AuthSrv { + + @Inject() def this( + ws: WSClient, + ssoMapper: UserMapper, + userSrv: UserSrv, + configuration: Configuration, + ec: ExecutionContext, + mat: Materializer) = this( + ws, + userSrv, + ssoMapper, + OAuth2Config(configuration), + ec, + mat) + + override val name: String = "oauth2" + private val logger = Logger(getClass) + + val Oauth2TokenQueryString = "code" + + private def withOAuth2Config[A](body: OAuth2Config ⇒ Future[A]): Future[A] = { + oauth2Config.fold[Future[A]](Future.failed(AuthenticationError("OAuth2 not configured properly")))(body) + } + + override def authenticate()(implicit request: RequestHeader): Future[AuthContext] = { + withOAuth2Config { cfg ⇒ + request.queryString + .get(Oauth2TokenQueryString) + .flatMap(_.headOption) + .fold(createOauth2Redirect(cfg.clientId)) { code ⇒ + getAuthTokenAndAuthenticate(cfg.clientId, code) + } + } + } + + private def getAuthTokenAndAuthenticate(clientId: String, code: String)(implicit request: RequestHeader): Future[AuthContext] = { + logger.debug("Getting user token with the code from the response!") + withOAuth2Config { cfg ⇒ + val acceptHeader = "Accept" → cfg.responseType + ws.url(cfg.tokenUrl) + .addHttpHeaders(acceptHeader) + .post(Map( + "code" → code, + "grant_type" → cfg.grantType, + "client_secret" → cfg.clientSecret, + "redirect_uri" → cfg.redirectUri, + "client_id" → clientId)) + .recoverWith { + case error ⇒ + logger.error(s"Token verification failure", error) + Future.failed(AuthenticationError("Token verification failure")) + } + .flatMap { r ⇒ + r.status match { + case Status.OK ⇒ + val accessToken = (r.json \ "access_token").asOpt[String].getOrElse("") + val authHeader = "Authorization" → s"bearer $accessToken" + ws.url(cfg.userUrl) + .addHttpHeaders(authHeader) + .get().flatMap { userResponse ⇒ + if (userResponse.status != Status.OK) { + Future.failed(AuthenticationError(s"unexpected response from server: ${userResponse.status} ${userResponse.body}")) + } + else { + val response = userResponse.json.asInstanceOf[JsObject] + getOrCreateUser(response, authHeader) + } + } + case _ ⇒ + logger.error(s"unexpected response from server: ${r.status} ${r.body}") + Future.failed(AuthenticationError("unexpected response from server")) + } + } + } + } + + private def getOrCreateUser(response: JsValue, authHeader: (String, String))(implicit request: RequestHeader): Future[AuthContext] = { + withOAuth2Config { cfg ⇒ + ssoMapper.getUserFields(response, Some(authHeader)).flatMap { + userFields ⇒ + val userId = userFields.getString("login").getOrElse("") + userSrv.get(userId).flatMap(user ⇒ { + userSrv.getFromUser(request, user, name) + }).recoverWith { + case authErr: AuthorizationError ⇒ Future.failed(authErr) + case _ if cfg.autocreate ⇒ + userSrv.inInitAuthContext { implicit authContext ⇒ + userSrv.create(userFields).flatMap(user ⇒ { + userSrv.getFromUser(request, user, name) + }) + } + } + } + } + } + + private def createOauth2Redirect(clientId: String): Future[AuthContext] = { + withOAuth2Config { cfg ⇒ + val queryStringParams = Map[String, Seq[String]]( + "scope" → Seq(cfg.scope), + "response_type" → Seq(cfg.responseType), + "redirect_uri" → Seq(cfg.redirectUri), + "client_id" → Seq(clientId)) + Future.failed(OAuth2Redirect(cfg.authorizationUrl, queryStringParams)) + } + } +} + diff --git a/app/org/thp/cortex/services/ProcessJobRunnerSrv.scala b/app/org/thp/cortex/services/ProcessJobRunnerSrv.scala new file mode 100644 index 000000000..ff4edbc71 --- /dev/null +++ b/app/org/thp/cortex/services/ProcessJobRunnerSrv.scala @@ -0,0 +1,65 @@ +package org.thp.cortex.services + +import java.nio.charset.StandardCharsets +import java.nio.file.{ Files, Path, Paths } + +import scala.concurrent.duration.FiniteDuration +import scala.concurrent.{ ExecutionContext, Future } +import scala.sys.process.{ Process, ProcessLogger } + +import play.api.Logger + +import akka.actor.ActorSystem +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models._ + +import org.elastic4play.utils.RichFuture +import scala.sys.process._ +import scala.util.Try + +import play.api.libs.json.Json + +@Singleton +class ProcessJobRunnerSrv @Inject() (implicit val system: ActorSystem) { + + lazy val logger = Logger(getClass) + + private val pythonPackageVersionRegex = "^Version: ([0-9]*)\\.([0-9]*)\\.([0-9]*)".r + def checkCortexUtilsVersion(pythonVersion: String): Option[(Int, Int, Int)] = + Try { + (s"pip$pythonVersion" :: "show" :: "cortexutils" :: Nil) + .lineStream + .collectFirst { + case pythonPackageVersionRegex(major, minor, patch) ⇒ (major.toInt, minor.toInt, patch.toInt) + } + }.getOrElse(None) + + def run(jobDirectory: Path, command: String, job: Job, timeout: Option[FiniteDuration])(implicit ec: ExecutionContext): Future[Unit] = { + val baseDirectory = Paths.get(command).getParent.getParent + logger.info(s"Execute $command in $baseDirectory, timeout is ${timeout.fold("none")(_.toString)}") + val process = Process(Seq(command, jobDirectory.toString), baseDirectory.toFile) + .run(ProcessLogger(s ⇒ logger.info(s" Job ${job.id}: $s"))) + val execution = Future + .apply { + process.exitValue() + () + } + .recoverWith { + case error ⇒ + logger.error(s"Execution of command $command failed", error) + Future.apply { + val report = Json.obj( + "success" -> false, + "errorMessage" -> error.getMessage) + Files.write(jobDirectory.resolve("output").resolve("output.json"), report.toString.getBytes(StandardCharsets.UTF_8)) + () + } + } + timeout.fold(execution)(t ⇒ execution.withTimeout(t, killProcess(process))) + } + + def killProcess(process: Process): Unit = { + logger.info("Timeout reached, killing process") + process.destroy() + } +} diff --git a/app/org/thp/cortex/services/ResponderConfigSrv.scala b/app/org/thp/cortex/services/ResponderConfigSrv.scala index 50712a4ff..aa3e6baba 100644 --- a/app/org/thp/cortex/services/ResponderConfigSrv.scala +++ b/app/org/thp/cortex/services/ResponderConfigSrv.scala @@ -2,6 +2,8 @@ package org.thp.cortex.services import scala.concurrent.{ ExecutionContext, Future } +import play.api.Configuration + import akka.stream.Materializer import javax.inject.{ Inject, Singleton } import org.thp.cortex.models.{ BaseConfig, WorkerConfigModel, WorkerType } @@ -10,6 +12,7 @@ import org.elastic4play.services.{ CreateSrv, FindSrv, UpdateSrv } @Singleton class ResponderConfigSrv @Inject() ( + val configuration: Configuration, val workerConfigModel: WorkerConfigModel, val userSrv: UserSrv, val organizationSrv: OrganizationSrv, diff --git a/app/org/thp/cortex/services/UserSrv.scala b/app/org/thp/cortex/services/UserSrv.scala index 5fc3edec8..e88fa1b4a 100644 --- a/app/org/thp/cortex/services/UserSrv.scala +++ b/app/org/thp/cortex/services/UserSrv.scala @@ -63,22 +63,22 @@ class UserSrv( cache, ec) - private case class AuthContextImpl(userId: String, userName: String, requestId: String, roles: Seq[Role]) extends AuthContext + private case class AuthContextImpl(userId: String, userName: String, requestId: String, roles: Seq[Role], authMethod: String) extends AuthContext private def invalidateCache(userId: String) = { cache.remove(s"user-$userId") cache.remove(s"user-org-$userId") } - override def getFromId(request: RequestHeader, userId: String): Future[AuthContext] = { - get(userId).flatMap { user ⇒ getFromUser(request, user) } + override def getFromId(request: RequestHeader, userId: String, authMethod: String): Future[AuthContext] = { + get(userId).flatMap { user ⇒ getFromUser(request, user, authMethod) } } - override def getFromUser(request: RequestHeader, user: org.elastic4play.services.User): Future[AuthContext] = { + override def getFromUser(request: RequestHeader, user: org.elastic4play.services.User, authMethod: String): Future[AuthContext] = { user match { case u: User if u.status() == UserStatus.Ok ⇒ organizationSrv.get(u.organization()).flatMap { - case o if o.status() == OrganizationStatus.Active ⇒ Future.successful(AuthContextImpl(user.id, user.getUserName, Instance.getRequestId(request), user.getRoles)) + case o if o.status() == OrganizationStatus.Active ⇒ Future.successful(AuthContextImpl(user.id, user.getUserName, Instance.getRequestId(request), user.getRoles, authMethod)) case _ ⇒ Future.failed(AuthorizationError("Your account is locked")) } case _ ⇒ Future.failed(AuthorizationError("Your account is locked")) @@ -89,11 +89,11 @@ class UserSrv( override def getInitialUser(request: RequestHeader): Future[AuthContext] = dbIndex.getSize(userModel.modelName).map { case size if size > 0 ⇒ throw AuthenticationError(s"Use of initial user is forbidden because users exist in database") - case _ ⇒ AuthContextImpl("init", "", Instance.getRequestId(request), Roles.roles) + case _ ⇒ AuthContextImpl("init", "", Instance.getRequestId(request), Roles.roles, "init") } override def inInitAuthContext[A](block: AuthContext ⇒ Future[A]): Future[A] = { - val authContext = AuthContextImpl("init", "", Instance.getInternalId, Roles.roles) + val authContext = AuthContextImpl("init", "", Instance.getInternalId, Roles.roles, "init") eventSrv.publish(StreamActor.Initialize(authContext.requestId)) block(authContext).andThen { case _ ⇒ eventSrv.publish(StreamActor.Commit(authContext.requestId)) diff --git a/app/org/thp/cortex/services/WorkerConfigSrv.scala b/app/org/thp/cortex/services/WorkerConfigSrv.scala index 87484767d..ecda03f79 100644 --- a/app/org/thp/cortex/services/WorkerConfigSrv.scala +++ b/app/org/thp/cortex/services/WorkerConfigSrv.scala @@ -2,6 +2,7 @@ package org.thp.cortex.services import scala.concurrent.{ ExecutionContext, Future } +import play.api.Configuration import play.api.libs.json._ import akka.NotUsed @@ -16,6 +17,7 @@ import org.elastic4play.database.ModifyConfig import org.elastic4play.services._ trait WorkerConfigSrv { + val configuration: Configuration val userSrv: UserSrv val createSrv: CreateSrv val updateSrv: UpdateSrv @@ -41,7 +43,7 @@ trait WorkerConfigSrv { .mapMaterializedValue(_ ⇒ NotUsed) .runWith(Sink.seq) .map { baseConfigs ⇒ - (BaseConfig.global(workerType) +: baseConfigs) + (BaseConfig.global(workerType, configuration) +: baseConfigs) .map(c ⇒ c.name → c) .toMap } @@ -94,10 +96,9 @@ trait WorkerConfigSrv { import org.elastic4play.services.QueryDSL._ for { configItems ← definitions - workerConfigItems = configItems workerConfigs ← findForUser(userId, any, Some("all"), Nil) ._1 - .runFold(workerConfigItems) { (definitionConfig, workerConfig) ⇒ updateDefinitionConfig(definitionConfig, workerConfig) } + .runFold(configItems) { (definitionConfig, workerConfig) ⇒ updateDefinitionConfig(definitionConfig, workerConfig) } } yield workerConfigs.values.toSeq } diff --git a/app/org/thp/cortex/services/WorkerSrv.scala b/app/org/thp/cortex/services/WorkerSrv.scala index 31f906248..d5ca255c8 100644 --- a/app/org/thp/cortex/services/WorkerSrv.scala +++ b/app/org/thp/cortex/services/WorkerSrv.scala @@ -1,13 +1,15 @@ package org.thp.cortex.services +import java.net.URL import java.nio.file.{ Files, Path, Paths } -import javax.inject.{ Inject, Singleton } +import javax.inject.{ Inject, Provider, Singleton } import scala.collection.JavaConverters._ import scala.concurrent.{ ExecutionContext, Future } -import scala.util.Try +import scala.util.{ Failure, Success, Try } -import play.api.libs.json.{ JsObject, JsString } +import play.api.libs.json.{ JsObject, JsString, Json } +import play.api.libs.ws.WSClient import play.api.{ Configuration, Logger } import akka.NotUsed @@ -24,55 +26,33 @@ import org.scalactic.Accumulation._ import org.elastic4play.database.ModifyConfig @Singleton -class WorkerSrv( - analyzersPaths: Seq[Path], - respondersPaths: Seq[Path], +class WorkerSrv @Inject() ( + config: Configuration, workerModel: WorkerModel, organizationSrv: OrganizationSrv, + jobRunnerSrvProvider: Provider[JobRunnerSrv], userSrv: UserSrv, createSrv: CreateSrv, getSrv: GetSrv, updateSrv: UpdateSrv, deleteSrv: DeleteSrv, findSrv: FindSrv, + ws: WSClient, implicit val ec: ExecutionContext, implicit val mat: Materializer) { - @Inject() def this( - config: Configuration, - workerModel: WorkerModel, - organizationSrv: OrganizationSrv, - userSrv: UserSrv, - createSrv: CreateSrv, - getSrv: GetSrv, - updateSrv: UpdateSrv, - deleteSrv: DeleteSrv, - findSrv: FindSrv, - ec: ExecutionContext, - mat: Materializer) = this( - config.get[Seq[String]]("analyzer.path").map(p ⇒ Paths.get(p)), - config.get[Seq[String]]("responder.path").map(p ⇒ Paths.get(p)), - workerModel, - organizationSrv, - userSrv, - createSrv, - getSrv, - updateSrv, - deleteSrv, - findSrv, - ec, - mat) - private lazy val logger = Logger(getClass) + private val analyzersURLs: Seq[String] = config.getDeprecated[Seq[String]]("analyzer.urls", "analyzer.path") + private val respondersURLs: Seq[String] = config.getDeprecated[Seq[String]]("responder.urls", "responder.path") + private lazy val jobRunnerSrv: JobRunnerSrv = jobRunnerSrvProvider.get private var workerMap = Map.empty[String, WorkerDefinition] - private object workerMapLock rescan() - def getDefinition(workerId: String): Future[WorkerDefinition] = workerMap.get(workerId) match { - case Some(worker) ⇒ Future.successful(worker) - case None ⇒ Future.failed(NotFoundError(s"Worker $workerId not found")) + def getDefinition(workerId: String): Try[WorkerDefinition] = workerMap.get(workerId) match { + case Some(worker) ⇒ Success(worker) + case None ⇒ Failure(NotFoundError(s"Worker $workerId not found")) } // def listDefinitions: (Source[WorkerDefinition, NotUsed], Future[Long]) = Source(workerMap.values.toList) → Future.successful(workerMap.size.toLong) @@ -135,37 +115,82 @@ class WorkerSrv( } def rescan(): Unit = { - scan(analyzersPaths.map(_ → WorkerType.analyzer) ++ - respondersPaths.map(_ → WorkerType.responder)) + scan(analyzersURLs.map(_ → WorkerType.analyzer) ++ + respondersURLs.map(_ → WorkerType.responder)) } - def scan(workerPaths: Seq[(Path, WorkerType.Type)]): Unit = { - val workers = (for { - (workerPath, workerType) ← workerPaths - workerDir ← Try(Files.newDirectoryStream(workerPath).asScala).getOrElse { - logger.warn(s"Worker directory ($workerPath) is not found") - Nil + def scan(workerUrls: Seq[(String, WorkerType.Type)]): Unit = { + def readUrl(url: URL, workerType: WorkerType.Type): Future[Seq[WorkerDefinition]] = { + url.getProtocol match { + case "file" ⇒ Future.successful(readFile(Paths.get(url.toURI), workerType)) + case "http" | "https" ⇒ + val reads = WorkerDefinition.reads(workerType) + ws.url(url.toString).get().map(response ⇒ response.json.as(reads)) + .map(_.filterNot(_.command.isDefined)) + } + } + + def readFile(path: Path, workerType: WorkerType.Type): Seq[WorkerDefinition] = { + val reads = WorkerDefinition.reads(workerType) + val source = scala.io.Source.fromFile(path.toFile) + lazy val basePath = path.getParent.getParent + val workerDefinitions = + for { + w ← Try(source.mkString).map(Json.parse(_).as(reads)).getOrElse { + logger.error(s"File $path has invalid format") + Nil + } + command = w.command.map(cmd ⇒ basePath.resolve(cmd)) + if command.isEmpty || command.exists(_.normalize().startsWith(basePath)) + } yield w.copy(command = command) + source.close() + workerDefinitions.filter { + case w if w.command.isDefined && jobRunnerSrv.processRunnerIsEnable ⇒ true + case w if w.dockerImage.isDefined && jobRunnerSrv.dockerRunnerIsEnable ⇒ true + case w ⇒ + val reason = if (w.command.isDefined) "process runner is disabled" + else if (w.dockerImage.isDefined) "Docker runner is disabled" + else "it doesn't have image nor command" + + logger.warn(s"$workerType ${w.name} is disabled because $reason") + false } - if Files.isDirectory(workerDir) - infoFile ← Files.newDirectoryStream(workerDir, "*.json").asScala - workerDefinition ← WorkerDefinition.fromPath(infoFile, workerType).fold( - error ⇒ { - logger.warn("Worker definition file read error", error) - Nil - }, - ad ⇒ Seq(ad)) - } yield workerDefinition.id → workerDefinition) - .toMap - - workerMapLock.synchronized { - workerMap = workers } - logger.info(s"New worker list:\n\n\t${workerMap.values.map(a ⇒ s"${a.name} ${a.version}").mkString("\n\t")}\n") + + def readDirectory(path: Path, workerType: WorkerType.Type): Seq[WorkerDefinition] = { + for { + workerDir ← Files.newDirectoryStream(path).asScala.toSeq + infoFile ← Files.newDirectoryStream(workerDir, "*.json").asScala + workerDefinition ← readFile(infoFile, workerType) + } yield workerDefinition + } + + Future + .traverse(workerUrls) { + case (workerUrl, workerType) ⇒ + Future(new URL(workerUrl)).flatMap(readUrl(_, workerType)) + .recover { + case _ ⇒ + val path = Paths.get(workerUrl) + if (Files.isRegularFile(path)) readFile(path, workerType) + else if (Files.isDirectory(path)) readDirectory(path, workerType) + else { + logger.warn(s"Worker path ($workerUrl) is not found") + Nil + } + } + } + .foreach { worker ⇒ + val wmap = worker.flatten.map(w ⇒ w.id -> w).toMap + workerMapLock.synchronized(workerMap = wmap) + logger.info(s"New worker list:\n\n\t${workerMap.values.map(a ⇒ s"${a.name} ${a.version}").mkString("\n\t")}\n") + } + } def create(organization: Organization, workerDefinition: WorkerDefinition, workerFields: Fields)(implicit authContext: AuthContext): Future[Worker] = { val rawConfig = workerFields.getValue("configuration").fold(JsObject.empty)(_.as[JsObject]) - val configItems = workerDefinition.configurationItems ++ BaseConfig.global(workerDefinition.tpe).items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items + val configItems = workerDefinition.configurationItems ++ BaseConfig.global(workerDefinition.tpe, config).items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items val configOrErrors = configItems .validatedBy(_.read(rawConfig)) .map(JsObject.apply) @@ -181,6 +206,13 @@ class WorkerSrv( createSrv[WorkerModel, Worker, Organization](workerModel, organization, workerFields .set("workerDefinitionId", workerDefinition.id) .set("description", workerDefinition.description) + .set("author", workerDefinition.author) + .set("version", workerDefinition.version) + .set("dockerImage", workerDefinition.dockerImage.map(JsString)) + .set("command", workerDefinition.command.map(p ⇒ JsString(p.toString))) + .set("url", workerDefinition.url) + .set("license", workerDefinition.license) + .set("baseConfig", workerDefinition.baseConfiguration.map(JsString.apply)) .set("configuration", cfg.toString) .set("type", workerDefinition.tpe.toString) .addIfAbsent("dataTypeList", StringInputValue(workerDefinition.dataTypeList))) diff --git a/app/org/thp/cortex/services/mappers/GroupUserMapper.scala b/app/org/thp/cortex/services/mappers/GroupUserMapper.scala new file mode 100644 index 000000000..3c940d4d4 --- /dev/null +++ b/app/org/thp/cortex/services/mappers/GroupUserMapper.scala @@ -0,0 +1,72 @@ +package org.thp.cortex.services.mappers + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.Configuration +import play.api.libs.json._ +import play.api.libs.ws.WSClient + +import javax.inject.Inject + +import org.elastic4play.AuthenticationError +import org.elastic4play.controllers.Fields + +class GroupUserMapper( + loginAttrName: String, + nameAttrName: String, + rolesAttrName: Option[String], + groupAttrName: String, + organizationAttrName: Option[String], + defaultRoles: Seq[String], + defaultOrganization: Option[String], + groupsUrl: String, + mappings: Map[String, Seq[String]], + ws: WSClient, + implicit val ec: ExecutionContext) extends UserMapper { + + @Inject() def this( + + configuration: Configuration, + ws: WSClient, + ec: ExecutionContext) = this( + configuration.getOptional[String]("auth.sso.attributes.login").getOrElse("name"), + configuration.getOptional[String]("auth.sso.attributes.name").getOrElse("username"), + configuration.getOptional[String]("auth.sso.attributes.roles"), + configuration.getOptional[String]("auth.sso.attributes.groups").getOrElse(""), + configuration.getOptional[String]("auth.sso.attributes.organization"), + configuration.getOptional[Seq[String]]("auth.sso.defaultRoles").getOrElse(Seq()), + configuration.getOptional[String]("auth.sso.defaultOrganization"), + configuration.getOptional[String]("auth.sso.groups.url").getOrElse(""), + configuration.getOptional[Map[String, Seq[String]]]("auth.sso.groups.mappings").getOrElse(Map()), + ws, + ec) + + override val name: String = "group" + + override def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)]): Future[Fields] = { + + val apiCall = authHeader.fold(ws.url(groupsUrl))(headers ⇒ ws.url(groupsUrl).addHttpHeaders(headers)) + apiCall.get.flatMap { r ⇒ + val jsonGroups = (r.json \ groupAttrName).as[Seq[String]] + val mappedRoles = jsonGroups.flatMap(mappings.get).maxBy(_.length) + val roles = if (mappedRoles.nonEmpty) mappedRoles else defaultRoles + + val fields = for { + login ← (jsValue \ loginAttrName).validate[String] + name ← (jsValue \ nameAttrName).validate[String] + organization ← organizationAttrName + .flatMap(o ⇒ (jsValue \ o).asOpt[String]) + .orElse(defaultOrganization) + .fold[JsResult[String]](JsError())(o ⇒ JsSuccess(o)) + } yield Fields(Json.obj( + "login" → login, + "name" → name, + "roles" → roles, + "organization" → organization)) + fields match { + case JsSuccess(f, _) ⇒ Future.successful(f) + case JsError(errors) ⇒ Future.failed(AuthenticationError(s"User info fails: ${errors.map(_._1).mkString}")) + } + } + } +} diff --git a/app/org/thp/cortex/services/mappers/MultiUserMapperSrv.scala b/app/org/thp/cortex/services/mappers/MultiUserMapperSrv.scala new file mode 100644 index 000000000..238605b2f --- /dev/null +++ b/app/org/thp/cortex/services/mappers/MultiUserMapperSrv.scala @@ -0,0 +1,32 @@ +package org.thp.cortex.services.mappers + +import scala.collection.immutable +import scala.concurrent.Future + +import play.api.Configuration +import play.api.libs.json.JsValue + +import javax.inject.{ Inject, Singleton } + +import org.elastic4play.controllers.Fields + +object MultiUserMapperSrv { + def getMapper(configuration: Configuration, ssoMapperModules: immutable.Set[UserMapper]): UserMapper = { + val name = configuration.getOptional[String]("auth.sso.mapper").getOrElse("simple") + ssoMapperModules.find(_.name == name).get + } +} + +@Singleton +class MultiUserMapperSrv @Inject() ( + configuration: Configuration, + ssoMapperModules: immutable.Set[UserMapper]) extends UserMapper { + + override val name: String = "usermapper" + private lazy val mapper: UserMapper = MultiUserMapperSrv.getMapper(configuration, ssoMapperModules) + + override def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)]): Future[Fields] = { + mapper.getUserFields(jsValue, authHeader) + } + +} diff --git a/app/org/thp/cortex/services/mappers/SimpleUserMapper.scala b/app/org/thp/cortex/services/mappers/SimpleUserMapper.scala new file mode 100644 index 000000000..b7e6919fe --- /dev/null +++ b/app/org/thp/cortex/services/mappers/SimpleUserMapper.scala @@ -0,0 +1,52 @@ +package org.thp.cortex.services.mappers + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.Configuration +import play.api.libs.json._ + +import javax.inject.Inject + +import org.elastic4play.AuthenticationError +import org.elastic4play.controllers.Fields + +class SimpleUserMapper( + loginAttrName: String, + nameAttrName: String, + rolesAttrName: Option[String], + organizationAttrName: Option[String], + defaultRoles: Seq[String], + defaultOrganization: Option[String], + implicit val ec: ExecutionContext) extends UserMapper { + + @Inject() def this(configuration: Configuration, ec: ExecutionContext) = this( + configuration.getOptional[String]("auth.sso.attributes.login").getOrElse("name"), + configuration.getOptional[String]("auth.sso.attributes.name").getOrElse("username"), + configuration.getOptional[String]("auth.sso.attributes.roles"), + configuration.getOptional[String]("auth.sso.attributes.organization"), + configuration.getOptional[Seq[String]]("auth.sso.defaultRoles").getOrElse(Seq()), + configuration.getOptional[String]("auth.sso.defaultOrganization"), + ec) + + override val name: String = "simple" + + override def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)]): Future[Fields] = { + val fields = for { + login ← (jsValue \ loginAttrName).validate[String] + name ← (jsValue \ nameAttrName).validate[String] + roles = rolesAttrName.fold(defaultRoles)(r ⇒ (jsValue \ r).asOpt[Seq[String]].getOrElse(defaultRoles)) + organization ← organizationAttrName + .flatMap(o ⇒ (jsValue \ o).asOpt[String]) + .orElse(defaultOrganization) + .fold[JsResult[String]](JsError())(o ⇒ JsSuccess(o)) + } yield Fields(Json.obj( + "login" → login, + "name" → name, + "roles" → roles, + "organization" → organization)) + fields match { + case JsSuccess(f, _) ⇒ Future.successful(f) + case JsError(errors) ⇒ Future.failed(AuthenticationError(s"User info fails: ${errors.map(_._1).mkString}")) + } + } +} diff --git a/app/org/thp/cortex/services/mappers/UserMapper.scala b/app/org/thp/cortex/services/mappers/UserMapper.scala new file mode 100644 index 000000000..02bf2ae61 --- /dev/null +++ b/app/org/thp/cortex/services/mappers/UserMapper.scala @@ -0,0 +1,16 @@ +package org.thp.cortex.services.mappers + +import scala.concurrent.Future + +import play.api.libs.json.JsValue + +import org.elastic4play.controllers.Fields + +/** + * User mapper trait to be used when converting a JS response from a third party API to a valid Fields object. Used in + * the SSO process to create new users if the option is selected. + */ +trait UserMapper { + val name: String + def getUserFields(jsValue: JsValue, authHeader: Option[(String, String)] = None): Future[Fields] +} diff --git a/build.sbt b/build.sbt index 0a87204bd..00fae0098 100644 --- a/build.sbt +++ b/build.sbt @@ -8,12 +8,14 @@ lazy val cortex = (project in file(".")) libraryDependencies ++= Seq( Dependencies.Play.cache, Dependencies.Play.ws, + Dependencies.Play.ahc, Dependencies.Play.specs2 % Test, Dependencies.Play.guice, Dependencies.scalaGuice, Dependencies.elastic4play, Dependencies.reflections, - Dependencies.zip4j + Dependencies.zip4j, + Dependencies.dockerClient ) resolvers += Resolver.sbtPluginRepo("releases") diff --git a/conf/reference.conf b/conf/reference.conf index 7e65ca6f2..ba7bb932f 100644 --- a/conf/reference.conf +++ b/conf/reference.conf @@ -1,4 +1,4 @@ -http.port=9001 +http.port = 9001 # handler for errors (transform exception to related http status code play.http.errorHandler = org.thp.cortex.services.ErrorHandler play.modules.enabled += org.thp.cortex.Module @@ -9,6 +9,10 @@ cache { organization = 5 minutes } +job { + timeout = 30 minutes + runners = [docker, process] +} # HTTP filters play.filters { @@ -89,7 +93,7 @@ audit.name = audit analyzer { # Directory that holds analyzers - path = [] + urls = [] fork-join-executor { # Min number of threads available for analyze @@ -104,7 +108,7 @@ analyzer { responder { # Directory that holds responders - path = [] + urls = [] fork-join-executor { # Min number of threads available for analyze diff --git a/conf/routes b/conf/routes index 4145627fd..300dd360d 100644 --- a/conf/routes +++ b/conf/routes @@ -7,6 +7,7 @@ GET / org.thp.cort GET /api/health org.thp.cortex.controllers.StatusCtrl.health GET /api/logout org.thp.cortex.controllers.AuthenticationCtrl.logout() POST /api/login org.thp.cortex.controllers.AuthenticationCtrl.login() +POST /api/ssoLogin org.thp.cortex.controllers.AuthenticationCtrl.ssoLogin() ################### # API used by TheHive diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 247b0fcf9..364d0ce38 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,11 +1,12 @@ import sbt._ object Dependencies { - val scalaVersion = "2.12.7" + val scalaVersion = "2.12.8" object Play { val version = play.core.PlayVersion.current val ws = "com.typesafe.play" %% "play-ws" % version + val ahc = "com.typesafe.play" %% "play-ahc-ws" % version val cache = "com.typesafe.play" %% "play-ehcache" % version val test = "com.typesafe.play" %% "play-test" % version val specs2 = "com.typesafe.play" %% "play-specs2" % version @@ -17,6 +18,7 @@ object Dependencies { val reflections = "org.reflections" % "reflections" % "0.9.11" val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2" - val elastic4play = "org.thehive-project" %% "elastic4play" % "1.7.2" + val elastic4play = "org.thehive-project" %% "elastic4play" % "1.10.0" + val dockerClient = "com.spotify" % "docker-client" % "8.14.4" } diff --git a/project/plugins.sbt b/project/plugins.sbt index c2e5b0186..5c35a4bb3 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ logLevel := Level.Info // The Play plugin -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.20") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.21") addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.1") diff --git a/test/resources/analyzers/echoAnalyzer/Dockerfile b/test/resources/analyzers/echoAnalyzer/Dockerfile new file mode 100644 index 000000000..f2ae0eb3e --- /dev/null +++ b/test/resources/analyzers/echoAnalyzer/Dockerfile @@ -0,0 +1,6 @@ +FROM debian:latest + +WORKDIR /analyzer +RUN apt update && apt install -y jq +COPY echoAnalyzer.sh echoAnalyzer/echoAnalyzer.sh +ENTRYPOINT ["echoAnalyzer/echoAnalyzer.sh"] \ No newline at end of file diff --git a/test/resources/analyzers/echoAnalyzer/echoAnalyzer.json b/test/resources/analyzers/echoAnalyzer/echoAnalyzer.json index 98c3117ff..4c7024114 100644 --- a/test/resources/analyzers/echoAnalyzer/echoAnalyzer.json +++ b/test/resources/analyzers/echoAnalyzer/echoAnalyzer.json @@ -55,5 +55,6 @@ ], "description": "Fake analyzer used for functional tests", "dataTypeList": ["domain", "thehive:case", "thehive:case_task", "thehive:case_artifact", "thehive:alert", "thehive:case_task_log"], - "command": "echoAnalyzer/echoAnalyzer.sh" + "command": "echoAnalyzer/echoAnalyzer.sh", + "image": "echo_analyzer" } diff --git a/test/resources/analyzers/echoAnalyzer/echoAnalyzer.sh b/test/resources/analyzers/echoAnalyzer/echoAnalyzer.sh index 39e505a92..0ccbdcc23 100755 --- a/test/resources/analyzers/echoAnalyzer/echoAnalyzer.sh +++ b/test/resources/analyzers/echoAnalyzer/echoAnalyzer.sh @@ -1,21 +1,40 @@ -#!/usr/bin/env bash +#!/bin/bash -ARTIFACT=$(cat) -DATA=$(jq .data <<< ${ARTIFACT}) -DATATYPE=$(jq .dataType <<< ${ARTIFACT}) +set -x -cat << EOF -{ - "success": true, - "summary": { - "taxonomies": [ - { "namespace": "test", "predicate": "data", "value": "echo", "level": "info" } - ] - }, - "full": ${ARTIFACT}, - "operations": [ - { "type": "AddTagToCase", "tag": "From Action Operation" }, - { "type": "CreateTask", "title": "task created by action", "description": "yop !" } - ] -} +echo starting with parameters: $* +for JOB +do + echo executing $JOB + if [[ -d "${JOB}" ]]; then + echo directory $JOB exists + if [[ -r "${JOB}/input/input.json" ]]; then + INPUT=$(cat ${JOB}/input/input.json) + else + INPUT="{}" + fi + echo input is $INPUT + DATA=$(jq .data <<< ${INPUT}) + DATATYPE=$(jq .dataType <<< ${INPUT}) + + echo building output + mkdir -p "${JOB}/output" + cat > "${JOB}/output/output.json" <<- EOF + { + "success": true, + "summary": { + "taxonomies": [ + { "namespace": "test", "predicate": "data", "value": "echo", "level": "info" } + ] + }, + "full": ${INPUT}, + "operations": [ + { "type": "AddTagToCase", "tag": "From Action Operation" }, + { "type": "CreateTask", "title": "task created by action", "description": "yop !" } + ] + } EOF + echo output is: + cat "${JOB}/output/output.json" + fi +done diff --git a/test/resources/analyzers/testAnalyzer/testAnalyzer.json b/test/resources/analyzers/testAnalyzer/testAnalyzer.json new file mode 100644 index 000000000..a74e9d57d --- /dev/null +++ b/test/resources/analyzers/testAnalyzer/testAnalyzer.json @@ -0,0 +1,13 @@ +{ + "name": "testAnalyzer", + "version": "1.0", + "author": "TheHive-Project", + "url": "https://github.com/thehive-project/thehive", + "license": "AGPL-V3", + "baseConfig": "testAnalyzer", + "config": {}, + "configurationItems": [], + "description": "Fake analyzer used for functional tests", + "dataTypeList": ["domain"], + "command": "testAnalyzer/testAnalyzer.py" +} diff --git a/test/resources/analyzers/testAnalyzer/testAnalyzer.py b/test/resources/analyzers/testAnalyzer/testAnalyzer.py new file mode 100755 index 000000000..e1d8df86e --- /dev/null +++ b/test/resources/analyzers/testAnalyzer/testAnalyzer.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from cortexutils.analyzer import Analyzer +from cortexutils import runner + + +class TestAnalyzer(Analyzer): + + def artifacts(self, raw): + return [ + self.build_artifact("ip", "127.0.0.1", tags=["localhost"]), + self.build_artifact("file", "/etc/passwd", tlp=3) + ] + + def summary(self, raw): + return {"taxonomies": [self.build_taxonomy("info", "test", "data", "test")]} + + def run(self): + Analyzer.run(self) + + self.report({ + 'data': self.get_data(), + 'input': self._input + }) + + +if __name__ == '__main__': + runner(TestAnalyzer) diff --git a/version.sbt b/version.sbt index 4de3efb74..b4332167a 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.1.3-1" +version in ThisBuild := "3.0.0-RC1" diff --git a/www/package.json b/www/package.json index 22c06c72e..18ef7fd27 100755 --- a/www/package.json +++ b/www/package.json @@ -1,6 +1,6 @@ { "name": "cortex", - "version": "2.1.2", + "version": "3.0.0-RC1", "description": "A powerfull observable analysis engine", "license": "AGPL-v3", "homepage": "https://github.com/TheHive-Project/Cortex", @@ -53,6 +53,7 @@ "html-loader": "^0.4.4", "html-webpack-plugin": "^2.22.0", "jquery": "^3.2.1", + "js-url": "^2.3.0", "lodash": "^4.17.4", "manifest-revision-webpack-plugin": "^0.3.0", "moment": "^2.20.1", diff --git a/www/src/app/components/about/about.html b/www/src/app/components/about/about.html index c05cd44a9..678415557 100644 --- a/www/src/app/components/about/about.html +++ b/www/src/app/components/about/about.html @@ -20,8 +20,8 @@
Set to True to enable automatic observables extraction from analysis reports.
Define the maximum number of requests and the associated unit if applicable.
-Define the number minutes for analysis report caching, or use the globally defined value.
+Define the maximum number of requests and the associated unit if applicable.
Invalid analyzers have no definition and cannot be run on any observable. You have to remove them.
-Invalid analyzers have no definition and cannot be run on any observable. You have to remove them.
+Available analyzers ({{$ctrl.definitionsIds.length || 0}}) Refresh analyzers
+Available analyzers ({{$ctrl.definitionsIds.length || 0}}) Refresh analyzers