From 4555bf39fd11ed0d8a78fed8f99867a232839e26 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 6 Jul 2018 16:07:05 +0200 Subject: [PATCH] #110 Add responder APIs --- .../controllers/AnalyzerConfigCtrl.scala | 7 +- .../thp/cortex/controllers/AnalyzerCtrl.scala | 44 ++-- app/org/thp/cortex/controllers/MispCtrl.scala | 4 +- .../controllers/ResponderConfigCtrl.scala | 48 ++++ .../cortex/controllers/ResponderCtrl.scala | 123 +++++++++ .../thp/cortex/controllers/StatusCtrl.scala | 4 +- .../thp/cortex/models/AnalyzerConfig.scala | 22 -- app/org/thp/cortex/models/BaseConfig.scala | 31 +++ app/org/thp/cortex/models/Errors.scala | 4 +- app/org/thp/cortex/models/Job.scala | 10 +- app/org/thp/cortex/models/JsonFormat.scala | 2 + app/org/thp/cortex/models/Migration.scala | 19 +- app/org/thp/cortex/models/Organization.scala | 18 +- .../models/{Analyzer.scala => Worker.scala} | 26 +- app/org/thp/cortex/models/WorkerConfig.scala | 25 ++ ...efinition.scala => WorkerDefinition.scala} | 56 ++-- app/org/thp/cortex/models/package.scala | 2 +- .../cortex/services/AnalyzerConfigSrv.scala | 145 +--------- app/org/thp/cortex/services/AnalyzerSrv.scala | 204 -------------- .../thp/cortex/services/ErrorHandler.scala | 12 +- app/org/thp/cortex/services/JobSrv.scala | 88 ++++--- app/org/thp/cortex/services/MispSrv.scala | 8 +- .../cortex/services/ResponderConfigSrv.scala | 25 ++ .../thp/cortex/services/WorkerConfigSrv.scala | 120 +++++++++ app/org/thp/cortex/services/WorkerSrv.scala | 249 ++++++++++++++++++ conf/reference.conf | 15 ++ conf/routes | 26 +- 27 files changed, 840 insertions(+), 497 deletions(-) create mode 100644 app/org/thp/cortex/controllers/ResponderConfigCtrl.scala create mode 100644 app/org/thp/cortex/controllers/ResponderCtrl.scala delete mode 100644 app/org/thp/cortex/models/AnalyzerConfig.scala create mode 100644 app/org/thp/cortex/models/BaseConfig.scala rename app/org/thp/cortex/models/{Analyzer.scala => Worker.scala} (57%) create mode 100644 app/org/thp/cortex/models/WorkerConfig.scala rename app/org/thp/cortex/models/{AnalyzerDefinition.scala => WorkerDefinition.scala} (70%) delete mode 100644 app/org/thp/cortex/services/AnalyzerSrv.scala create mode 100644 app/org/thp/cortex/services/ResponderConfigSrv.scala create mode 100644 app/org/thp/cortex/services/WorkerConfigSrv.scala create mode 100644 app/org/thp/cortex/services/WorkerSrv.scala diff --git a/app/org/thp/cortex/controllers/AnalyzerConfigCtrl.scala b/app/org/thp/cortex/controllers/AnalyzerConfigCtrl.scala index 294122574..0ffa714c7 100644 --- a/app/org/thp/cortex/controllers/AnalyzerConfigCtrl.scala +++ b/app/org/thp/cortex/controllers/AnalyzerConfigCtrl.scala @@ -1,14 +1,13 @@ package org.thp.cortex.controllers import javax.inject.{ Inject, Singleton } - import scala.concurrent.{ ExecutionContext, Future } import play.api.libs.json.JsObject import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } -import org.thp.cortex.models.Roles -import org.thp.cortex.services.{ AnalyzerConfigSrv, BaseConfig, UserSrv } +import org.thp.cortex.models.{ BaseConfig, Roles } +import org.thp.cortex.services.{ AnalyzerConfigSrv, UserSrv } import org.elastic4play.BadRequestError import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } @@ -29,7 +28,7 @@ class AnalyzerConfigCtrl @Inject() ( } def list(): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request ⇒ - analyzerConfigSrv.listForUser(request.userId) + analyzerConfigSrv.listConfigForUser(request.userId) .map { bc ⇒ renderer.toOutput(OK, bc.sortWith { case (BaseConfig("global", _, _, _), _) ⇒ true diff --git a/app/org/thp/cortex/controllers/AnalyzerCtrl.scala b/app/org/thp/cortex/controllers/AnalyzerCtrl.scala index 7f326a7f1..695f1abc6 100644 --- a/app/org/thp/cortex/controllers/AnalyzerCtrl.scala +++ b/app/org/thp/cortex/controllers/AnalyzerCtrl.scala @@ -3,13 +3,13 @@ package org.thp.cortex.controllers import javax.inject.{ Inject, Singleton } import scala.concurrent.{ ExecutionContext, Future } -import play.api.libs.json.{ JsNumber, JsObject, Json } +import play.api.libs.json.{ JsNumber, JsObject, JsString, Json } import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } import akka.stream.Materializer import akka.stream.scaladsl.Sink -import org.thp.cortex.models.{ Analyzer, AnalyzerDefinition, Roles } -import org.thp.cortex.services.{ AnalyzerSrv, UserSrv } +import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition } +import org.thp.cortex.services.{ UserSrv, WorkerSrv } import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites @@ -18,7 +18,7 @@ import org.elastic4play.services.{ QueryDSL, QueryDef } @Singleton class AnalyzerCtrl @Inject() ( - analyzerSrv: AnalyzerSrv, + workerSrv: WorkerSrv, userSrv: UserSrv, authenticated: Authenticated, fieldsBodyParser: FieldsBodyParser, @@ -32,14 +32,14 @@ class AnalyzerCtrl @Inject() ( val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) val isAdmin = request.roles.contains(Roles.orgAdmin) - val (analyzers, analyzerTotal) = analyzerSrv.findForUser(request.userId, query, range, sort) + val (analyzers, analyzerTotal) = workerSrv.findAnalyzersForUser(request.userId, query, range, sort) val enrichedAnalyzers = analyzers.mapAsync(2)(analyzerJson(isAdmin)) renderer.toOutput(OK, enrichedAnalyzers, analyzerTotal) } def get(analyzerId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ val isAdmin = request.roles.contains(Roles.orgAdmin) - analyzerSrv.getForUser(request.userId, analyzerId) + workerSrv.getForUser(request.userId, analyzerId) .flatMap(analyzerJson(isAdmin)) .map(renderer.toOutput(OK, _)) } @@ -52,7 +52,7 @@ class AnalyzerCtrl @Inject() ( "url" -> "unknown", "license" -> "unknown") - private def analyzerJson(analyzer: Analyzer, analyzerDefinition: Option[AnalyzerDefinition]) = { + private def analyzerJson(analyzer: Worker, analyzerDefinition: Option[WorkerDefinition]) = { analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad ⇒ Json.obj( "maxTlp" -> (analyzer.config \ "max_tlp").asOpt[JsNumber], @@ -63,11 +63,11 @@ class AnalyzerCtrl @Inject() ( "url" -> ad.url, "license" -> ad.license, "baseConfig" -> ad.baseConfiguration) - } + } + ("analyzerDefinitionId" -> JsString(analyzer.workerDefinitionId())) // For compatibility reason } - private def analyzerJson(isAdmin: Boolean)(analyzer: Analyzer): Future[JsObject] = { - analyzerSrv.getDefinition(analyzer.analyzerDefinitionId()) + private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): Future[JsObject] = { + workerSrv.getDefinition(analyzer.workerDefinitionId()) .map(analyzerDefinition ⇒ analyzerJson(analyzer, Some(analyzerDefinition))) .recover { case _ ⇒ analyzerJson(analyzer, None) } .map { @@ -78,10 +78,10 @@ class AnalyzerCtrl @Inject() ( def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ import org.elastic4play.services.QueryDSL._ - analyzerSrv.findForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) + workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) ._1 .mapAsyncUnordered(2) { analyzer ⇒ - analyzerSrv.getDefinition(analyzer.analyzerDefinitionId()) + workerSrv.getDefinition(analyzer.workerDefinitionId()) .map(ad ⇒ analyzerJson(analyzer, Some(ad))) } .runWith(Sink.seq) @@ -91,31 +91,33 @@ class AnalyzerCtrl @Inject() ( def create(analyzerDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ for { organizationId ← userSrv.getOrganizationId(request.userId) - analyzer ← analyzerSrv.create(organizationId, analyzerDefinitionId, request.body) - } yield renderer.toOutput(CREATED, analyzer) + workerDefinition ← workerSrv.getDefinition(analyzerDefinitionId) + analyzer ← workerSrv.create(organizationId, workerDefinition, request.body) + } yield renderer.toOutput(CREATED, analyzerJson(analyzer, Some(workerDefinition))) } def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒ - val (analyzers, analyzerTotal) = analyzerSrv.listDefinitions + val (analyzers, analyzerTotal) = workerSrv.listAnalyzerDefinitions renderer.toOutput(OK, analyzers, analyzerTotal) } def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒ - analyzerSrv.rescan() + workerSrv.rescan() NoContent } def delete(analyzerId: String): Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒ for { - analyzer ← analyzerSrv.getForUser(request.userId, analyzerId) - _ ← analyzerSrv.delete(analyzer) + analyzer ← workerSrv.getForUser(request.userId, analyzerId) + _ ← workerSrv.delete(analyzer) } yield NoContent } def update(analyzerId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ for { - analyzer ← analyzerSrv.getForUser(request.userId, analyzerId) - updatedAnalyzer ← analyzerSrv.update(analyzer, request.body) - } yield renderer.toOutput(OK, updatedAnalyzer) + analyzer ← workerSrv.getForUser(request.userId, analyzerId) + updatedAnalyzer ← workerSrv.update(analyzer, request.body) + updatedAnalyzerJson ← analyzerJson(isAdmin = true)(updatedAnalyzer) + } yield renderer.toOutput(OK, updatedAnalyzerJson) } } \ No newline at end of file diff --git a/app/org/thp/cortex/controllers/MispCtrl.scala b/app/org/thp/cortex/controllers/MispCtrl.scala index 783493b92..9ff6759ce 100644 --- a/app/org/thp/cortex/controllers/MispCtrl.scala +++ b/app/org/thp/cortex/controllers/MispCtrl.scala @@ -3,7 +3,7 @@ package org.thp.cortex.controllers import javax.inject.Inject import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.thp.cortex.models.Roles -import org.thp.cortex.services.{ AnalyzerSrv, MispSrv } +import org.thp.cortex.services.{ WorkerSrv, MispSrv } import play.api.Logger import play.api.libs.json.{ JsObject, JsValue } import play.api.mvc._ @@ -12,7 +12,7 @@ import scala.concurrent.{ ExecutionContext, Future } class MispCtrl @Inject() ( mispSrv: MispSrv, - analyzerSrv: AnalyzerSrv, + analyzerSrv: WorkerSrv, authenticated: Authenticated, fieldsBodyParser: FieldsBodyParser, renderer: Renderer, diff --git a/app/org/thp/cortex/controllers/ResponderConfigCtrl.scala b/app/org/thp/cortex/controllers/ResponderConfigCtrl.scala new file mode 100644 index 000000000..c53bd4a2c --- /dev/null +++ b/app/org/thp/cortex/controllers/ResponderConfigCtrl.scala @@ -0,0 +1,48 @@ +package org.thp.cortex.controllers + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.libs.json.JsObject +import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } + +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models.{ BaseConfig, Roles } +import org.thp.cortex.services.{ AnalyzerConfigSrv, UserSrv } + +import org.elastic4play.BadRequestError +import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } + +@Singleton +class ResponderConfigCtrl @Inject() ( + analyzerConfigSrv: AnalyzerConfigSrv, + userSrv: UserSrv, + authenticated: Authenticated, + fieldsBodyParser: FieldsBodyParser, + renderer: Renderer, + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) { + + def get(analyzerConfigName: String): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request ⇒ + analyzerConfigSrv.getForUser(request.userId, analyzerConfigName) + .map(renderer.toOutput(OK, _)) + } + + def list(): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request ⇒ + analyzerConfigSrv.listConfigForUser(request.userId) + .map { bc ⇒ + renderer.toOutput(OK, bc.sortWith { + case (BaseConfig("global", _, _, _), _) ⇒ true + case (_, BaseConfig("global", _, _, _)) ⇒ false + case (BaseConfig(a, _, _, _), BaseConfig(b, _, _, _)) ⇒ a.compareTo(b) < 0 + }) + } + } + + def update(analyzerConfigName: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ + request.body.getValue("config").flatMap(_.asOpt[JsObject]) match { + case Some(config) ⇒ analyzerConfigSrv.updateOrCreate(request.userId, analyzerConfigName, config) + .map(renderer.toOutput(OK, _)) + case None ⇒ Future.failed(BadRequestError("attribute config has invalid format")) + } + } +} \ No newline at end of file diff --git a/app/org/thp/cortex/controllers/ResponderCtrl.scala b/app/org/thp/cortex/controllers/ResponderCtrl.scala new file mode 100644 index 000000000..7ba15896b --- /dev/null +++ b/app/org/thp/cortex/controllers/ResponderCtrl.scala @@ -0,0 +1,123 @@ +package org.thp.cortex.controllers + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.libs.json.{ JsNumber, JsObject, Json } +import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } + +import akka.stream.Materializer +import akka.stream.scaladsl.Sink +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition } +import org.thp.cortex.services.{ UserSrv, WorkerSrv } + +import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } +import org.elastic4play.models.JsonFormat.baseModelEntityWrites +import org.elastic4play.services.JsonFormat.queryReads +import org.elastic4play.services.{ QueryDSL, QueryDef } + +@Singleton +class ResponderCtrl @Inject() ( + workerSrv: WorkerSrv, + userSrv: UserSrv, + authenticated: Authenticated, + fieldsBodyParser: FieldsBodyParser, + renderer: Renderer, + components: ControllerComponents, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) extends AbstractController(components) { + + def find: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { request ⇒ + val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) + val range = request.body.getString("range") + val sort = request.body.getStrings("sort").getOrElse(Nil) + val isAdmin = request.roles.contains(Roles.orgAdmin) + val (responders, responderTotal) = workerSrv.findRespondersForUser(request.userId, query, range, sort) + val enrichedResponders = responders.mapAsync(2)(responderJson(isAdmin)) + renderer.toOutput(OK, enrichedResponders, responderTotal) + } + + def get(responderId: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ + val isAdmin = request.roles.contains(Roles.orgAdmin) + workerSrv.getForUser(request.userId, responderId) + .flatMap(responderJson(isAdmin)) + .map(renderer.toOutput(OK, _)) + } + + private val emptyResponderDefinitionJson = Json.obj( + "version" -> "0.0", + "description" -> "unknown", + "dataTypeList" -> Nil, + "author" -> "unknown", + "url" -> "unknown", + "license" -> "unknown") + + private def responderJson(responder: Worker, responderDefinition: Option[WorkerDefinition]) = { + responder.toJson ++ responderDefinition.fold(emptyResponderDefinitionJson) { ad ⇒ + Json.obj( + "maxTlp" -> (responder.config \ "max_tlp").asOpt[JsNumber], + "maxPap" -> (responder.config \ "max_pap").asOpt[JsNumber], + "version" -> ad.version, + "description" -> ad.description, + "author" -> ad.author, + "url" -> ad.url, + "license" -> ad.license, + "baseConfig" -> ad.baseConfiguration) + } + } + + private def responderJson(isAdmin: Boolean)(responder: Worker): Future[JsObject] = { + workerSrv.getDefinition(responder.workerDefinitionId()) + .map(responderDefinition ⇒ responderJson(responder, Some(responderDefinition))) + .recover { case _ ⇒ responderJson(responder, None) } + .map { + case a if isAdmin ⇒ a + ("configuration" -> Json.parse(responder.configuration())) + case a ⇒ a + } + } + + def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request ⇒ + import org.elastic4play.services.QueryDSL._ + workerSrv.findRespondersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil) + ._1 + .mapAsyncUnordered(2) { responder ⇒ + workerSrv.getDefinition(responder.workerDefinitionId()) + .map(ad ⇒ responderJson(responder, Some(ad))) + } + .runWith(Sink.seq) + .map(responders ⇒ renderer.toOutput(OK, responders)) + } + + def create(responderDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ + for { + organizationId ← userSrv.getOrganizationId(request.userId) + workerDefinition ← workerSrv.getDefinition(responderDefinitionId) + responder ← workerSrv.create(organizationId, workerDefinition, request.body) + } yield renderer.toOutput(CREATED, responderJson(responder, Some(workerDefinition))) + } + + def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒ + val (responders, responderTotal) = workerSrv.listResponderDefinitions + renderer.toOutput(OK, responders, responderTotal) + } + + def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request ⇒ + workerSrv.rescan() + NoContent + } + + def delete(responderId: String): Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request ⇒ + for { + responder ← workerSrv.getForUser(request.userId, responderId) + _ ← workerSrv.delete(responder) + } yield NoContent + } + + def update(responderId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request ⇒ + for { + responder ← workerSrv.getForUser(request.userId, responderId) + updatedResponder ← workerSrv.update(responder, request.body) + updatedResponderJson ← responderJson(isAdmin = true)(updatedResponder) + } yield renderer.toOutput(OK, updatedResponderJson) + } +} \ No newline at end of file diff --git a/app/org/thp/cortex/controllers/StatusCtrl.scala b/app/org/thp/cortex/controllers/StatusCtrl.scala index b380eda7b..ef337d78b 100644 --- a/app/org/thp/cortex/controllers/StatusCtrl.scala +++ b/app/org/thp/cortex/controllers/StatusCtrl.scala @@ -11,7 +11,7 @@ import play.api.libs.json.Json.toJsFieldJsValueWrapper import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents } import com.sksamuel.elastic4s.ElasticDsl -import org.thp.cortex.models.Analyzer +import org.thp.cortex.models.Worker import org.elastic4play.database.DBIndex import org.elastic4play.services.AuthSrv @@ -31,7 +31,7 @@ class StatusCtrl @Inject() ( dbIndex.clusterVersions.map { versions ⇒ Ok(Json.obj( "versions" → Json.obj( - "Cortex" → getVersion(classOf[Analyzer]), + "Cortex" → getVersion(classOf[Worker]), "Elastic4Play" → getVersion(classOf[AuthSrv]), "Play" → getVersion(classOf[AbstractController]), "Elastic4s" → getVersion(classOf[ElasticDsl]), diff --git a/app/org/thp/cortex/models/AnalyzerConfig.scala b/app/org/thp/cortex/models/AnalyzerConfig.scala deleted file mode 100644 index 2fe35c2bd..000000000 --- a/app/org/thp/cortex/models/AnalyzerConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package org.thp.cortex.models - -import javax.inject.{ Inject, Singleton } - -import play.api.libs.json.{ JsObject, Json } - -import org.elastic4play.models.{ AttributeDef, ChildModelDef, EntityDef, AttributeFormat ⇒ F, AttributeOption ⇒ O } - -trait AnalyzerConfigAttributes { _: AttributeDef ⇒ - val name = attribute("name", F.stringFmt, "Analyzer name") - val config = attribute("config", F.textFmt, "Configuration of analyzer", O.sensitive) -} - -@Singleton -class AnalyzerConfigModel @Inject() ( - organizationModel: OrganizationModel) extends ChildModelDef[AnalyzerConfigModel, AnalyzerConfig, OrganizationModel, Organization](organizationModel, "analyzerConfig", "AnalyzerConfig", "/analyzer/config") with AnalyzerConfigAttributes { -} - -class AnalyzerConfig(model: AnalyzerConfigModel, attributes: JsObject) extends EntityDef[AnalyzerConfigModel, AnalyzerConfig](model, attributes) with AnalyzerConfigAttributes { - def organization = parentId.get - def jsonConfig = Json.parse(config()).as[JsObject] -} diff --git a/app/org/thp/cortex/models/BaseConfig.scala b/app/org/thp/cortex/models/BaseConfig.scala new file mode 100644 index 000000000..357ce3274 --- /dev/null +++ b/app/org/thp/cortex/models/BaseConfig.scala @@ -0,0 +1,31 @@ +package org.thp.cortex.models + +import play.api.libs.json._ + +import org.elastic4play.utils.Collection.distinctBy + +case class BaseConfig(name: String, workerNames: Seq[String], items: Seq[ConfigurationDefinitionItem], config: Option[WorkerConfig]) { + def +(other: BaseConfig) = BaseConfig(name, workerNames ++ other.workerNames, distinctBy(items ++ other.items)(_.name), config.orElse(other.config)) +} +object BaseConfig { + implicit val writes: Writes[BaseConfig] = Writes[BaseConfig] { baseConfig ⇒ + Json.obj( + "name" -> baseConfig.name, + "workers" -> baseConfig.workerNames, + "configurationItems" -> baseConfig.items, + "config" -> baseConfig.config.fold(JsObject.empty)(_.jsonConfig)) + } + val global = BaseConfig("global", Nil, Seq( + ConfigurationDefinitionItem("proxy_http", "url of http proxy", WorkerConfigItemType.string, multi = false, required = false, None), + ConfigurationDefinitionItem("proxy_https", "url of https proxy", WorkerConfigItemType.string, multi = false, required = false, None), + ConfigurationDefinitionItem("auto_extract_artifacts", "extract artifacts from full report automatically", WorkerConfigItemType.boolean, multi = false, required = false, Some(JsFalse))), + None) + val tlp = BaseConfig("tlp", Nil, Seq( + ConfigurationDefinitionItem("check_tlp", "", WorkerConfigItemType.boolean, multi = false, required = false, None), + ConfigurationDefinitionItem("max_tlp", "", WorkerConfigItemType.number, multi = false, required = false, None)), + None) + val pap = BaseConfig("pap", Nil, Seq( + ConfigurationDefinitionItem("check_pap", "", WorkerConfigItemType.boolean, multi = false, required = false, None), + ConfigurationDefinitionItem("max_pap", "", WorkerConfigItemType.number, multi = false, required = false, None)), + None) +} \ No newline at end of file diff --git a/app/org/thp/cortex/models/Errors.scala b/app/org/thp/cortex/models/Errors.scala index 64bd22089..0b2ccdad2 100644 --- a/app/org/thp/cortex/models/Errors.scala +++ b/app/org/thp/cortex/models/Errors.scala @@ -3,6 +3,6 @@ package org.thp.cortex.models abstract class CortexError(message: String) extends Exception(message) case class JobNotFoundError(jobId: String) extends CortexError(s"Job $jobId not found") -case class AnalyzerNotFoundError(analyzerId: String) extends CortexError(s"Analyzer $analyzerId not found") +case class WorkerNotFoundError(analyzerId: String) extends CortexError(s"Worker $analyzerId not found") case class UnknownConfigurationItem(item: String) extends CortexError(s"Configuration item $item is not known") -case class RateLimitExceeded(analyzer: Analyzer) extends CortexError(s"Rate limit of ${analyzer.rate().getOrElse("(not set ?!)")} per ${analyzer.rateUnit().getOrElse("(not set ?!)")} reached for the analyzer ${analyzer.name()}. Job cannot be started") \ No newline at end of file +case class RateLimitExceeded(analyzer: Worker) extends CortexError(s"Rate limit of ${analyzer.rate().getOrElse("(not set ?!)")} per ${analyzer.rateUnit().getOrElse("(not set ?!)")} reached for the analyzer ${analyzer.name()}. Job cannot be started") \ No newline at end of file diff --git a/app/org/thp/cortex/models/Job.scala b/app/org/thp/cortex/models/Job.scala index afeb12d48..cbd35a5aa 100644 --- a/app/org/thp/cortex/models/Job.scala +++ b/app/org/thp/cortex/models/Job.scala @@ -16,9 +16,9 @@ object JobStatus extends Enumeration with HiveEnumeration { } trait JobAttributes { _: AttributeDef ⇒ - val analyzerDefinitionId = attribute("analyzerDefinitionId", F.stringFmt, "Analyzer definition id", O.readonly) - val analyzerId = attribute("analyzerId", F.stringFmt, "Analyzer id", O.readonly) - val analyzerName = attribute("analyzerName", F.stringFmt, "Analyzer name", O.readonly) + val workerDefinitionId = attribute("workerDefinitionId", F.stringFmt, "Worker definition id", O.readonly) + val workerId = attribute("workerId", F.stringFmt, "Worker id", O.readonly) + val workerName = attribute("workerName", F.stringFmt, "Worker name", O.readonly) val organization = attribute("organization", F.stringFmt, "Organization ID", O.readonly) val status = attribute("status", F.enumFmt(JobStatus), "Status of the job") val startDate = optionalAttribute("startDate", F.dateFmt, "Analysis start date") @@ -28,9 +28,9 @@ trait JobAttributes { _: AttributeDef ⇒ val attachment = optionalAttribute("attachment", F.attachmentFmt, "Artifact file content", O.readonly) val tlp = attribute("tlp", TlpAttributeFormat, "TLP level", 2L) val message = optionalAttribute("message", F.textFmt, "Message associated to the analysis") - val errorMessage = optionalAttribute("message", F.textFmt, "Message returned by the analyzer when it fails") + val errorMessage = optionalAttribute("message", F.textFmt, "Message returned by the worker when it fails") val parameters = attribute("parameters", F.stringFmt, "Parameters for this job", "{}") - val input = optionalAttribute("input", F.textFmt, "Data sent to analyzer") + val input = optionalAttribute("input", F.textFmt, "Data sent to worker") val fromCache = optionalAttribute("fromCache", F.booleanFmt, "Indicates if cache is used", O.form) } diff --git a/app/org/thp/cortex/models/JsonFormat.scala b/app/org/thp/cortex/models/JsonFormat.scala index e56379ade..34ba0e3fd 100644 --- a/app/org/thp/cortex/models/JsonFormat.scala +++ b/app/org/thp/cortex/models/JsonFormat.scala @@ -2,6 +2,7 @@ package org.thp.cortex.models import play.api.libs.json._ +import org.elastic4play.models.JsonFormat.enumFormat import org.elastic4play.services.Role object JsonFormat { @@ -11,4 +12,5 @@ object JsonFormat { case _ ⇒ JsError(Seq(JsPath → Seq(JsonValidationError(s"error.expected.role(${Roles.roleNames}")))) } implicit val roleFormat: Format[Role] = Format[Role](roleReads, roleWrites) + implicit val workerTypeFormat: Format[WorkerType.Value] = enumFormat(WorkerType) } diff --git a/app/org/thp/cortex/models/Migration.scala b/app/org/thp/cortex/models/Migration.scala index bc26b0bf5..44b22b06c 100644 --- a/app/org/thp/cortex/models/Migration.scala +++ b/app/org/thp/cortex/models/Migration.scala @@ -1,15 +1,15 @@ package org.thp.cortex.models import javax.inject.{ Inject, Singleton } - import scala.concurrent.{ ExecutionContext, Future } import scala.util.Success -import play.api.libs.json.Json +import play.api.libs.json.{ JsString, Json } import org.thp.cortex.services.{ OrganizationSrv, UserSrv } import org.elastic4play.controllers.Fields +import org.elastic4play.services.Operation._ import org.elastic4play.services.{ DatabaseState, MigrationOperations, Operation } @Singleton @@ -31,6 +31,21 @@ class Migration @Inject() ( } val operations: PartialFunction[DatabaseState, Seq[Operation]] = { + case DatabaseState(1) ⇒ Seq( + // add type to analyzer + addAttribute("analyzer", "type" -> JsString("analyzer")), + + renameAttribute("job", "workerDefinitionId", "analyzerDefinitionId"), + renameAttribute("job", "workerId", "analyzerId"), + renameAttribute("job", "workerName", "analyzerName"), + + renameEntity("analyzer", "worker"), + renameAttribute("worker", "workerDefinitionId", "analyzerDefinitionId"), + addAttribute("worker", "type" -> JsString(WorkerType.analyzer.toString)), + + renameEntity("analyzerConfig", "workerConfig"), + addAttribute("workerConfig", "type" -> JsString(WorkerType.analyzer.toString))) + case _ ⇒ Nil } } diff --git a/app/org/thp/cortex/models/Organization.scala b/app/org/thp/cortex/models/Organization.scala index 79937b5e6..ee8f027ad 100644 --- a/app/org/thp/cortex/models/Organization.scala +++ b/app/org/thp/cortex/models/Organization.scala @@ -28,12 +28,12 @@ trait OrganizationAttributes { _: AttributeDef ⇒ class OrganizationModel @Inject() ( findSrv: FindSrv, userModelProvider: Provider[UserModel], - analyzerModelProvider: Provider[AnalyzerModel], + workerModelProvider: Provider[WorkerModel], implicit val ec: ExecutionContext) extends ModelDef[OrganizationModel, Organization]("organization", "Organization", "/organization") with OrganizationAttributes with AuditedModel { private lazy val logger = Logger(getClass) lazy val userModel = userModelProvider.get - lazy val analyzerModel = analyzerModelProvider.get + lazy val workerModel = workerModelProvider.get override def removeAttribute = Json.obj("status" -> "Locked") override def creationHook(parent: Option[BaseEntity], attrs: JsObject): Future[JsObject] = @@ -59,19 +59,19 @@ class OrganizationModel @Inject() ( } } - private def buildAnalyzerStats(organization: Organization): Future[JsObject] = { + private def buildWorkerStats(organization: Organization): Future[JsObject] = { import org.elastic4play.services.QueryDSL._ findSrv( - analyzerModel, + workerModel, withParent(organization), groupByField("status", selectCount)) - .map { analyzerStatsJson ⇒ - val (analyzerCount, analyzerStats) = analyzerStatsJson.value.foldLeft((0L, JsObject.empty)) { + .map { workerStatsJson ⇒ + val (workerCount, workerStats) = workerStatsJson.value.foldLeft((0L, JsObject.empty)) { case ((total, s), (key, value)) ⇒ val count = (value \ "count").as[Long] (total + count, s + (key → JsNumber(count))) } - Json.obj("analyzers" → (analyzerStats + ("total" → JsNumber(analyzerCount)))) + Json.obj("workers" → (workerStats + ("total" → JsNumber(workerCount)))) } } @@ -80,8 +80,8 @@ class OrganizationModel @Inject() ( case organization: Organization ⇒ for { userStats ← buildUserStats(organization) - analyzerStats ← buildAnalyzerStats(organization) - } yield userStats ++ analyzerStats + workerStats ← buildWorkerStats(organization) + } yield userStats ++ workerStats case other ⇒ logger.warn(s"Request caseStats from a non-case entity ?! ${other.getClass}:$other") Future.successful(Json.obj()) diff --git a/app/org/thp/cortex/models/Analyzer.scala b/app/org/thp/cortex/models/Worker.scala similarity index 57% rename from app/org/thp/cortex/models/Analyzer.scala rename to app/org/thp/cortex/models/Worker.scala index d276c769f..7c00e77dc 100644 --- a/app/org/thp/cortex/models/Analyzer.scala +++ b/app/org/thp/cortex/models/Worker.scala @@ -11,6 +11,8 @@ import org.elastic4play.models.JsonFormat.enumFormat import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O } import org.elastic4play.utils.Hasher +import org.thp.cortex.models.JsonFormat.workerTypeFormat + object RateUnit extends Enumeration with HiveEnumeration { type Type = Value val Day = Value(1) @@ -18,20 +20,26 @@ object RateUnit extends Enumeration with HiveEnumeration { implicit val reads = enumFormat(this) } -trait AnalyzerAttributes { _: AttributeDef ⇒ - val analyzerId = attribute("_id", F.stringFmt, "Analyzer id", O.model) - val name = attribute("name", F.stringFmt, "Analyzer name") - val analyzerDefinitionId = attribute("analyzerDefinitionId", F.stringFmt, "Analyzer definition id", O.readonly) - val description = attribute("description", F.textFmt, "Analyzer description") - val dataTypeList = multiAttribute("dataTypeList", F.stringFmt, "List of data type this analyzer can manage") - val configuration = attribute("configuration", F.textFmt, "Configuration of analyzer", O.sensitive) +object WorkerType extends Enumeration with HiveEnumeration { + type Type = Value + val analyzer, responder = Value +} + +trait WorkerAttributes { _: AttributeDef ⇒ + val workerId = attribute("_id", F.stringFmt, "Worker id", O.model) + val name = attribute("name", F.stringFmt, "Worker name") + val workerDefinitionId = attribute("workerDefinitionId", F.stringFmt, "Worker definition id", O.readonly) + val description = attribute("description", F.textFmt, "Worker description") + val dataTypeList = multiAttribute("dataTypeList", F.stringFmt, "List of data type this worker can manage") + val configuration = attribute("configuration", F.textFmt, "Configuration of the worker", O.sensitive) val rate = optionalAttribute("rate", F.numberFmt, "Number ") val rateUnit = optionalAttribute("rateUnit", F.enumFmt(RateUnit), "") val jobCache = optionalAttribute("jobCache", F.numberFmt, "") + val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly) } @Singleton -class AnalyzerModel @Inject() (organizationModel: OrganizationModel) extends ChildModelDef[AnalyzerModel, Analyzer, OrganizationModel, Organization](organizationModel, "analyzer", "Analyzer", "/analyzer") with AnalyzerAttributes with AuditedModel { +class WorkerModel @Inject() (organizationModel: OrganizationModel) extends ChildModelDef[WorkerModel, Worker, OrganizationModel, Organization](organizationModel, "worker", "Worker", "/worker") with WorkerAttributes with AuditedModel { override def creationHook(parent: Option[BaseEntity], attrs: JsObject): Future[JsObject] = { val hasher = Hasher("md5") val id = for { @@ -42,7 +50,7 @@ class AnalyzerModel @Inject() (organizationModel: OrganizationModel) extends Chi } } -class Analyzer(model: AnalyzerModel, attributes: JsObject) extends EntityDef[AnalyzerModel, Analyzer](model, attributes) with AnalyzerAttributes { +class Worker(model: WorkerModel, attributes: JsObject) extends EntityDef[WorkerModel, Worker](model, attributes) with WorkerAttributes { def config: JsObject = Try(Json.parse(configuration()).as[JsObject]).getOrElse(JsObject.empty) def organization = parentId.get } diff --git a/app/org/thp/cortex/models/WorkerConfig.scala b/app/org/thp/cortex/models/WorkerConfig.scala new file mode 100644 index 000000000..75dd67336 --- /dev/null +++ b/app/org/thp/cortex/models/WorkerConfig.scala @@ -0,0 +1,25 @@ +package org.thp.cortex.models + +import javax.inject.{ Inject, Singleton } + +import play.api.libs.json.{ JsObject, Json } + +import org.elastic4play.models.{ AttributeDef, ChildModelDef, EntityDef, AttributeFormat ⇒ F, AttributeOption ⇒ O } + +import org.thp.cortex.models.JsonFormat.workerTypeFormat + +trait WorkerConfigAttributes { _: AttributeDef ⇒ + val name = attribute("name", F.stringFmt, "Worker name") + val config = attribute("config", F.textFmt, "Configuration of worker", O.sensitive) + val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly) +} + +@Singleton +class WorkerConfigModel @Inject() ( + organizationModel: OrganizationModel) extends ChildModelDef[WorkerConfigModel, WorkerConfig, OrganizationModel, Organization](organizationModel, "workerConfig", "WorkerConfig", "/worker/config") with WorkerConfigAttributes { +} + +class WorkerConfig(model: WorkerConfigModel, attributes: JsObject) extends EntityDef[WorkerConfigModel, WorkerConfig](model, attributes) with WorkerConfigAttributes { + def organization = parentId.get + def jsonConfig = Json.parse(config()).as[JsObject] +} diff --git a/app/org/thp/cortex/models/AnalyzerDefinition.scala b/app/org/thp/cortex/models/WorkerDefinition.scala similarity index 70% rename from app/org/thp/cortex/models/AnalyzerDefinition.scala rename to app/org/thp/cortex/models/WorkerDefinition.scala index f9269d54e..e5994aa01 100644 --- a/app/org/thp/cortex/models/AnalyzerDefinition.scala +++ b/app/org/thp/cortex/models/WorkerDefinition.scala @@ -17,16 +17,16 @@ import org.elastic4play.models.HiveEnumeration import org.elastic4play.models.JsonFormat.enumFormat import org.elastic4play.{ AttributeError, InvalidFormatAttributeError, MissingAttributeError } -object AnalyzerConfigItemType extends Enumeration with HiveEnumeration { +object WorkerConfigItemType extends Enumeration with HiveEnumeration { type Type = Value val string, number, boolean = Value - implicit val reads = enumFormat(this) + implicit val reads: Format[WorkerConfigItemType.Type] = enumFormat(this) } case class ConfigurationDefinitionItem( name: String, description: String, - `type`: AnalyzerConfigItemType.Type, + `type`: WorkerConfigItemType.Type, multi: Boolean, required: Boolean, defaultValue: Option[JsValue]) { @@ -35,7 +35,7 @@ case class ConfigurationDefinitionItem( def isMulti: Boolean = multi private def check(v: JsValue): JsValue Or Every[AttributeError] = { - import AnalyzerConfigItemType._ + import WorkerConfigItemType._ v match { case _: JsString if `type` == string ⇒ Good(v) case _: JsNumber if `type` == number ⇒ Good(v) @@ -65,14 +65,14 @@ object ConfigurationDefinitionItem { implicit val reads: Reads[ConfigurationDefinitionItem] = ( (JsPath \ "name").read[String] and (JsPath \ "description").read[String] and - (JsPath \ "type").read[AnalyzerConfigItemType.Type] and + (JsPath \ "type").read[WorkerConfigItemType.Type] and (JsPath \ "multi").readWithDefault[Boolean](false) and (JsPath \ "required").readWithDefault[Boolean](false) and (JsPath \ "defaultValue").readNullable[JsValue])(ConfigurationDefinitionItem.apply _) implicit val writes: Writes[ConfigurationDefinitionItem] = Json.writes[ConfigurationDefinitionItem] } -case class AnalyzerDefinition( +case class WorkerDefinition( name: String, version: String, description: String, @@ -84,26 +84,27 @@ case class AnalyzerDefinition( command: String, baseConfiguration: Option[String], configurationItems: Seq[ConfigurationDefinitionItem], - configuration: JsObject) { - val id = (name + "_" + version).replaceAll("\\.", "_") + configuration: JsObject, + tpe: WorkerType.Type) { + val id: String = (name + "_" + version).replaceAll("\\.", "_") def canProcessDataType(dataType: String): Boolean = dataTypeList.contains(dataType) } -object AnalyzerDefinition { +object WorkerDefinition { lazy val logger = Logger(getClass) - def fromPath(definitionFile: Path): Try[AnalyzerDefinition] = { + def fromPath(definitionFile: Path, workerType: WorkerType.Type): Try[WorkerDefinition] = { readJsonFile(definitionFile) .recoverWith { case error ⇒ - logger.warn(s"Load of analyzer $definitionFile fails", error) + logger.warn(s"Load of worker $definitionFile fails", error) Failure(error) } - .map(_.validate(AnalyzerDefinition.reads(definitionFile.getParent.getParent))) + .map(_.validate(WorkerDefinition.reads(definitionFile.getParent.getParent, workerType))) .flatMap { - case JsSuccess(analyzerDefinition, _) ⇒ Success(analyzerDefinition) - case JsError(errors) ⇒ sys.error(s"Json description file $definitionFile is invalid: $errors") + case JsSuccess(workerDefinition, _) ⇒ Success(workerDefinition) + case JsError(errors) ⇒ sys.error(s"Json description file $definitionFile is invalid: $errors") } } @@ -114,7 +115,7 @@ object AnalyzerDefinition { json } - def reads(path: Path): Reads[AnalyzerDefinition] = ( + def reads(path: Path, workerType: WorkerType.Type): Reads[WorkerDefinition] = ( (JsPath \ "name").read[String] and (JsPath \ "version").read[String] and (JsPath \ "description").read[String] and @@ -126,18 +127,19 @@ object AnalyzerDefinition { (JsPath \ "command").read[String] and (JsPath \ "baseConfig").readNullable[String] and (JsPath \ "configurationItems").read[Seq[ConfigurationDefinitionItem]].orElse(Reads.pure(Nil)) and - (JsPath \ "config").read[JsObject].orElse(Reads.pure(JsObject.empty)))(AnalyzerDefinition.apply _) - implicit val writes: Writes[AnalyzerDefinition] = Writes[AnalyzerDefinition] { analyzerDefinition ⇒ + (JsPath \ "config").read[JsObject].orElse(Reads.pure(JsObject.empty)) and + Reads.pure(workerType))(WorkerDefinition.apply _) + implicit val writes: Writes[WorkerDefinition] = Writes[WorkerDefinition] { workerDefinition ⇒ Json.obj( - "id" -> analyzerDefinition.id, - "name" -> analyzerDefinition.name, - "version" -> analyzerDefinition.version, - "description" -> analyzerDefinition.description, - "dataTypeList" -> analyzerDefinition.dataTypeList, - "author" -> analyzerDefinition.author, - "url" -> analyzerDefinition.url, - "license" -> analyzerDefinition.license, - "baseConfig" -> analyzerDefinition.baseConfiguration, - "configurationItems" -> analyzerDefinition.configurationItems) + "id" -> workerDefinition.id, + "name" -> workerDefinition.name, + "version" -> workerDefinition.version, + "description" -> workerDefinition.description, + "dataTypeList" -> workerDefinition.dataTypeList, + "author" -> workerDefinition.author, + "url" -> workerDefinition.url, + "license" -> workerDefinition.license, + "baseConfig" -> workerDefinition.baseConfiguration, + "configurationItems" -> workerDefinition.configurationItems) } } \ No newline at end of file diff --git a/app/org/thp/cortex/models/package.scala b/app/org/thp/cortex/models/package.scala index a916ffefd..4ee41c710 100644 --- a/app/org/thp/cortex/models/package.scala +++ b/app/org/thp/cortex/models/package.scala @@ -1,5 +1,5 @@ package org.thp.cortex package object models { - val modelVersion = 1 + val modelVersion = 2 } diff --git a/app/org/thp/cortex/services/AnalyzerConfigSrv.scala b/app/org/thp/cortex/services/AnalyzerConfigSrv.scala index 22f969a34..fbc1d2fd4 100644 --- a/app/org/thp/cortex/services/AnalyzerConfigSrv.scala +++ b/app/org/thp/cortex/services/AnalyzerConfigSrv.scala @@ -1,145 +1,26 @@ package org.thp.cortex.services -import javax.inject.{ Inject, Singleton } - import scala.concurrent.{ ExecutionContext, Future } -import play.api.libs.json._ - -import akka.NotUsed import akka.stream.Materializer -import akka.stream.scaladsl.{ Sink, Source } -import org.thp.cortex.models._ -import org.scalactic.Accumulation._ +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models.{ BaseConfig, WorkerConfigModel, WorkerType } -import org.elastic4play.{ AttributeCheckingError, NotFoundError } -import org.elastic4play.controllers.Fields -import org.elastic4play.database.ModifyConfig -import org.elastic4play.services._ -import org.elastic4play.utils.Collection.distinctBy - -case class BaseConfig(name: String, analyzerNames: Seq[String], items: Seq[ConfigurationDefinitionItem], config: Option[AnalyzerConfig]) { - def +(other: BaseConfig) = BaseConfig(name, analyzerNames ++ other.analyzerNames, distinctBy(items ++ other.items)(_.name), config.orElse(other.config)) -} -object BaseConfig { - implicit val writes: Writes[BaseConfig] = Writes[BaseConfig] { baseConfig ⇒ - Json.obj( - "name" -> baseConfig.name, - "analyzers" -> baseConfig.analyzerNames, - "configurationItems" -> baseConfig.items, - "config" -> baseConfig.config.fold(JsObject.empty)(_.jsonConfig)) - } - val global = BaseConfig("global", Nil, Seq( - ConfigurationDefinitionItem("proxy_http", "url of http proxy", AnalyzerConfigItemType.string, multi = false, required = false, None), - ConfigurationDefinitionItem("proxy_https", "url of https proxy", AnalyzerConfigItemType.string, multi = false, required = false, None), - ConfigurationDefinitionItem("auto_extract_artifacts", "extract artifacts from full report automatically", AnalyzerConfigItemType.boolean, multi = false, required = false, Some(JsFalse))), - None) - val tlp = BaseConfig("tlp", Nil, Seq( - ConfigurationDefinitionItem("check_tlp", "", AnalyzerConfigItemType.boolean, multi = false, required = false, None), - ConfigurationDefinitionItem("max_tlp", "", AnalyzerConfigItemType.number, multi = false, required = false, None)), - None) - val pap = BaseConfig("pap", Nil, Seq( - ConfigurationDefinitionItem("check_pap", "", AnalyzerConfigItemType.boolean, multi = false, required = false, None), - ConfigurationDefinitionItem("max_pap", "", AnalyzerConfigItemType.number, multi = false, required = false, None)), - None) -} +import org.elastic4play.services.{ CreateSrv, FindSrv, UpdateSrv } @Singleton class AnalyzerConfigSrv @Inject() ( - analyzerConfigModel: AnalyzerConfigModel, - userSrv: UserSrv, - organizationSrv: OrganizationSrv, - analyzerSrv: AnalyzerSrv, - createSrv: CreateSrv, - updateSrv: UpdateSrv, - findSrv: FindSrv, + val workerConfigModel: WorkerConfigModel, + val userSrv: UserSrv, + val organizationSrv: OrganizationSrv, + val workerSrv: WorkerSrv, + val createSrv: CreateSrv, + val updateSrv: UpdateSrv, + val findSrv: FindSrv, implicit val ec: ExecutionContext, - implicit val mat: Materializer) { + implicit val mat: Materializer) extends WorkerConfigSrv { + override val workerType: WorkerType.Type = WorkerType.analyzer def definitions: Future[Map[String, BaseConfig]] = - analyzerSrv.listDefinitions._1 - .filter(_.baseConfiguration.isDefined) - .map(ad ⇒ ad.copy(configurationItems = ad.configurationItems.map(_.copy(required = false)))) - .groupBy(200, _.baseConfiguration.get) - .map(ad ⇒ BaseConfig(ad.baseConfiguration.get, Seq(ad.name), ad.configurationItems, None)) - .reduce(_ + _) - .filterNot(_.items.isEmpty) - .mergeSubstreams - .mapMaterializedValue(_ ⇒ NotUsed) - .runWith(Sink.seq) - .map { baseConfigs ⇒ - (BaseConfig.global +: baseConfigs) - .map(c ⇒ c.name -> c) - .toMap - } - - def getForUser(userId: String, analyzerConfigName: String): Future[BaseConfig] = { - userSrv.getOrganizationId(userId) - .flatMap(organizationId ⇒ getForOrganization(organizationId, analyzerConfigName)) - } - - def getForOrganization(organizationId: String, analyzerConfigName: String): Future[BaseConfig] = { - import org.elastic4play.services.QueryDSL._ - for { - analyzerConfig ← findForOrganization(organizationId, "name" ~= analyzerConfigName, Some("0-1"), Nil) - ._1 - .runWith(Sink.headOption) - d ← definitions - baseConfig ← d.get(analyzerConfigName).fold[Future[BaseConfig]](Future.failed(NotFoundError(s"analyzerConfig $analyzerConfigName not found")))(Future.successful) - } yield baseConfig.copy(config = analyzerConfig) - } - - def create(organization: Organization, fields: Fields)(implicit authContext: AuthContext): Future[AnalyzerConfig] = { - createSrv[AnalyzerConfigModel, AnalyzerConfig, Organization](analyzerConfigModel, organization, fields) - } - - def update(analyzerConfig: AnalyzerConfig, fields: Fields)(implicit authContext: AuthContext): Future[AnalyzerConfig] = { - updateSrv(analyzerConfig, fields, ModifyConfig.default) - } - - def updateOrCreate(userId: String, analyzerConfigName: String, config: JsObject)(implicit authContext: AuthContext): Future[BaseConfig] = { - for { - organizationId ← userSrv.getOrganizationId(userId) - organization ← organizationSrv.get(organizationId) - baseConfig ← getForOrganization(organizationId, analyzerConfigName) - validatedConfig ← baseConfig.items.validatedBy(_.read(config)) - .map(_.filterNot(_._2 == JsNull)) - .fold(c ⇒ Future.successful(Fields.empty.set("config", JsObject(c).toString).set("name", analyzerConfigName)), errors ⇒ Future.failed(AttributeCheckingError("analyzerConfig", errors.toSeq))) - newAnalyzerConfig ← baseConfig.config.fold(create(organization, validatedConfig))(analyzerConfig ⇒ update(analyzerConfig, validatedConfig)) - } yield baseConfig.copy(config = Some(newAnalyzerConfig)) - } - - def updateDefinitionConfig(definitionConfig: Map[String, BaseConfig], analyzerConfig: AnalyzerConfig): Map[String, BaseConfig] = { - definitionConfig.get(analyzerConfig.name()) - .fold(definitionConfig) { baseConfig ⇒ - definitionConfig + (analyzerConfig.name() -> baseConfig.copy(config = Some(analyzerConfig))) - } - } - - def listForUser(userId: String): Future[Seq[BaseConfig]] = { - import org.elastic4play.services.QueryDSL._ - for { - analyzerConfigItems ← definitions - analyzerConfigs ← findForUser(userId, any, Some("all"), Nil) - ._1 - .runFold(analyzerConfigItems) { (definitionConfig, analyzerConfig) ⇒ updateDefinitionConfig(definitionConfig, analyzerConfig) } - } yield analyzerConfigs.values.toSeq - } - - def findForUser(userId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[AnalyzerConfig, NotUsed], Future[Long]) = { - val analyzerConfigs = userSrv.getOrganizationId(userId) - .map(organizationId ⇒ findForOrganization(organizationId, queryDef, range, sortBy)) - val analyserConfigSource = Source.fromFutureSource(analyzerConfigs.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) - val analyserConfigTotal = analyzerConfigs.flatMap(_._2) - analyserConfigSource -> analyserConfigTotal - } - - def findForOrganization(organizationId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[AnalyzerConfig, NotUsed], Future[Long]) = { - import org.elastic4play.services.QueryDSL._ - find(and(withParent("organization", organizationId), queryDef), range, sortBy) - } - - def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[AnalyzerConfig, NotUsed], Future[Long]) = { - findSrv[AnalyzerConfigModel, AnalyzerConfig](analyzerConfigModel, queryDef, range, sortBy) - } + buildDefinitionMap(workerSrv.listAnalyzerDefinitions._1) } diff --git a/app/org/thp/cortex/services/AnalyzerSrv.scala b/app/org/thp/cortex/services/AnalyzerSrv.scala deleted file mode 100644 index d023ee1d0..000000000 --- a/app/org/thp/cortex/services/AnalyzerSrv.scala +++ /dev/null @@ -1,204 +0,0 @@ -package org.thp.cortex.services - -import java.nio.file.{ Files, Path, Paths } -import javax.inject.{ Inject, Singleton } - -import scala.collection.JavaConverters._ -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.Try - -import play.api.libs.json.{ JsObject, JsString } -import play.api.{ Configuration, Logger } - -import akka.NotUsed -import akka.stream.Materializer -import akka.stream.scaladsl.{ Sink, Source } -import org.thp.cortex.models._ - -import org.elastic4play._ -import org.elastic4play.controllers.{ Fields, StringInputValue } -import org.elastic4play.services._ -import org.scalactic._ -import org.scalactic.Accumulation._ - -import org.elastic4play.database.ModifyConfig - -@Singleton -class AnalyzerSrv( - analyzersPaths: Seq[Path], - analyzerModel: AnalyzerModel, - organizationSrv: OrganizationSrv, - userSrv: UserSrv, - createSrv: CreateSrv, - getSrv: GetSrv, - updateSrv: UpdateSrv, - deleteSrv: DeleteSrv, - findSrv: FindSrv, - implicit val ec: ExecutionContext, - implicit val mat: Materializer) { - - @Inject() def this( - config: Configuration, - analyzerModel: AnalyzerModel, - organizationSrv: OrganizationSrv, - userSrv: UserSrv, - createSrv: CreateSrv, - getSrv: GetSrv, - updateSrv: UpdateSrv, - deleteSrv: DeleteSrv, - findSrv: FindSrv, - ec: ExecutionContext, - mat: Materializer) = this( - config.get[Seq[String]]("analyzer.path").map(p ⇒ Paths.get(p)), - analyzerModel, - organizationSrv, - userSrv, - createSrv, - getSrv, - updateSrv, - deleteSrv, - findSrv, - ec, - mat) - - private lazy val logger = Logger(getClass) - private var analyzerMap = Map.empty[String, AnalyzerDefinition] - - private object analyzerMapLock - - rescan() - - def getDefinition(analyzerId: String): Future[AnalyzerDefinition] = analyzerMap.get(analyzerId) match { - case Some(analyzer) ⇒ Future.successful(analyzer) - case None ⇒ Future.failed(NotFoundError(s"Analyzer $analyzerId not found")) - } - - def listDefinitions: (Source[AnalyzerDefinition, NotUsed], Future[Long]) = Source(analyzerMap.values.toList) -> Future.successful(analyzerMap.size.toLong) - - def get(analyzerId: String): Future[Analyzer] = getSrv[AnalyzerModel, Analyzer](analyzerModel, analyzerId) - - def getForUser(userId: String, analyzerId: String): Future[Analyzer] = { - userSrv.getOrganizationId(userId) - .flatMap(organization ⇒ getForOrganization(organization, analyzerId)) - } - - def getForOrganization(organizationId: String, analyzerId: String): Future[Analyzer] = { - import org.elastic4play.services.QueryDSL._ - find( - and(withParent("organization", organizationId), withId(analyzerId)), - Some("0-1"), Nil)._1 - .runWith(Sink.headOption) - .map(_.getOrElse(throw NotFoundError(s"analyzer $analyzerId not found"))) - } - - def listForOrganization(organizationId: String): (Source[Analyzer, NotUsed], Future[Long]) = { - import org.elastic4play.services.QueryDSL._ - findForOrganization(organizationId, any, Some("all"), Nil) - } - - def listForUser(userId: String): (Source[Analyzer, NotUsed], Future[Long]) = { - import org.elastic4play.services.QueryDSL._ - findForUser(userId, any, Some("all"), Nil) - } - - def findForUser(userId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Analyzer, NotUsed], Future[Long]) = { - val analyzers = for { - user ← userSrv.get(userId) - organizationId = user.organization() - } yield findForOrganization(organizationId, queryDef, range, sortBy) - val analyserSource = Source.fromFutureSource(analyzers.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) - val analyserTotal = analyzers.flatMap(_._2) - analyserSource -> analyserTotal - } - - def findForOrganization(organizationId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Analyzer, NotUsed], Future[Long]) = { - import org.elastic4play.services.QueryDSL._ - find(and(withParent("organization", organizationId), queryDef), range, sortBy) - } - - def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Analyzer, NotUsed], Future[Long]) = { - findSrv[AnalyzerModel, Analyzer](analyzerModel, queryDef, range, sortBy) - } - - def rescan(): Unit = { - scan(analyzersPaths) - } - - def scan(analyzerPaths: Seq[Path]): Unit = { - val analyzers = (for { - analyzerPath ← analyzerPaths - analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).asScala).getOrElse { - logger.warn(s"Analyzer directory ($analyzerPath) is not found") - Nil - } - if Files.isDirectory(analyzerDir) - infoFile ← Files.newDirectoryStream(analyzerDir, "*.json").asScala - analyzerDefinition ← AnalyzerDefinition.fromPath(infoFile).fold( - error ⇒ { - logger.warn("Analyzer definition file read error", error) - Nil - }, - ad ⇒ Seq(ad)) - } yield analyzerDefinition.id -> analyzerDefinition) - .toMap - - analyzerMapLock.synchronized { - analyzerMap = analyzers - } - logger.info(s"New analyzer list:\n\n\t${analyzerMap.values.map(a ⇒ s"${a.name} ${a.version}").mkString("\n\t")}\n") - } - - def create(organization: Organization, analyzerDefinition: AnalyzerDefinition, analyzerFields: Fields)(implicit authContext: AuthContext): Future[Analyzer] = { - val rawConfig = analyzerFields.getValue("configuration").fold(JsObject.empty)(_.as[JsObject]) - val configItems = analyzerDefinition.configurationItems ++ BaseConfig.global.items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items - val configOrErrors = configItems - .validatedBy(_.read(rawConfig)) - .map(JsObject.apply) - - val unknownConfigItems = (rawConfig.value.keySet -- configItems.map(_.name)) - .foldLeft[Unit Or Every[AttributeError]](Good(())) { - case (Good(_), ci) ⇒ Bad(One(UnknownAttributeError("analyzer.config", JsString(ci)))) - case (Bad(e), ci) ⇒ Bad(UnknownAttributeError("analyzer.config", JsString(ci)) +: e) - } - - withGood(configOrErrors, unknownConfigItems)((c, _) ⇒ c) - .fold(cfg ⇒ { - createSrv[AnalyzerModel, Analyzer, Organization](analyzerModel, organization, analyzerFields - .set("analyzerDefinitionId", analyzerDefinition.id) - .set("description", analyzerDefinition.description) - .set("configuration", cfg.toString) - .addIfAbsent("dataTypeList", StringInputValue(analyzerDefinition.dataTypeList))) - - }, { - case One(e) ⇒ Future.failed(e) - case Every(es @ _*) ⇒ Future.failed(AttributeCheckingError(s"analyzer(${analyzerDefinition.name}).configuration", es)) - }) - } - - def create(organizationId: String, analyzerDefinitionId: String, analyzerFields: Fields)(implicit authContext: AuthContext): Future[Analyzer] = { - for { - organization ← organizationSrv.get(organizationId) - analyzerDefinition ← getDefinition(analyzerDefinitionId) - analyzer ← create(organization, analyzerDefinition, analyzerFields) - } yield analyzer - } - - def delete(analyzer: Analyzer)(implicit authContext: AuthContext): Future[Unit] = - deleteSrv.realDelete(analyzer) - - def delete(analyzerId: String)(implicit authContext: AuthContext): Future[Unit] = - deleteSrv.realDelete[AnalyzerModel, Analyzer](analyzerModel, analyzerId) - - def update(analyzer: Analyzer, fields: Fields)(implicit authContext: AuthContext): Future[Analyzer] = update(analyzer, fields, ModifyConfig.default) - - def update(analyzer: Analyzer, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Analyzer] = { - val analyzerFields = fields.getValue("configuration").fold(fields)(cfg ⇒ fields.set("configuration", cfg.toString)) - updateSrv(analyzer, analyzerFields, modifyConfig) - } - - def update(analyzerId: String, fields: Fields)(implicit authContext: AuthContext): Future[Analyzer] = update(analyzerId, fields, ModifyConfig.default) - - def update(analyzerId: String, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Analyzer] = { - get(analyzerId).flatMap(analyzer ⇒ update(analyzer, fields, modifyConfig)) - } -} \ No newline at end of file diff --git a/app/org/thp/cortex/services/ErrorHandler.scala b/app/org/thp/cortex/services/ErrorHandler.scala index 000e5f45c..dde025a18 100644 --- a/app/org/thp/cortex/services/ErrorHandler.scala +++ b/app/org/thp/cortex/services/ErrorHandler.scala @@ -1,6 +1,6 @@ package org.thp.cortex.services -import org.thp.cortex.models.{ AnalyzerNotFoundError, JobNotFoundError, RateLimitExceeded } +import org.thp.cortex.models.{ WorkerNotFoundError, JobNotFoundError, RateLimitExceeded } import play.api.Logger import play.api.http.{ HttpErrorHandler, Status, Writeable } import play.api.mvc.{ RequestHeader, ResponseHeader, Result, Results } @@ -43,11 +43,11 @@ class ErrorHandler extends HttpErrorHandler { case Some((_, j)) ⇒ j } Some(Status.MULTI_STATUS → Json.obj("type" → "MultiError", "error" → message, "suberrors" → suberrors)) - case JobNotFoundError(jobId) ⇒ Some(Status.NOT_FOUND -> Json.obj("type" -> "JobNotFoundError", "message" -> s"Job $jobId not found")) - case AnalyzerNotFoundError(analyzerId) ⇒ Some(Status.NOT_FOUND -> Json.obj("type" -> "AnalyzerNotFoundError", "message" -> s"analyzer $analyzerId not found")) - case _: IndexNotFoundException ⇒ Some(520 → JsNull) - case qse: QueryShardException ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "Invalid search query", "message" → qse.getMessage)) - case t: Throwable ⇒ Option(t.getCause).flatMap(toErrorResult) + case JobNotFoundError(jobId) ⇒ Some(Status.NOT_FOUND -> Json.obj("type" -> "JobNotFoundError", "message" -> s"Job $jobId not found")) + case WorkerNotFoundError(analyzerId) ⇒ Some(Status.NOT_FOUND -> Json.obj("type" -> "AnalyzerNotFoundError", "message" -> s"analyzer $analyzerId not found")) + case _: IndexNotFoundException ⇒ Some(520 → JsNull) + case qse: QueryShardException ⇒ Some(Status.BAD_REQUEST → Json.obj("type" → "Invalid search query", "message" → qse.getMessage)) + case t: Throwable ⇒ Option(t.getCause).flatMap(toErrorResult) } } diff --git a/app/org/thp/cortex/services/JobSrv.scala b/app/org/thp/cortex/services/JobSrv.scala index c22419e02..31297532f 100644 --- a/app/org/thp/cortex/services/JobSrv.scala +++ b/app/org/thp/cortex/services/JobSrv.scala @@ -31,7 +31,7 @@ class JobSrv( jobModel: JobModel, reportModel: ReportModel, artifactModel: ArtifactModel, - analyzerSrv: AnalyzerSrv, + workerSrv: WorkerSrv, userSrv: UserSrv, getSrv: GetSrv, createSrv: CreateSrv, @@ -48,7 +48,7 @@ class JobSrv( jobModel: JobModel, reportModel: ReportModel, artifactModel: ArtifactModel, - analyzerSrv: AnalyzerSrv, + workerSrv: WorkerSrv, userSrv: UserSrv, getSrv: GetSrv, createSrv: CreateSrv, @@ -63,7 +63,7 @@ class JobSrv( jobModel, reportModel, artifactModel, - analyzerSrv, + workerSrv, userSrv, getSrv, createSrv, @@ -75,7 +75,8 @@ class JobSrv( ec, mat) private lazy val logger = Logger(getClass) - private lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") + private lazy val analyzerExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer") + private lazy val responderExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("responder") private val osexec = if (System.getProperty("os.name").toLowerCase.contains("win")) (c: String) ⇒ s"""cmd /c $c""" @@ -91,9 +92,9 @@ class JobSrv( ._1 .runForeach { job ⇒ (for { - analyzer ← analyzerSrv.get(job.analyzerId()) - analyzerDefinition ← analyzerSrv.getDefinition(job.analyzerId()) - updatedJob ← run(analyzerDefinition, analyzer, job) + worker ← workerSrv.get(job.workerId()) + workerDefinition ← workerSrv.getDefinition(job.workerId()) + updatedJob ← run(workerDefinition, worker, job) } yield updatedJob) .onComplete { case Success(j) ⇒ logger.info(s"Job ${job.id} has finished with status ${j.status()}") @@ -159,7 +160,7 @@ class JobSrv( def delete(job: Job)(implicit authContext: AuthContext): Future[Job] = deleteSrv(job) - def legacyCreate(analyzer: Analyzer, attributes: JsObject, fields: Fields)(implicit authContext: AuthContext): Future[Job] = { + def legacyCreate(worker: Worker, attributes: JsObject, fields: Fields)(implicit authContext: AuthContext): Future[Job] = { val dataType = Or.from((attributes \ "dataType").asOpt[String], One(MissingAttributeError("dataType"))) val dataFiv = fields.get("data") match { case Some(fiv: FileInputValue) ⇒ Good(Right(fiv)) @@ -178,7 +179,7 @@ class JobSrv( } .fold( typeDataAttachment ⇒ typeDataAttachment._2.flatMap( - da ⇒ create(analyzer, typeDataAttachment._1, da, tlp, message, parameters, force)), + da ⇒ create(worker, typeDataAttachment._1, da, tlp, message, parameters, force)), errors ⇒ { val attributeError = AttributeCheckingError("job", errors) logger.error("legacy job create fails", attributeError) @@ -187,7 +188,7 @@ class JobSrv( } def create(analyzerId: String, fields: Fields)(implicit authContext: AuthContext): Future[Job] = { - analyzerSrv.getForUser(authContext.userId, analyzerId).flatMap { analyzer ⇒ + workerSrv.getForUser(authContext.userId, analyzerId).flatMap { worker ⇒ /* In Cortex 1, fields looks like: { @@ -218,7 +219,7 @@ class JobSrv( "optional parameters": "value" } */ - fields.getValue("attributes").map(attributes ⇒ legacyCreate(analyzer, attributes.as[JsObject], fields)).getOrElse { + fields.getValue("attributes").map(attributes ⇒ legacyCreate(worker, attributes.as[JsObject], fields)).getOrElse { val dataType = Or.from(fields.getString("dataType"), One(MissingAttributeError("dataType"))) val dataFiv = (fields.get("data"), fields.getString("data"), fields.get("attachment")) match { case (_, Some(data), None) ⇒ Good(Left(data)) @@ -242,24 +243,24 @@ class JobSrv( case (dt, Left(data)) ⇒ dt -> Future.successful(Left(data)) } .fold( - typeDataAttachment ⇒ typeDataAttachment._2.flatMap(da ⇒ create(analyzer, typeDataAttachment._1, da, tlp, message, parameters, force)), + typeDataAttachment ⇒ typeDataAttachment._2.flatMap(da ⇒ create(worker, typeDataAttachment._1, da, tlp, message, parameters, force)), errors ⇒ Future.failed(AttributeCheckingError("job", errors))) } } } - def create(analyzer: Analyzer, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, message: String, parameters: JsObject, force: Boolean)(implicit authContext: AuthContext): Future[Job] = { + def create(worker: Worker, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, message: String, parameters: JsObject, force: Boolean)(implicit authContext: AuthContext): Future[Job] = { val previousJob = if (force) Future.successful(None) - else findSimilarJob(analyzer, dataType, dataAttachment, tlp, parameters) + else findSimilarJob(worker, dataType, dataAttachment, tlp, parameters) previousJob.flatMap { case Some(job) ⇒ Future.successful(job) - case None ⇒ isUnderRateLimit(analyzer).flatMap { + case None ⇒ isUnderRateLimit(worker).flatMap { case true ⇒ val fields = Fields(Json.obj( - "analyzerDefinitionId" -> analyzer.analyzerDefinitionId(), - "analyzerId" -> analyzer.id, - "analyzerName" -> analyzer.name(), - "organization" -> analyzer.parentId, + "analyzerDefinitionId" -> worker.workerDefinitionId(), + "analyzerId" -> worker.id, + "analyzerName" -> worker.name(), + "organization" -> worker.parentId, "status" -> JobStatus.Waiting, "dataType" -> dataType, "tlp" -> tlp, @@ -269,10 +270,10 @@ class JobSrv( case Left(data) ⇒ fields.set("data", data) case Right(attachment) ⇒ fields.set("attachment", AttachmentInputValue(attachment)) } - analyzerSrv.getDefinition(analyzer.analyzerDefinitionId()).flatMap { analyzerDefinition ⇒ + workerSrv.getDefinition(worker.workerDefinitionId()).flatMap { workerDefinition ⇒ createSrv[JobModel, Job](jobModel, fieldWithData).andThen { case Success(job) ⇒ - run(analyzerDefinition, analyzer, job) + run(workerDefinition, worker, job) .onComplete { case Success(j) ⇒ logger.info(s"Job ${job.id} has finished with status ${j.status()}") case Failure(e) ⇒ logger.error(s"Job ${job.id} has failed", e) @@ -280,22 +281,22 @@ class JobSrv( } } case false ⇒ - Future.failed(RateLimitExceeded(analyzer)) + Future.failed(RateLimitExceeded(worker)) } } } - private def isUnderRateLimit(analyzer: Analyzer): Future[Boolean] = { + private def isUnderRateLimit(worker: Worker): Future[Boolean] = { (for { - rate ← analyzer.rate() - rateUnit ← analyzer.rateUnit() + rate ← worker.rate() + rateUnit ← worker.rateUnit() } yield { import org.elastic4play.services.QueryDSL._ val now = new Date().getTime - logger.info(s"Checking rate limit on analyzer ${analyzer.name()} from ${new Date(now - rateUnit.id.toLong * 24 * 60 * 60 * 1000)}") - stats(and("createdAt" ~>= (now - rateUnit.id.toLong * 24 * 60 * 60 * 1000), "analyzerId" ~= analyzer.id), Seq(selectCount)).map { stats ⇒ - val count = (stats \ "count").as[Long] + logger.info(s"Checking rate limit on worker ${worker.name()} from ${new Date(now - rateUnit.id.toLong * 24 * 60 * 60 * 1000)}") + stats(and("createdAt" ~>= (now - rateUnit.id.toLong * 24 * 60 * 60 * 1000), "analyzerId" ~= worker.id), Seq(selectCount)).map { s ⇒ + val count = (s \ "count").as[Long] logger.info(s"$count analysis found (limit is $rate)") count < rate } @@ -303,18 +304,18 @@ class JobSrv( .getOrElse(Future.successful(true)) } - def findSimilarJob(analyzer: Analyzer, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject): Future[Option[Job]] = { - val cache = analyzer.jobCache().fold(jobCache)(_.minutes) + def findSimilarJob(worker: Worker, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject): Future[Option[Job]] = { + val cache = worker.jobCache().fold(jobCache)(_.minutes) if (cache.length == 0) { logger.info("Job cache is disabled") Future.successful(None) } else { import org.elastic4play.services.QueryDSL._ - logger.info(s"Looking for similar job (analyzer=${analyzer.id}, dataType=$dataType, data=$dataAttachment, tlp=$tlp, parameters=$parameters") + logger.info(s"Looking for similar job (worker=${worker.id}, dataType=$dataType, data=$dataAttachment, tlp=$tlp, parameters=$parameters") val now = new Date().getTime find(and( - "analyzerId" ~= analyzer.id, + "analyzerId" ~= worker.id, "status" ~!= JobStatus.Failure, "status" ~!= JobStatus.Deleted, "startDate" ~>= (now - cache.toMillis), @@ -336,15 +337,19 @@ class JobSrv( rename("type", "dataType"))(artifact) } - def run(analyzerDefinition: AnalyzerDefinition, analyzer: Analyzer, job: Job)(implicit authContext: AuthContext): Future[Job] = { - buildInput(analyzerDefinition, analyzer, job) + def run(workerDefinition: WorkerDefinition, worker: Worker, job: Job)(implicit authContext: AuthContext): Future[Job] = { + val executionContext = workerDefinition.tpe match { + case WorkerType.analyzer ⇒ analyzerExecutionContext + case WorkerType.responder ⇒ responderExecutionContext + } + buildInput(workerDefinition, worker, job) .flatMap { input ⇒ startJob(job) var output = "" var error = "" try { - logger.info(s"Execute ${osexec(analyzerDefinition.command)} in ${analyzerDefinition.baseDirectory}") - Process(osexec(analyzerDefinition.command), analyzerDefinition.baseDirectory.toFile).run( + logger.info(s"Execute ${osexec(workerDefinition.command)} in ${workerDefinition.baseDirectory}") + Process(osexec(workerDefinition.command), workerDefinition.baseDirectory.toFile).run( new ProcessIO( { stdin ⇒ Try(stdin.write(input.toString.getBytes("UTF-8"))); stdin.close() }, { stdout ⇒ output = readStream(stdout) }, @@ -383,7 +388,7 @@ class JobSrv( val errorMessage = (error + output).take(8192) endJob(job, JobStatus.Failure, Some(s"Invalid output\n$errorMessage")) } - }(analyzeExecutionContext) + }(executionContext) } def getReport(jobId: String)(implicit authContext: AuthContext): Future[Report] = getForUser(authContext.userId, jobId).flatMap(getReport) @@ -395,7 +400,7 @@ class JobSrv( .map(_.getOrElse(throw NotFoundError(s"Job ${job.id} has no report"))) } - private def buildInput(analyzerDefinition: AnalyzerDefinition, analyzer: Analyzer, job: Job): Future[JsObject] = { + private def buildInput(workerDefinition: WorkerDefinition, worker: Worker, job: Job): Future[JsObject] = { job.attachment() .map { attachment ⇒ val tempFile = Files.createTempFile(s"cortex-job-${job.id}-", "") @@ -417,9 +422,9 @@ class JobSrv( "data" -> job.data().get) } .map { artifact ⇒ - (BaseConfig.global.items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items ++ analyzerDefinition.configurationItems) - .validatedBy(_.read(analyzer.config)) - .map(cfg ⇒ Json.obj("config" -> JsObject(cfg).deepMerge(analyzerDefinition.configuration))) + (BaseConfig.global.items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items ++ workerDefinition.configurationItems) + .validatedBy(_.read(worker.config)) + .map(cfg ⇒ Json.obj("config" -> JsObject(cfg).deepMerge(workerDefinition.configuration))) .map { cfg ⇒ val proxy_http = (cfg \ "config" \ "proxy_http").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" -> Json.obj("http" -> proxy)) } val proxy_https = (cfg \ "config" \ "proxy_https").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" -> Json.obj("https" -> proxy)) } @@ -436,6 +441,7 @@ class JobSrv( .flatMap(Future.fromTry) } + // private def startJob(job: Job)(implicit authContext: AuthContext): Future[Job] = { val fields = Fields.empty .set("status", JobStatus.InProgress.toString) diff --git a/app/org/thp/cortex/services/MispSrv.scala b/app/org/thp/cortex/services/MispSrv.scala index fbe9263c6..2586472ab 100644 --- a/app/org/thp/cortex/services/MispSrv.scala +++ b/app/org/thp/cortex/services/MispSrv.scala @@ -21,7 +21,7 @@ import org.elastic4play.services._ @Singleton class MispSrv @Inject() ( - analyzerSrv: AnalyzerSrv, + workerSrv: WorkerSrv, attachmentSrv: AttachmentSrv, jobSrv: JobSrv, @Named("audit") auditActor: ActorRef, @@ -31,11 +31,11 @@ class MispSrv @Inject() ( private[MispSrv] lazy val logger = Logger(getClass) def moduleList(implicit authContext: AuthContext): (Source[JsObject, NotUsed], Future[Long]) = { - val (analyzers, analyzerCount) = analyzerSrv.findForUser(authContext.userId, QueryDSL.any, Some("all"), Nil) + val (analyzers, analyzerCount) = workerSrv.findAnalyzersForUser(authContext.userId, QueryDSL.any, Some("all"), Nil) val mispAnalyzers = analyzers .mapAsyncUnordered(1) { analyzer ⇒ - analyzerSrv.getDefinition(analyzer.analyzerDefinitionId()) + workerSrv.getDefinition(analyzer.workerDefinitionId()) .map(ad ⇒ Some(analyzer -> ad)) .recover { case _ ⇒ None } } @@ -62,7 +62,7 @@ class MispSrv @Inject() ( val duration = 20.minutes // TODO configurable for { - analyzer ← analyzerSrv.get(module) + analyzer ← workerSrv.get(module) job ← jobSrv.create(analyzer, mispType2dataType(mispType), artifact, 0, "", JsObject.empty, force = false) _ ← auditActor.ask(Register(job.id, duration))(Timeout(duration)) updatedJob ← jobSrv.getForUser(authContext.userId, job.id) diff --git a/app/org/thp/cortex/services/ResponderConfigSrv.scala b/app/org/thp/cortex/services/ResponderConfigSrv.scala new file mode 100644 index 000000000..50712a4ff --- /dev/null +++ b/app/org/thp/cortex/services/ResponderConfigSrv.scala @@ -0,0 +1,25 @@ +package org.thp.cortex.services + +import scala.concurrent.{ ExecutionContext, Future } + +import akka.stream.Materializer +import javax.inject.{ Inject, Singleton } +import org.thp.cortex.models.{ BaseConfig, WorkerConfigModel, WorkerType } + +import org.elastic4play.services.{ CreateSrv, FindSrv, UpdateSrv } + +@Singleton +class ResponderConfigSrv @Inject() ( + val workerConfigModel: WorkerConfigModel, + val userSrv: UserSrv, + val organizationSrv: OrganizationSrv, + val workerSrv: WorkerSrv, + val createSrv: CreateSrv, + val updateSrv: UpdateSrv, + val findSrv: FindSrv, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) extends WorkerConfigSrv { + + override val workerType: WorkerType.Type = WorkerType.responder + def definitions: Future[Map[String, BaseConfig]] = buildDefinitionMap(workerSrv.listResponderDefinitions._1) +} diff --git a/app/org/thp/cortex/services/WorkerConfigSrv.scala b/app/org/thp/cortex/services/WorkerConfigSrv.scala new file mode 100644 index 000000000..96ebbcad2 --- /dev/null +++ b/app/org/thp/cortex/services/WorkerConfigSrv.scala @@ -0,0 +1,120 @@ +package org.thp.cortex.services + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.libs.json._ + +import akka.NotUsed +import akka.stream.Materializer +import akka.stream.scaladsl.{ Sink, Source } +import org.thp.cortex.models._ +import org.scalactic.Accumulation._ + +import org.elastic4play.{ AttributeCheckingError, NotFoundError } +import org.elastic4play.controllers.Fields +import org.elastic4play.database.ModifyConfig +import org.elastic4play.services._ + +trait WorkerConfigSrv { + val userSrv: UserSrv + val createSrv: CreateSrv + val updateSrv: UpdateSrv + val workerConfigModel: WorkerConfigModel + val organizationSrv: OrganizationSrv + val findSrv: FindSrv + implicit val ec: ExecutionContext + implicit val mat: Materializer + + val workerType: WorkerType.Type + + def definitions: Future[Map[String, BaseConfig]] + + protected def buildDefinitionMap(definitionSource: Source[WorkerDefinition, NotUsed]): Future[Map[String, BaseConfig]] = { + definitionSource + .filter(_.baseConfiguration.isDefined) + .map(d ⇒ d.copy(configurationItems = d.configurationItems.map(_.copy(required = false)))) + .groupBy(200, _.baseConfiguration.get) // TODO replace groupBy by fold to prevent "too many streams" error + .map(d ⇒ BaseConfig(d.baseConfiguration.get, Seq(d.name), d.configurationItems, None)) + .reduce(_ + _) + .filterNot(_.items.isEmpty) + .mergeSubstreams + .mapMaterializedValue(_ ⇒ NotUsed) + .runWith(Sink.seq) + .map { baseConfigs ⇒ + (BaseConfig.global +: baseConfigs) + .map(c ⇒ c.name -> c) + .toMap + } + } + + def getForUser(userId: String, configName: String): Future[BaseConfig] = { + userSrv.getOrganizationId(userId) + .flatMap(organizationId ⇒ getForOrganization(organizationId, configName)) + } + + def getForOrganization(organizationId: String, configName: String): Future[BaseConfig] = { + import org.elastic4play.services.QueryDSL._ + for { + analyzerConfig ← findForOrganization(organizationId, "name" ~= configName, Some("0-1"), Nil) + ._1 + .runWith(Sink.headOption) + d ← definitions + baseConfig ← d.get(configName).fold[Future[BaseConfig]](Future.failed(NotFoundError(s"config $configName not found")))(Future.successful) + } yield baseConfig.copy(config = analyzerConfig) + } + + def create(organization: Organization, fields: Fields)(implicit authContext: AuthContext): Future[WorkerConfig] = { + createSrv[WorkerConfigModel, WorkerConfig, Organization](workerConfigModel, organization, fields.set("type", workerType.toString)) + } + + def update(analyzerConfig: WorkerConfig, fields: Fields)(implicit authContext: AuthContext): Future[WorkerConfig] = { + updateSrv(analyzerConfig, fields, ModifyConfig.default) + } + + def updateOrCreate(userId: String, analyzerConfigName: String, config: JsObject)(implicit authContext: AuthContext): Future[BaseConfig] = { + for { + organizationId ← userSrv.getOrganizationId(userId) + organization ← organizationSrv.get(organizationId) + baseConfig ← getForOrganization(organizationId, analyzerConfigName) + validatedConfig ← baseConfig.items.validatedBy(_.read(config)) + .map(_.filterNot(_._2 == JsNull)) + .fold(c ⇒ Future.successful(Fields.empty.set("config", JsObject(c).toString).set("name", analyzerConfigName)), errors ⇒ Future.failed(AttributeCheckingError("analyzerConfig", errors.toSeq))) + newAnalyzerConfig ← baseConfig.config.fold(create(organization, validatedConfig))(analyzerConfig ⇒ update(analyzerConfig, validatedConfig)) + } yield baseConfig.copy(config = Some(newAnalyzerConfig)) + } + + private def updateDefinitionConfig(definitionConfig: Map[String, BaseConfig], analyzerConfig: WorkerConfig): Map[String, BaseConfig] = { + definitionConfig.get(analyzerConfig.name()) + .fold(definitionConfig) { baseConfig ⇒ + definitionConfig + (analyzerConfig.name() -> baseConfig.copy(config = Some(analyzerConfig))) + } + } + + def listConfigForUser(userId: String): Future[Seq[BaseConfig]] = { + import org.elastic4play.services.QueryDSL._ + for { + configItems ← definitions + analyzerConfigItems = configItems + analyzerConfigs ← findForUser(userId, any, Some("all"), Nil) + ._1 + .runFold(analyzerConfigItems) { (definitionConfig, analyzerConfig) ⇒ updateDefinitionConfig(definitionConfig, analyzerConfig) } + } yield analyzerConfigs.values.toSeq + } + + def findForUser(userId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[WorkerConfig, NotUsed], Future[Long]) = { + val configs = userSrv.getOrganizationId(userId) + .map(organizationId ⇒ findForOrganization(organizationId, queryDef, range, sortBy)) + val configSource = Source.fromFutureSource(configs.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val configTotal = configs.flatMap(_._2) + configSource -> configTotal + } + + def findForOrganization(organizationId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[WorkerConfig, NotUsed], Future[Long]) = { + import org.elastic4play.services.QueryDSL._ + find(and(withParent("organization", organizationId), "type" ~= workerType, queryDef), range, sortBy) + } + + def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[WorkerConfig, NotUsed], Future[Long]) = { + findSrv[WorkerConfigModel, WorkerConfig](workerConfigModel, queryDef, range, sortBy) + } +} diff --git a/app/org/thp/cortex/services/WorkerSrv.scala b/app/org/thp/cortex/services/WorkerSrv.scala new file mode 100644 index 000000000..2315ddf4f --- /dev/null +++ b/app/org/thp/cortex/services/WorkerSrv.scala @@ -0,0 +1,249 @@ +package org.thp.cortex.services + +import java.nio.file.{ Files, Path, Paths } +import javax.inject.{ Inject, Singleton } + +import scala.collection.JavaConverters._ +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Try + +import play.api.libs.json.{ JsObject, JsString } +import play.api.{ Configuration, Logger } + +import akka.NotUsed +import akka.stream.Materializer +import akka.stream.scaladsl.{ Sink, Source } +import org.thp.cortex.models._ + +import org.elastic4play._ +import org.elastic4play.controllers.{ Fields, StringInputValue } +import org.elastic4play.services._ +import org.scalactic._ +import org.scalactic.Accumulation._ + +import org.elastic4play.database.ModifyConfig + +@Singleton +class WorkerSrv( + analyzersPaths: Seq[Path], + workersPaths: Seq[Path], + workerModel: WorkerModel, + organizationSrv: OrganizationSrv, + userSrv: UserSrv, + createSrv: CreateSrv, + getSrv: GetSrv, + updateSrv: UpdateSrv, + deleteSrv: DeleteSrv, + findSrv: FindSrv, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) { + + @Inject() def this( + config: Configuration, + analyzerModel: WorkerModel, + organizationSrv: OrganizationSrv, + userSrv: UserSrv, + createSrv: CreateSrv, + getSrv: GetSrv, + updateSrv: UpdateSrv, + deleteSrv: DeleteSrv, + findSrv: FindSrv, + ec: ExecutionContext, + mat: Materializer) = this( + config.get[Seq[String]]("analyzer.path").map(p ⇒ Paths.get(p)), + config.get[Seq[String]]("responder.path").map(p ⇒ Paths.get(p)), + analyzerModel, + organizationSrv, + userSrv, + createSrv, + getSrv, + updateSrv, + deleteSrv, + findSrv, + ec, + mat) + + private lazy val logger = Logger(getClass) + private var workerMap = Map.empty[String, WorkerDefinition] + + private object workerMapLock + + rescan() + + def getDefinition(workerId: String): Future[WorkerDefinition] = workerMap.get(workerId) match { + case Some(worker) ⇒ Future.successful(worker) + case None ⇒ Future.failed(NotFoundError(s"Worker $workerId not found")) + } + + // def listDefinitions: (Source[WorkerDefinition, NotUsed], Future[Long]) = Source(workerMap.values.toList) -> Future.successful(workerMap.size.toLong) + + def listAnalyzerDefinitions: (Source[WorkerDefinition, NotUsed], Future[Long]) = { + val analyzerDefinitions = workerMap.values.filter(_.tpe == WorkerType.analyzer) + Source(analyzerDefinitions.toList) -> Future.successful(analyzerDefinitions.size.toLong) + } + + def listResponderDefinitions: (Source[WorkerDefinition, NotUsed], Future[Long]) = { + val analyzerDefinitions = workerMap.values.filter(_.tpe == WorkerType.responder) + Source(analyzerDefinitions.toList) -> Future.successful(analyzerDefinitions.size.toLong) + } + + def get(workerId: String): Future[Worker] = getSrv[WorkerModel, Worker](workerModel, workerId) + + def getForUser(userId: String, workerId: String): Future[Worker] = { + userSrv.getOrganizationId(userId) + .flatMap(organization ⇒ getForOrganization(organization, workerId)) + } + + def getForOrganization(organizationId: String, workerId: String): Future[Worker] = { + import org.elastic4play.services.QueryDSL._ + find( + and(withParent("organization", organizationId), withId(workerId)), + Some("0-1"), Nil)._1 + .runWith(Sink.headOption) + .map(_.getOrElse(throw NotFoundError(s"worker $workerId not found"))) + } + + // private def listForOrganization(organizationId: String): (Source[Worker, NotUsed], Future[Long]) = { + // import org.elastic4play.services.QueryDSL._ + // findForOrganization(organizationId, any, Some("all"), Nil) + // } + // + // def listAnalyzerForOrganization(organizationId: String): (Source[Worker, NotUsed], Future[Long]) = { + // import org.elastic4play.services.QueryDSL._ + // findForOrganization(organizationId, "type" ~= WorkerType.analyzer, Some("all"), Nil) + // } + // + // private def listForUser(userId: String): (Source[Worker, NotUsed], Future[Long]) = { + // import org.elastic4play.services.QueryDSL._ + // findForUser(userId, any, Some("all"), Nil) + // } + // + // def listAnalyzerForUser(userId: String): (Source[Worker, NotUsed], Future[Long]) = { + // import org.elastic4play.services.QueryDSL._ + // findForUser(userId, "type" ~= WorkerType.analyzer, Some("all"), Nil) + // } + // + // def findForUser(userId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Worker, NotUsed], Future[Long]) = { + // val workers = for { + // user ← userSrv.get(userId) + // organizationId = user.organization() + // } yield findForOrganization(organizationId, queryDef, range, sortBy) + // val analyserSource = Source.fromFutureSource(workers.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + // val analyserTotal = workers.flatMap(_._2) + // analyserSource -> analyserTotal + // } + + def findAnalyzersForUser(userId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Worker, NotUsed], Future[Long]) = { + import org.elastic4play.services.QueryDSL._ + val analyzers = for { + user ← userSrv.get(userId) + organizationId = user.organization() + } yield findForOrganization(organizationId, and(queryDef, "type" ~= WorkerType.analyzer), range, sortBy) + val analyserSource = Source.fromFutureSource(analyzers.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val analyserTotal = analyzers.flatMap(_._2) + analyserSource -> analyserTotal + } + + def findRespondersForUser(userId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Worker, NotUsed], Future[Long]) = { + import org.elastic4play.services.QueryDSL._ + val analyzers = for { + user ← userSrv.get(userId) + organizationId = user.organization() + } yield findForOrganization(organizationId, and(queryDef, "type" ~= WorkerType.responder), range, sortBy) + val analyserSource = Source.fromFutureSource(analyzers.map(_._1)).mapMaterializedValue(_ ⇒ NotUsed) + val analyserTotal = analyzers.flatMap(_._2) + analyserSource -> analyserTotal + } + + private def findForOrganization(organizationId: String, queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Worker, NotUsed], Future[Long]) = { + import org.elastic4play.services.QueryDSL._ + find(and(withParent("organization", organizationId), queryDef), range, sortBy) + } + + private def find(queryDef: QueryDef, range: Option[String], sortBy: Seq[String]): (Source[Worker, NotUsed], Future[Long]) = { + findSrv[WorkerModel, Worker](workerModel, queryDef, range, sortBy) + } + + def rescan(): Unit = { + scan(analyzersPaths.map(_ -> WorkerType.analyzer) ++ + workersPaths.map(_ -> WorkerType.responder)) + } + + def scan(analyzerPaths: Seq[(Path, WorkerType.Type)]): Unit = { + val analyzers = (for { + (analyzerPath, analyzerType) ← analyzerPaths + analyzerDir ← Try(Files.newDirectoryStream(analyzerPath).asScala).getOrElse { + logger.warn(s"Analyzer directory ($analyzerPath) is not found") + Nil + } + if Files.isDirectory(analyzerDir) + infoFile ← Files.newDirectoryStream(analyzerDir, "*.json").asScala + analyzerDefinition ← WorkerDefinition.fromPath(infoFile, analyzerType).fold( + error ⇒ { + logger.warn("Analyzer definition file read error", error) + Nil + }, + ad ⇒ Seq(ad)) + } yield analyzerDefinition.id -> analyzerDefinition) + .toMap + + workerMapLock.synchronized { + workerMap = analyzers + } + logger.info(s"New analyzer list:\n\n\t${workerMap.values.map(a ⇒ s"${a.name} ${a.version}").mkString("\n\t")}\n") + } + + def create(organization: Organization, workerDefinition: WorkerDefinition, workerFields: Fields)(implicit authContext: AuthContext): Future[Worker] = { + val rawConfig = workerFields.getValue("configuration").fold(JsObject.empty)(_.as[JsObject]) + val configItems = workerDefinition.configurationItems ++ BaseConfig.global.items ++ BaseConfig.tlp.items ++ BaseConfig.pap.items + val configOrErrors = configItems + .validatedBy(_.read(rawConfig)) + .map(JsObject.apply) + + val unknownConfigItems = (rawConfig.value.keySet -- configItems.map(_.name)) + .foldLeft[Unit Or Every[AttributeError]](Good(())) { + case (Good(_), ci) ⇒ Bad(One(UnknownAttributeError("worker.config", JsString(ci)))) + case (Bad(e), ci) ⇒ Bad(UnknownAttributeError("worker.config", JsString(ci)) +: e) + } + + withGood(configOrErrors, unknownConfigItems)((c, _) ⇒ c) + .fold(cfg ⇒ { + createSrv[WorkerModel, Worker, Organization](workerModel, organization, workerFields + .set("workerDefinitionId", workerDefinition.id) + .set("description", workerDefinition.description) + .set("configuration", cfg.toString) + .set("type", workerDefinition.tpe.toString) + .addIfAbsent("dataTypeList", StringInputValue(workerDefinition.dataTypeList))) + + }, { + case One(e) ⇒ Future.failed(e) + case Every(es @ _*) ⇒ Future.failed(AttributeCheckingError(s"analyzer(${workerDefinition.name}).configuration", es)) + }) + } + + def create(organizationId: String, workerDefinition: WorkerDefinition, workerFields: Fields)(implicit authContext: AuthContext): Future[Worker] = { + for { + organization ← organizationSrv.get(organizationId) + analyzer ← create(organization, workerDefinition, workerFields) + } yield analyzer + } + + def delete(analyzer: Worker)(implicit authContext: AuthContext): Future[Unit] = + deleteSrv.realDelete(analyzer) + + def delete(analyzerId: String)(implicit authContext: AuthContext): Future[Unit] = + deleteSrv.realDelete[WorkerModel, Worker](workerModel, analyzerId) + + def update(analyzer: Worker, fields: Fields)(implicit authContext: AuthContext): Future[Worker] = update(analyzer, fields, ModifyConfig.default) + + def update(analyzer: Worker, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Worker] = { + val analyzerFields = fields.getValue("configuration").fold(fields)(cfg ⇒ fields.set("configuration", cfg.toString)) + updateSrv(analyzer, analyzerFields, modifyConfig) + } + + def update(analyzerId: String, fields: Fields)(implicit authContext: AuthContext): Future[Worker] = update(analyzerId, fields, ModifyConfig.default) + + def update(analyzerId: String, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Worker] = { + get(analyzerId).flatMap(analyzer ⇒ update(analyzer, fields, modifyConfig)) + } +} \ No newline at end of file diff --git a/conf/reference.conf b/conf/reference.conf index 9f91ef006..7e65ca6f2 100644 --- a/conf/reference.conf +++ b/conf/reference.conf @@ -101,3 +101,18 @@ analyzer { } } + +responder { + # Directory that holds responders + path = [] + + fork-join-executor { + # Min number of threads available for analyze + parallelism-min = 2 + # Parallelism (threads) ... ceil(available processors * factor) + parallelism-factor = 2.0 + # Max number of threads available for analyze + parallelism-max = 4 + } +} + diff --git a/conf/routes b/conf/routes index 1d83a12b1..2865b85f6 100644 --- a/conf/routes +++ b/conf/routes @@ -16,6 +16,13 @@ POST /api/analyzer/_search org.thp.cort GET /api/analyzer/:id org.thp.cortex.controllers.AnalyzerCtrl.get(id) GET /api/analyzer/type/:dataType org.thp.cortex.controllers.AnalyzerCtrl.listForType(dataType) POST /api/analyzer/:id/run org.thp.cortex.controllers.JobCtrl.create(id) + +GET /api/responder org.thp.cortex.controllers.ResponderCtrl.find +POST /api/responder/_search org.thp.cortex.controllers.ResponderCtrl.find +GET /api/responder/:id org.thp.cortex.controllers.ResponderCtrl.get(id) +GET /api/responder/type/:dataType org.thp.cortex.controllers.ResponderCtrl.listForType(dataType) +POST /api/responder/:id/run org.thp.cortex.controllers.JobCtrl.create(id) + GET /api/job/:id/waitreport org.thp.cortex.controllers.JobCtrl.waitReport(id, atMost ?= "1minute") #################### # API used by MISP @@ -23,23 +30,34 @@ GET /modules org.thp.cort POST /query org.thp.cortex.controllers.MispCtrl.query #################### +GET /api/job/:id/artifacts org.thp.cortex.controllers.JobCtrl.listArtifacts(id) + DELETE /api/analyzer/:id org.thp.cortex.controllers.AnalyzerCtrl.delete(id) PATCH /api/analyzer/:id org.thp.cortex.controllers.AnalyzerCtrl.update(id) -GET /api/job/:id/artifacts org.thp.cortex.controllers.JobCtrl.listArtifacts(id) GET /api/analyzerdefinition org.thp.cortex.controllers.AnalyzerCtrl.listDefinitions POST /api/analyzerdefinition/scan org.thp.cortex.controllers.AnalyzerCtrl.scan -#POST /api/organization/analyzer/:analyzerId org.thp.cortex.controllers.AnalyzerCtrl.create(analyzerId) GET /api/organization/analyzer org.thp.cortex.controllers.AnalyzerCtrl.find GET /api/organization/analyzer/_search org.thp.cortex.controllers.AnalyzerCtrl.find POST /api/organization/analyzer/:analyzerId org.thp.cortex.controllers.AnalyzerCtrl.create(analyzerId) -#GET /api/organization/:organizationId/analyzer org.thp.cortex.controllers.AnalyzerCtrl.findForOrganization(organizationId) -#POST /api/organization/:organizationId/analyzer/_search org.thp.cortex.controllers.AnalyzerCtrl.findForOrganization(organizationId) + +DELETE /api/responder/:id org.thp.cortex.controllers.ResponderCtrl.delete(id) +PATCH /api/responder/:id org.thp.cortex.controllers.ResponderCtrl.update(id) +GET /api/responderdefinition org.thp.cortex.controllers.ResponderCtrl.listDefinitions +POST /api/responderdefinition/scan org.thp.cortex.controllers.ResponderCtrl.scan +GET /api/organization/responder org.thp.cortex.controllers.ResponderCtrl.find +GET /api/organization/responder/_search org.thp.cortex.controllers.ResponderCtrl.find +POST /api/organization/responder/:responderId org.thp.cortex.controllers.ResponderCtrl.create(responderId) GET /api/analyzerconfig/:analyzerConfigName org.thp.cortex.controllers.AnalyzerConfigCtrl.get(analyzerConfigName) GET /api/analyzerconfig org.thp.cortex.controllers.AnalyzerConfigCtrl.list PATCH /api/analyzerconfig/:analyzerConfigName org.thp.cortex.controllers.AnalyzerConfigCtrl.update(analyzerConfigName) +GET /api/responderconfig/:responderConfigName org.thp.cortex.controllers.ResponderConfigCtrl.get(responderConfigName) +GET /api/responderconfig org.thp.cortex.controllers.ResponderConfigCtrl.list +PATCH /api/responderconfig/:responderConfigName org.thp.cortex.controllers.ResponderConfigCtrl.update(responderConfigName) + + GET /api/job org.thp.cortex.controllers.JobCtrl.list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], range: Option[String]) POST /api/job/_search org.thp.cortex.controllers.JobCtrl.find DELETE /api/job/:id org.thp.cortex.controllers.JobCtrl.delete(id)