diff --git a/CHANGELOG.md b/CHANGELOG.md index cbce21785e..382d661da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,34 @@ # Change Log -## [2.12.1](https://github.com/CERT-BDF/TheHive/tree/2.12.1) (2017-08-01) +## [2.13](https://github.com/CERT-BDF/TheHive/tree/2.13) (2017-09-15) + +[Full Changelog](https://github.com/CERT-BDF/TheHive/compare/2.12.1...2.13) + +**Implemented enhancements:** + +- Group ownership in Docker image prevents running on OpenShift [\#307](https://github.com/CERT-BDF/TheHive/issues/307) +- Improve the content of alert flow items [\#304](https://github.com/CERT-BDF/TheHive/issues/304) +- Add a basic support for webhooks [\#293](https://github.com/CERT-BDF/TheHive/issues/293) +- Add basic authentication to Stream API [\#291](https://github.com/CERT-BDF/TheHive/issues/291) +- Add Support for Play 2.6.x and Elasticsearch 5.x [\#275](https://github.com/CERT-BDF/TheHive/issues/275) +- Fine grained user permissions for API access [\#263](https://github.com/CERT-BDF/TheHive/issues/263) +- Alert Pane: Catch Incorrect Keywords [\#241](https://github.com/CERT-BDF/TheHive/issues/241) +- Specify multiple AD servers in TheHive configuration [\#231](https://github.com/CERT-BDF/TheHive/issues/231) +- Export cases in MISP events [\#52](https://github.com/CERT-BDF/TheHive/issues/52) + +**Fixed bugs:** +- Download attachment with non-latin filename [\#302](https://github.com/CERT-BDF/TheHive/issues/302) +- Undefined threat level from MISP events becomes severity "4" [\#300](https://github.com/CERT-BDF/TheHive/issues/300) +- File name is not displayed in observable conflict dialog [\#295](https://github.com/CERT-BDF/TheHive/issues/295) +- A colon punctuation mark in a search query results in 500 [\#285](https://github.com/CERT-BDF/TheHive/issues/285) +- Previewing alerts fails with "too many substreams open" due to case similarity process [\#280](https://github.com/CERT-BDF/TheHive/issues/280) + +**Closed issues:** + +- Threat level/severity code inverted between The Hive and MISP [\#292](https://github.com/CERT-BDF/TheHive/issues/292) + +## [2.12.1](https://github.com/CERT-BDF/TheHive/tree/2.12.1) (2017-08-01) [Full Changelog](https://github.com/CERT-BDF/TheHive/compare/2.12.0...2.12.1) **Implemented enhancements:** @@ -11,12 +38,12 @@ **Fixed bugs:** +- Cortex Connector Not Found [\#256](https://github.com/CERT-BDF/TheHive/issues/256) - Case similarity reports merged cases [\#272](https://github.com/CERT-BDF/TheHive/issues/272) - Closing a case with an open task does not dismiss task in "My tasks" [\#269](https://github.com/CERT-BDF/TheHive/issues/269) - API: cannot create alert if one alert artifact contains the IOC field set [\#268](https://github.com/CERT-BDF/TheHive/issues/268) - Can't get logs of a task via API [\#259](https://github.com/CERT-BDF/TheHive/issues/259) - Add multiple attachments in a single task log doesn't work [\#257](https://github.com/CERT-BDF/TheHive/issues/257) -- Cortex Connector Not Found [\#256](https://github.com/CERT-BDF/TheHive/issues/256) - TheHive doesn't send the file name to Cortex [\#254](https://github.com/CERT-BDF/TheHive/issues/254) - Renaming of users does not work [\#249](https://github.com/CERT-BDF/TheHive/issues/249) diff --git a/build.sbt b/build.sbt index 96d2bfee54..1d05af0bde 100644 --- a/build.sbt +++ b/build.sbt @@ -1,17 +1,21 @@ name := "TheHive" lazy val thehiveBackend = (project in file("thehive-backend")) + .enablePlugins(PlayScala) .settings(publish := {}) lazy val thehiveMetrics = (project in file("thehive-metrics")) + .enablePlugins(PlayScala) .dependsOn(thehiveBackend) .settings(publish := {}) lazy val thehiveMisp = (project in file("thehive-misp")) + .enablePlugins(PlayScala) .dependsOn(thehiveBackend) .settings(publish := {}) lazy val thehiveCortex = (project in file("thehive-cortex")) + .enablePlugins(PlayScala) .dependsOn(thehiveBackend) .settings(publish := {}) .settings(SbtScalariform.scalariformSettings: _*) @@ -26,6 +30,11 @@ lazy val thehive = (project in file(".")) .settings(PublishToBinTray.settings: _*) .settings(Release.settings: _*) + +// Redirect logs from ElasticSearch (which uses log4j2) to slf4j +libraryDependencies += "org.apache.logging.log4j" % "log4j-to-slf4j" % "2.9.0" +excludeDependencies += "org.apache.logging.log4j" % "log4j-core" + lazy val rpmPackageRelease = (project in file("package/rpm-release")) .enablePlugins(RpmPlugin) .settings( @@ -33,7 +42,7 @@ lazy val rpmPackageRelease = (project in file("package/rpm-release")) maintainer := "TheHive Project ", version := "1.0.0", rpmRelease := "3", - rpmVendor in Rpm := "TheHive Project", + rpmVendor := "TheHive Project", rpmUrl := Some("http://thehive-project.org/"), rpmLicense := Some("AGPL"), maintainerScripts in Rpm := Map.empty, @@ -125,7 +134,7 @@ linuxMakeStartScript in Debian := None // RPM // rpmRelease := "1" -rpmVendor in Rpm := "TheHive Project" +rpmVendor := "TheHive Project" rpmUrl := Some("http://thehive-project.org/") rpmLicense := Some("AGPL") rpmRequirements += "java-1.8.0-openjdk-headless" @@ -163,12 +172,18 @@ mappings in Docker ~= (_.filterNot { case (_, filepath) => filepath == "/opt/thehive/conf/application.conf" }) dockerCommands ~= { dc => - val (dockerInitCmds, dockerTailCmds) = dc.splitAt(4) + val (dockerInitCmds, dockerTailCmds) = dc + .collect { + case ExecCmd("RUN", "chown", _*) => ExecCmd("RUN", "chown", "-R", "daemon:root", ".") + case other => other + } + .splitAt(4) dockerInitCmds ++ Seq( - Cmd("ADD", "var", "/var"), + Cmd("ADD", "var", "/var"), Cmd("ADD", "etc", "/etc"), - ExecCmd("RUN", "chown", "-R", "daemon:daemon", "/var/log/thehive")) ++ + ExecCmd("RUN", "chown", "-R", "daemon:root", "/var/log/thehive"), + ExecCmd("RUN", "chmod", "+x", "/opt/thehive/bin/thehive", "/opt/thehive/entrypoint")) ++ dockerTailCmds } diff --git a/package/docker/entrypoint b/package/docker/entrypoint index 07db120a82..8f3ca03306 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -59,7 +59,7 @@ then SECRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1) fi echo Using secret: $SECRET - echo play.crypto.secret=\"$SECRET\" >> $CONFIG_FILE + echo play.http.secret.key=\"$SECRET\" >> $CONFIG_FILE fi if test $CONFIG_ES = 1 diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 5fc2ab1da7..7629188722 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -13,7 +13,7 @@ object BasicSettings extends AutoPlugin { "-deprecation", // Emit warning and location for usages of deprecated APIs. "-feature", // Emit warning and location for usages of features that should be imported explicitly. "-unchecked", // Enable additional warnings where generated code depends on assumptions. - "-Xfatal-warnings", // Fail the compilation if there are any warnings. + //"-Xfatal-warnings", // Fail the compilation if there are any warnings. "-Xlint", // Enable recommended additional warnings. "-Ywarn-adapted-args", // Warn if an argument list is modified to match the receiver. "-Ywarn-dead-code", // Warn when dead code is identified. diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 129c4e93f0..92cf4717af 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,17 +1,19 @@ import sbt._ object Dependencies { - val scalaVersion = "2.11.8" + val scalaVersion = "2.12.3" object Library { object Play { val version = play.core.PlayVersion.current val ws = "com.typesafe.play" %% "play-ws" % version - val cache = "com.typesafe.play" %% "play-cache" % version + val ahc = "com.typesafe.play" %% "play-ahc-ws" % version + val cache = "com.typesafe.play" %% "play-ehcache" % version val test = "com.typesafe.play" %% "play-test" % version val specs2 = "com.typesafe.play" %% "play-specs2" % version val filters = "com.typesafe.play" %% "filters-helpers" % version + val guice = "com.typesafe.play" %% "play-guice" % version object Specs2 { private val version = "3.6.6" val matcherExtra = "org.specs2" %% "specs2-matcher-extra" % version @@ -20,16 +22,16 @@ object Dependencies { } object Specs2 { - private val version = "3.6.6" + private val version = "3.9.4" val core = "org.specs2" %% "specs2-core" % version val matcherExtra = "org.specs2" %% "specs2-matcher-extra" % version val mock = "org.specs2" %% "specs2-mock" % version } - val scalaGuice = "net.codingwell" %% "scala-guice" % "4.0.1" - val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % "2.4.7" - val reflections = "org.reflections" % "reflections" % "0.9.10" + val scalaGuice = "net.codingwell" %% "scala-guice" % "4.1.0" + val akkaTestkit = "com.typesafe.akka" %% "akka-testkit" % "2.5.4" + val reflections = "org.reflections" % "reflections" % "0.9.11" val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2" - val akkaTest = "com.typesafe.akka" %% "akka-stream-testkit" % "2.4.4" - val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.2.1" + val akkaTest = "com.typesafe.akka" %% "akka-stream-testkit" % "2.5.4" + val elastic4play = "org.cert-bdf" %% "elastic4play" % "1.3.0" } } diff --git a/project/build.properties b/project/build.properties index 27e88aa115..c091b86ca4 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.13 +sbt.version=0.13.16 diff --git a/project/plugins.sbt b/project/plugins.sbt index bd976428ba..b0115921fe 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,7 @@ // Comment to get more information during initialization logLevel := Level.Info -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.14") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.3") addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0") diff --git a/thehive-backend/app/connectors/Connectors.scala b/thehive-backend/app/connectors/Connectors.scala index 929fdf1269..df5e0ff218 100644 --- a/thehive-backend/app/connectors/Connectors.scala +++ b/thehive-backend/app/connectors/Connectors.scala @@ -1,15 +1,16 @@ package connectors -import javax.inject.Inject +import javax.inject.{ Inject, Singleton } + +import scala.collection.immutable -import com.google.inject.AbstractModule -import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder } import play.api.libs.json.{ JsObject, Json } -import play.api.mvc.{ Action, Handler, RequestHeader, Results } +import play.api.mvc._ import play.api.routing.sird.UrlContext import play.api.routing.{ Router, SimpleRouter } -import scala.collection.immutable +import com.google.inject.AbstractModule +import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder } trait Connector { val name: String @@ -17,14 +18,17 @@ trait Connector { val status: JsObject = Json.obj("enabled" → true) } -class ConnectorRouter @Inject() (connectors: immutable.Set[Connector]) extends SimpleRouter { +@Singleton +class ConnectorRouter @Inject() ( + connectors: immutable.Set[Connector], + actionBuilder: DefaultActionBuilder) extends SimpleRouter { def get(connectorName: String): Option[Connector] = connectors.find(_.name == connectorName) def routes: PartialFunction[RequestHeader, Handler] = { case request @ p"/$connector/$path<.*>" ⇒ get(connector) .flatMap(_.router.withPrefix(s"/$connector/").handlerFor(request)) - .getOrElse(Action { _ ⇒ Results.NotFound(s"connector $connector not found") }) + .getOrElse(actionBuilder { _ ⇒ Results.NotFound(s"connector $connector not found") }) } } diff --git a/thehive-backend/app/controllers/AlertCtrl.scala b/thehive-backend/app/controllers/AlertCtrl.scala index f90dff0be8..322f90f3dd 100644 --- a/thehive-backend/app/controllers/AlertCtrl.scala +++ b/thehive-backend/app/controllers/AlertCtrl.scala @@ -2,21 +2,24 @@ package controllers import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Try + +import play.api.Logger +import play.api.http.Status +import play.api.libs.json.{ JsArray, JsObject, Json } +import play.api.mvc._ + import akka.stream.Materializer +import models.Roles +import services.JsonFormat.caseSimilarityWrites +import services.{ AlertSrv, CaseSrv } + import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites import org.elastic4play.services.JsonFormat.{ aggReads, queryReads } import org.elastic4play.services._ import org.elastic4play.{ BadRequestError, Timed } -import play.api.Logger -import play.api.http.Status -import play.api.libs.json.{ JsArray, JsObject, Json } -import play.api.mvc.{ Action, AnyContent, Controller } -import services.{ AlertSrv, CaseSrv } -import services.JsonFormat.caseSimilarityWrites - -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.Try @Singleton class AlertCtrl @Inject() ( @@ -25,20 +28,25 @@ class AlertCtrl @Inject() ( auxSrv: AuxSrv, authenticated: Authenticated, renderer: Renderer, + components: ControllerComponents, fieldsBodyParser: FieldsBodyParser, implicit val ec: ExecutionContext, - implicit val mat: Materializer) extends Controller with Status { + implicit val mat: Materializer) extends AbstractController(components) with Status { - val log = Logger(getClass) + private[AlertCtrl] lazy val logger = Logger(getClass) @Timed - def create(): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ - alertSrv.create(request.body) + def create(): Action[Fields] = authenticated(Roles.alert).async(fieldsBodyParser) { implicit request ⇒ + alertSrv.create(request.body + .unset("lastSyncDate") + .unset("case") + .unset("status") + .unset("follow")) .map(alert ⇒ renderer.toOutput(CREATED, alert)) } @Timed - def mergeWithCase(alertId: String, caseId: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def mergeWithCase(alertId: String, caseId: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ for { alert ← alertSrv.get(alertId) caze ← caseSrv.get(caseId) @@ -47,7 +55,7 @@ class AlertCtrl @Inject() ( } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ val withStats = request .queryString .get("nstats") @@ -73,26 +81,26 @@ class AlertCtrl @Inject() ( } @Timed - def update(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ alertSrv.update(id, request.body) .map { alert ⇒ renderer.toOutput(OK, alert) } } @Timed - def bulkUpdate(): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def bulkUpdate(): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ request.body.getStrings("ids").fold(Future.successful(Ok(JsArray()))) { ids ⇒ alertSrv.bulkUpdate(ids, request.body.unset("ids")).map(multiResult ⇒ renderer.toMultiOutput(OK, multiResult)) } } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ alertSrv.delete(id) .map(_ ⇒ NoContent) } @Timed - def find(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -105,7 +113,7 @@ class AlertCtrl @Inject() ( } @Timed - def stats(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def stats(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query") .fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val aggs = request.body.getValue("stats") @@ -114,7 +122,7 @@ class AlertCtrl @Inject() ( } @Timed - def markAsRead(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def markAsRead(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ for { alert ← alertSrv.get(id) updatedAlert ← alertSrv.markAsRead(alert) @@ -122,7 +130,7 @@ class AlertCtrl @Inject() ( } @Timed - def markAsUnread(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def markAsUnread(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ for { alert ← alertSrv.get(id) updatedAlert ← alertSrv.markAsUnread(alert) @@ -130,7 +138,7 @@ class AlertCtrl @Inject() ( } @Timed - def createCase(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def createCase(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ for { alert ← alertSrv.get(id) customCaseTemplate = request.body.getString("caseTemplate") @@ -139,19 +147,19 @@ class AlertCtrl @Inject() ( } @Timed - def followAlert(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def followAlert(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ alertSrv.setFollowAlert(id, follow = true) .map { alert ⇒ renderer.toOutput(OK, alert) } } @Timed - def unfollowAlert(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def unfollowAlert(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ alertSrv.setFollowAlert(id, follow = false) .map { alert ⇒ renderer.toOutput(OK, alert) } } @Timed - def fixStatus(): Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ + def fixStatus(): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ alertSrv.fixStatus() .map(_ ⇒ NoContent) } diff --git a/thehive-backend/app/controllers/ArtifactCtrl.scala b/thehive-backend/app/controllers/ArtifactCtrl.scala index f6ac842c85..7d6908a84a 100644 --- a/thehive-backend/app/controllers/ArtifactCtrl.scala +++ b/thehive-backend/app/controllers/ArtifactCtrl.scala @@ -3,16 +3,19 @@ package controllers import javax.inject.{ Inject, Singleton } import scala.concurrent.{ ExecutionContext, Future } + import play.api.http.Status import play.api.libs.json.JsArray -import play.api.mvc.{ Action, AnyContent, Controller } -import org.elastic4play.{ BadRequestError, Timed } +import play.api.mvc._ + +import models.Roles +import services.ArtifactSrv + import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites -import org.elastic4play.services.{ Agg, AuxSrv } -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } import org.elastic4play.services.JsonFormat.{ aggReads, queryReads } -import services.ArtifactSrv +import org.elastic4play.services._ +import org.elastic4play.{ BadRequestError, Timed } @Singleton class ArtifactCtrl @Inject() ( @@ -20,11 +23,12 @@ class ArtifactCtrl @Inject() ( auxSrv: AuxSrv, authenticated: Authenticated, renderer: Renderer, + components: ControllerComponents, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller with Status { + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { @Timed - def create(caseId: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def create(caseId: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ val fields = request.body val data = fields.getStrings("data") .getOrElse(fields.getString("data").toSeq) @@ -47,32 +51,32 @@ class ArtifactCtrl @Inject() ( } @Timed - def get(id: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def get(id: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ artifactSrv.get(id) .map(artifact ⇒ renderer.toOutput(OK, artifact)) } @Timed - def update(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ artifactSrv.update(id, request.body) .map(artifact ⇒ renderer.toOutput(OK, artifact)) } @Timed - def bulkUpdate(): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def bulkUpdate(): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ request.body.getStrings("ids").fold(Future.successful(Ok(JsArray()))) { ids ⇒ artifactSrv.bulkUpdate(ids, request.body.unset("ids")).map(multiResult ⇒ renderer.toMultiOutput(OK, multiResult)) } } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ artifactSrv.delete(id) .map(_ ⇒ NoContent) } @Timed - def findInCase(caseId: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def findInCase(caseId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ import org.elastic4play.services.QueryDSL._ val childQuery = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val query = and(childQuery, "_parent" ~= caseId) @@ -84,7 +88,7 @@ class ArtifactCtrl @Inject() ( } @Timed - def find(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -97,7 +101,7 @@ class ArtifactCtrl @Inject() ( } @Timed - def findSimilar(artifactId: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def findSimilar(artifactId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ artifactSrv.get(artifactId).flatMap { artifact ⇒ val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -109,7 +113,7 @@ class ArtifactCtrl @Inject() ( } @Timed - def stats(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def stats(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val aggs = request.body.getValue("stats").getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]] artifactSrv.stats(query, aggs).map(s ⇒ Ok(s)) diff --git a/thehive-backend/app/controllers/Asset.scala b/thehive-backend/app/controllers/Asset.scala index 5ca4ddcd35..f2bfb27616 100644 --- a/thehive-backend/app/controllers/Asset.scala +++ b/thehive-backend/app/controllers/Asset.scala @@ -2,21 +2,23 @@ package controllers import javax.inject.{ Inject, Singleton } +import scala.concurrent.ExecutionContext + import play.api.Environment -import play.api.http.HttpErrorHandler -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.http.{ FileMimeTypes, HttpErrorHandler } +import play.api.mvc.{ Action, AnyContent } trait AssetCtrl { def get(file: String): Action[AnyContent] } @Singleton -class AssetCtrlProd @Inject() (errorHandler: HttpErrorHandler) extends Assets(errorHandler) with AssetCtrl { +class AssetCtrlProd @Inject() (errorHandler: HttpErrorHandler, meta: AssetsMetadata) extends Assets(errorHandler, meta) with AssetCtrl { def get(file: String): Action[AnyContent] = at("/ui", file) } @Singleton -class AssetCtrlDev @Inject() (environment: Environment) extends ExternalAssets(environment) with AssetCtrl { +class AssetCtrlDev @Inject() (environment: Environment)(implicit ec: ExecutionContext, fileMimeTypes: FileMimeTypes) extends ExternalAssets(environment) with AssetCtrl { def get(file: String): Action[AnyContent] = { if (file.startsWith("bower_components/")) { at("ui", file) diff --git a/thehive-backend/app/controllers/AttachmentCtrl.scala b/thehive-backend/app/controllers/AttachmentCtrl.scala index 7633d6680a..9d61cbf23e 100644 --- a/thehive-backend/app/controllers/AttachmentCtrl.scala +++ b/thehive-backend/app/controllers/AttachmentCtrl.scala @@ -1,21 +1,24 @@ package controllers +import java.net.URLEncoder +import java.nio.file.Files import javax.inject.{ Inject, Singleton } -import akka.stream.scaladsl.FileIO -import play.api.Configuration import play.api.http.HttpEntity -import play.api.libs.Files.TemporaryFile +import play.api.libs.Files.DefaultTemporaryFileCreator import play.api.mvc._ +import play.api.{ Configuration, mvc } + +import akka.stream.scaladsl.FileIO import net.lingala.zip4j.core.ZipFile import net.lingala.zip4j.model.ZipParameters import net.lingala.zip4j.util.Zip4jConstants +import models.Roles + import org.elastic4play.Timed -import org.elastic4play.services.{ AttachmentSrv, Role } -import org.elastic4play.models.AttachmentAttributeFormat +import org.elastic4play.controllers.{ Authenticated, Renderer } import org.elastic4play.models.AttachmentAttributeFormat -import org.elastic4play.controllers.Authenticated -import org.elastic4play.controllers.Renderer +import org.elastic4play.services.AttachmentSrv /** * Controller used to access stored attachments (plain or zipped) @@ -23,19 +26,25 @@ import org.elastic4play.controllers.Renderer @Singleton class AttachmentCtrl( password: String, + tempFileCreator: DefaultTemporaryFileCreator, attachmentSrv: AttachmentSrv, authenticated: Authenticated, - renderer: Renderer) extends Controller { + components: ControllerComponents, + renderer: Renderer) extends AbstractController(components) { @Inject() def this( configuration: Configuration, + tempFileCreator: DefaultTemporaryFileCreator, attachmentSrv: AttachmentSrv, authenticated: Authenticated, + components: ControllerComponents, renderer: Renderer) = this( - configuration.getString("datastore.attachment.password").get, + configuration.get[String]("datastore.attachment.password"), + tempFileCreator, attachmentSrv, authenticated, + components, renderer) /** @@ -44,17 +53,17 @@ class AttachmentCtrl( * open the document directly. It must be used only for safe file */ @Timed("controllers.AttachmentCtrl.download") - def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Role.read) { implicit request ⇒ + def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒ if (hash.startsWith("{{")) // angularjs hack NoContent - else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty()) - BadRequest("File name is invalid") + else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty) + mvc.Results.BadRequest("File name is invalid") else Result( header = ResponseHeader( 200, Map( - "Content-Disposition" → s"""attachment; filename="${name.getOrElse(hash)}"""", + "Content-Disposition" → s"""attachment; filename="${URLEncoder.encode(name.getOrElse(hash), "utf-8")}"""", "Content-Transfer-Encoding" → "binary")), body = HttpEntity.Streamed(attachmentSrv.source(hash), None, None)) } @@ -65,13 +74,13 @@ class AttachmentCtrl( * File name can be specified (zip extension is append) */ @Timed("controllers.AttachmentCtrl.downloadZip") - def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Role.read) { implicit request ⇒ - if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty()) + def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒ + if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty) BadRequest("File name is invalid") else { - val f = TemporaryFile("zip", hash).file - f.delete() - val zipFile = new ZipFile(f) + val f = tempFileCreator.create("zip", hash).path + Files.delete(f) + val zipFile = new ZipFile(f.toFile) val zipParams = new ZipParameters zipParams.setCompressionLevel(Zip4jConstants.DEFLATE_LEVEL_FASTEST) zipParams.setEncryptFiles(true) @@ -85,11 +94,11 @@ class AttachmentCtrl( header = ResponseHeader( 200, Map( - "Content-Disposition" → s"""attachment; filename="${name.getOrElse(hash)}.zip"""", + "Content-Disposition" → s"""attachment; filename="${URLEncoder.encode(name.getOrElse(hash), "utf-8")}.zip"""", "Content-Type" → "application/zip", "Content-Transfer-Encoding" → "binary", - "Content-Length" → f.length.toString)), - body = HttpEntity.Streamed(FileIO.fromPath(f.toPath), Some(f.length), Some("application/zip"))) + "Content-Length" → Files.size(f).toString)), + body = HttpEntity.Streamed(FileIO.fromPath(f), Some(Files.size(f)), Some("application/zip"))) } } } \ No newline at end of file diff --git a/thehive-backend/app/controllers/AuthenticationCtrl.scala b/thehive-backend/app/controllers/AuthenticationCtrl.scala index 7a0a360026..24c8261ac9 100644 --- a/thehive-backend/app/controllers/AuthenticationCtrl.scala +++ b/thehive-backend/app/controllers/AuthenticationCtrl.scala @@ -2,15 +2,17 @@ package controllers import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.mvc._ + import models.UserStatus -import org.elastic4play.{ AuthorizationError, Timed } +import services.UserSrv + import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.database.DBIndex import org.elastic4play.services.AuthSrv -import play.api.mvc.{ Action, Controller, Results } -import services.UserSrv - -import scala.concurrent.{ ExecutionContext, Future } +import org.elastic4play.{ AuthorizationError, Timed } @Singleton class AuthenticationCtrl @Inject() ( @@ -19,8 +21,9 @@ class AuthenticationCtrl @Inject() ( authenticated: Authenticated, dbIndex: DBIndex, renderer: Renderer, + components: ControllerComponents, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller { + implicit val ec: ExecutionContext) extends AbstractController(components) { @Timed def login: Action[Fields] = Action.async(fieldsBodyParser) { implicit request ⇒ diff --git a/thehive-backend/app/controllers/CaseCtrl.scala b/thehive-backend/app/controllers/CaseCtrl.scala index 818d3219be..5ba40b741c 100644 --- a/thehive-backend/app/controllers/CaseCtrl.scala +++ b/thehive-backend/app/controllers/CaseCtrl.scala @@ -2,22 +2,24 @@ package controllers import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Try + +import play.api.Logger +import play.api.http.Status +import play.api.libs.json.{ JsArray, JsObject, Json } +import play.api.mvc._ + import akka.stream.Materializer import akka.stream.scaladsl.Sink -import models.CaseStatus +import models.{ CaseStatus, Roles } +import services.{ CaseMergeSrv, CaseSrv, CaseTemplateSrv, TaskSrv } + import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites import org.elastic4play.services.JsonFormat.{ aggReads, queryReads } import org.elastic4play.services._ import org.elastic4play.{ BadRequestError, Timed } -import play.api.Logger -import play.api.http.Status -import play.api.libs.json.{ JsArray, JsObject, Json } -import play.api.mvc.{ Action, AnyContent, Controller } -import services.{ CaseMergeSrv, CaseSrv, CaseTemplateSrv, TaskSrv } - -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.Try @Singleton class CaseCtrl @Inject() ( @@ -28,14 +30,15 @@ class CaseCtrl @Inject() ( auxSrv: AuxSrv, authenticated: Authenticated, renderer: Renderer, + components: ControllerComponents, fieldsBodyParser: FieldsBodyParser, implicit val ec: ExecutionContext, - implicit val mat: Materializer) extends Controller with Status { + implicit val mat: Materializer) extends AbstractController(components) with Status { - val log = Logger(getClass) + private[CaseCtrl] lazy val logger = Logger(getClass) @Timed - def create(): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def create(): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ request.body .getString("template") .map { templateName ⇒ @@ -51,7 +54,7 @@ class CaseCtrl @Inject() ( } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ val withStats = for { statsValues ← request.queryString.get("nstats") firstValue ← statsValues.headOption @@ -64,18 +67,18 @@ class CaseCtrl @Inject() ( } @Timed - def update(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ val isCaseClosing = request.body.getString("status").contains(CaseStatus.Resolved.toString) for { // Closing the case, so lets close the open tasks caze ← caseSrv.update(id, request.body) - closedTasks ← if (isCaseClosing) taskSrv.closeTasksOfCase(id) else Future.successful(Nil) // FIXME log warning if closedTasks contains errors + _ ← if (isCaseClosing) taskSrv.closeTasksOfCase(id) else Future.successful(Nil) // FIXME log warning if closedTasks contains errors } yield renderer.toOutput(OK, caze) } @Timed - def bulkUpdate(): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def bulkUpdate(): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ val isCaseClosing = request.body.getString("status").contains(CaseStatus.Resolved.toString) request.body.getStrings("ids").fold(Future.successful(Ok(JsArray()))) { ids ⇒ @@ -85,13 +88,13 @@ class CaseCtrl @Inject() ( } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ caseSrv.delete(id) .map(_ ⇒ NoContent) } @Timed - def find(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -104,14 +107,14 @@ class CaseCtrl @Inject() ( } @Timed - def stats(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def stats(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val aggs = request.body.getValue("stats").getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]] caseSrv.stats(query, aggs).map(s ⇒ Ok(s)) } @Timed - def linkedCases(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def linkedCases(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ caseSrv.linkedCases(id) .runWith(Sink.seq) .map { cases ⇒ @@ -128,7 +131,7 @@ class CaseCtrl @Inject() ( } @Timed - def merge(caseId1: String, caseId2: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def merge(caseId1: String, caseId2: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ caseMergeSrv.merge(caseId1, caseId2).map { caze ⇒ renderer.toOutput(OK, caze) } diff --git a/thehive-backend/app/controllers/CaseTemplateCtrl.scala b/thehive-backend/app/controllers/CaseTemplateCtrl.scala index 35f40abc46..662e76d8fc 100644 --- a/thehive-backend/app/controllers/CaseTemplateCtrl.scala +++ b/thehive-backend/app/controllers/CaseTemplateCtrl.scala @@ -3,16 +3,18 @@ package controllers import javax.inject.{ Inject, Singleton } import scala.concurrent.ExecutionContext -import scala.reflect.runtime.universe + import play.api.http.Status -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc._ + +import models.Roles +import services.CaseTemplateSrv + import org.elastic4play.Timed import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } -import org.elastic4play.services.AuxSrv import org.elastic4play.services.JsonFormat.queryReads -import services.CaseTemplateSrv +import org.elastic4play.services.{ AuxSrv, QueryDSL, QueryDef } @Singleton class CaseTemplateCtrl @Inject() ( @@ -20,35 +22,36 @@ class CaseTemplateCtrl @Inject() ( auxSrv: AuxSrv, authenticated: Authenticated, renderer: Renderer, + components: ControllerComponents, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller with Status { + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { @Timed - def create: Action[Fields] = authenticated(Role.admin).async(fieldsBodyParser) { implicit request ⇒ + def create: Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ caseTemplateSrv.create(request.body) .map(caze ⇒ renderer.toOutput(CREATED, caze)) } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ caseTemplateSrv.get(id) .map(caze ⇒ renderer.toOutput(OK, caze)) } @Timed - def update(id: String): Action[Fields] = authenticated(Role.admin).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ caseTemplateSrv.update(id, request.body) .map(caze ⇒ renderer.toOutput(OK, caze)) } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ caseTemplateSrv.delete(id) .map(_ ⇒ NoContent) } @Timed - def find: Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) diff --git a/thehive-backend/app/controllers/DBListCtrl.scala b/thehive-backend/app/controllers/DBListCtrl.scala new file mode 100644 index 0000000000..870ce43f05 --- /dev/null +++ b/thehive-backend/app/controllers/DBListCtrl.scala @@ -0,0 +1,74 @@ +package org.elastic4play.controllers + +import javax.inject.{ Inject, Singleton } + +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.libs.json.{ JsValue, Json } +import play.api.mvc._ + +import models.Roles + +import org.elastic4play.services.DBLists +import org.elastic4play.{ MissingAttributeError, Timed } + +@Singleton +class DBListCtrl @Inject() ( + dblists: DBLists, + authenticated: Authenticated, + renderer: Renderer, + components: ControllerComponents, + fieldsBodyParser: FieldsBodyParser, + implicit val ec: ExecutionContext) extends AbstractController(components) { + + @Timed("controllers.DBListCtrl.list") + def list: Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ + dblists.listAll.map { listNames ⇒ + renderer.toOutput(OK, listNames) + } + } + + @Timed("controllers.DBListCtrl.listItems") + def listItems(listName: String): Action[AnyContent] = authenticated(Roles.read) { implicit request ⇒ + val (src, _) = dblists(listName).getItems[JsValue] + val items = src.map { case (id, value) ⇒ s""""$id":$value""" } + .intersperse("{", ",", "}") + Ok.chunked(items).as("application/json") + } + + @Timed("controllers.DBListCtrl.addItem") + def addItem(listName: String): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ + request.body.getValue("value").fold(Future.successful(NoContent)) { value ⇒ + dblists(listName).addItem(value).map { item ⇒ + renderer.toOutput(OK, item.id) + } + } + } + + @Timed("controllers.DBListCtrl.deleteItem") + def deleteItem(itemId: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ + dblists.deleteItem(itemId).map { _ ⇒ + NoContent + } + } + + @Timed("controllers.DBListCtrl.udpateItem") + def updateItem(itemId: String): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ + request.body.getValue("value") + .map { value ⇒ + for { + item ← dblists.getItem(itemId) + _ ← dblists.deleteItem(item) + newItem ← dblists(item.dblist).addItem(value) + } yield renderer.toOutput(OK, newItem.id) + } + .getOrElse(Future.failed(MissingAttributeError("value"))) + } + + @Timed("controllers.DBListCtrl.itemExists") + def itemExists(listName: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ + val itemKey = request.body.getString("key").getOrElse(throw MissingAttributeError("Parameter key is missing")) + val itemValue = request.body.getValue("value").getOrElse(throw MissingAttributeError("Parameter value is missing")) + dblists(listName).exists(itemKey, itemValue).map(r ⇒ Ok(Json.obj("found" → r))) + } +} \ No newline at end of file diff --git a/thehive-backend/app/controllers/FlowCtrl.scala b/thehive-backend/app/controllers/FlowCtrl.scala index 350c2e991a..315ce1f452 100644 --- a/thehive-backend/app/controllers/FlowCtrl.scala +++ b/thehive-backend/app/controllers/FlowCtrl.scala @@ -2,14 +2,17 @@ package controllers import javax.inject.{ Inject, Singleton } -import scala.annotation.implicitNotFound import scala.concurrent.ExecutionContext + import play.api.http.Status -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc._ + +import models.Roles +import services.FlowSrv + import org.elastic4play.Timed import org.elastic4play.controllers.{ Authenticated, Renderer } -import org.elastic4play.services.{ AuxSrv, Role } -import services.FlowSrv +import org.elastic4play.services.AuxSrv @Singleton class FlowCtrl @Inject() ( @@ -17,13 +20,14 @@ class FlowCtrl @Inject() ( auxSrv: AuxSrv, authenticated: Authenticated, renderer: Renderer, - implicit val ec: ExecutionContext) extends Controller with Status { + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { /** * Return audit logs. For each item, include ancestor entities */ @Timed - def flow(rootId: Option[String], count: Option[Int]): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def flow(rootId: Option[String], count: Option[Int]): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ val (audits, total) = flowSrv(rootId.filterNot(_ == "any"), count.getOrElse(10)) renderer.toOutput(OK, audits, total) } diff --git a/thehive-backend/app/controllers/LogCtrl.scala b/thehive-backend/app/controllers/LogCtrl.scala index 6dfffbd7cf..038224eaa5 100644 --- a/thehive-backend/app/controllers/LogCtrl.scala +++ b/thehive-backend/app/controllers/LogCtrl.scala @@ -2,16 +2,19 @@ package controllers import javax.inject.{ Inject, Singleton } -import org.elastic4play.Timed -import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } -import org.elastic4play.services.JsonFormat.queryReads -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } -import org.elastic4play.models.JsonFormat.baseModelEntityWrites +import scala.concurrent.ExecutionContext + import play.api.http.Status -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc._ + +import models.Roles import services.LogSrv -import scala.concurrent.ExecutionContext +import org.elastic4play.Timed +import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } +import org.elastic4play.models.JsonFormat.baseModelEntityWrites +import org.elastic4play.services.JsonFormat.queryReads +import org.elastic4play.services.{ QueryDSL, QueryDef } @Singleton class LogCtrl @Inject() ( @@ -19,34 +22,35 @@ class LogCtrl @Inject() ( authenticated: Authenticated, renderer: Renderer, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller with Status { + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { @Timed - def create(taskId: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def create(taskId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ logSrv.create(taskId, request.body) .map(log ⇒ renderer.toOutput(CREATED, log)) } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ logSrv.get(id) .map(log ⇒ renderer.toOutput(OK, log)) } @Timed - def update(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ logSrv.update(id, request.body) .map(log ⇒ renderer.toOutput(OK, log)) } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ logSrv.delete(id) .map(_ ⇒ Ok("")) } @Timed - def findInTask(taskId: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def findInTask(taskId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ import org.elastic4play.services.QueryDSL._ val childQuery = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val query = and(childQuery, parent("case_task", withId(taskId))) @@ -58,7 +62,7 @@ class LogCtrl @Inject() ( } @Timed - def find: Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) diff --git a/thehive-backend/app/controllers/SearchCtrl.scala b/thehive-backend/app/controllers/SearchCtrl.scala index dc84d7cb38..96322b23e6 100644 --- a/thehive-backend/app/controllers/SearchCtrl.scala +++ b/thehive-backend/app/controllers/SearchCtrl.scala @@ -3,13 +3,16 @@ package controllers import javax.inject.{ Inject, Singleton } import scala.concurrent.ExecutionContext + import play.api.http.Status -import play.api.mvc.{ Action, Controller } +import play.api.mvc.{ AbstractController, Action, ControllerComponents } + +import models.Roles + import org.elastic4play.Timed import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } -import org.elastic4play.services.{ AuxSrv, FindSrv } -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } import org.elastic4play.services.JsonFormat.queryReads +import org.elastic4play.services._ @Singleton class SearchCtrl @Inject() ( @@ -17,11 +20,12 @@ class SearchCtrl @Inject() ( auxSrv: AuxSrv, authenticated: Authenticated, renderer: Renderer, + components: ControllerComponents, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller with Status { + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { @Timed - def find(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ import org.elastic4play.services.QueryDSL._ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") diff --git a/thehive-backend/app/controllers/StatusCtrl.scala b/thehive-backend/app/controllers/StatusCtrl.scala index 6b2c6874b4..a5cbbc8ab8 100644 --- a/thehive-backend/app/controllers/StatusCtrl.scala +++ b/thehive-backend/app/controllers/StatusCtrl.scala @@ -5,27 +5,23 @@ import javax.inject.{ Inject, Singleton } import scala.collection.immutable import play.api.Configuration -import play.api.libs.json.Json +import play.api.libs.json.{ JsObject, JsString, Json } import play.api.libs.json.Json.toJsFieldJsValueWrapper -import play.api.mvc.{ Action, Controller } - -import org.elastic4play.Timed -import org.elasticsearch.Build +import play.api.mvc.{ AbstractController, ControllerComponents } import com.sksamuel.elastic4s.ElasticDsl - import connectors.Connector -import models.Case -import play.api.libs.json.JsObject -import org.elastic4play.services.auth.MultiAuthSrv -import play.api.libs.json.JsString + +import org.elastic4play.Timed import org.elastic4play.services.AuthSrv +import org.elastic4play.services.auth.MultiAuthSrv @Singleton class StatusCtrl @Inject() ( connectors: immutable.Set[Connector], configuration: Configuration, - authSrv: AuthSrv) extends Controller { + authSrv: AuthSrv, + components: ControllerComponents) extends AbstractController(components) { private[controllers] def getVersion(c: Class[_]) = Option(c.getPackage.getImplementationVersion).getOrElse("SNAPSHOT") @@ -35,12 +31,12 @@ class StatusCtrl @Inject() ( "versions" → Json.obj( "TheHive" → getVersion(classOf[models.Case]), "Elastic4Play" → getVersion(classOf[Timed]), - "Play" → getVersion(classOf[Controller]), + "Play" → getVersion(classOf[AbstractController]), "Elastic4s" → getVersion(classOf[ElasticDsl]), "ElasticSearch" → getVersion(classOf[org.elasticsearch.Build])), "connectors" → JsObject(connectors.map(c ⇒ c.name → c.status).toSeq), "config" → Json.obj( - "protectDownloadsWith" → configuration.getString("datastore.attachment.password").get, + "protectDownloadsWith" → configuration.get[String]("datastore.attachment.password"), "authType" → (authSrv match { case multiAuthSrv: MultiAuthSrv ⇒ multiAuthSrv.authProviders.map { a ⇒ JsString(a.name) } case _ ⇒ JsString(authSrv.name) diff --git a/thehive-backend/app/controllers/StreamCtrl.scala b/thehive-backend/app/controllers/StreamCtrl.scala index aaa6b06ca1..28a19c7695 100644 --- a/thehive-backend/app/controllers/StreamCtrl.scala +++ b/thehive-backend/app/controllers/StreamCtrl.scala @@ -2,29 +2,27 @@ package controllers import javax.inject.{ Inject, Singleton } -import scala.annotation.implicitNotFound -import scala.concurrent.ExecutionContext +import scala.collection.immutable +import scala.concurrent.{ ExecutionContext, Future } import scala.concurrent.duration.{ DurationLong, FiniteDuration } -import scala.reflect.runtime.universe import scala.util.Random -import akka.actor.{ ActorSystem, Props } -import akka.pattern.ask -import akka.util.Timeout -import play.api.{ Configuration, Logger } + import play.api.http.Status import play.api.libs.json.Json import play.api.libs.json.Json.toJsFieldJsValueWrapper -import play.api.mvc.{ Action, AnyContent, Controller } -import org.elastic4play.{ AuthenticationError, Timed } -import org.elastic4play.controllers.{ Authenticated, ExpirationError, ExpirationOk, ExpirationWarning, Renderer } -import org.elastic4play.services.{ AuxSrv, EventSrv, Role } +import play.api.mvc._ +import play.api.{ Configuration, Logger } + +import akka.actor.{ ActorSystem, Props } +import akka.pattern.ask +import akka.util.Timeout +import models.Roles import services.StreamActor import services.StreamActor.StreamMessages -import akka.actor.ActorPath -import org.elastic4play.services.MigrationSrv -import scala.collection.immutable -import scala.concurrent.Future +import org.elastic4play.controllers._ +import org.elastic4play.services.{ AuxSrv, EventSrv, MigrationSrv } +import org.elastic4play.Timed @Singleton class StreamCtrl( @@ -37,8 +35,9 @@ class StreamCtrl( eventSrv: EventSrv, auxSrv: AuxSrv, migrationSrv: MigrationSrv, + components: ControllerComponents, implicit val system: ActorSystem, - implicit val ec: ExecutionContext) extends Controller with Status { + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { @Inject() def this( configuration: Configuration, @@ -47,29 +46,31 @@ class StreamCtrl( eventSrv: EventSrv, auxSrv: AuxSrv, migrationSrv: MigrationSrv, + components: ControllerComponents, system: ActorSystem, ec: ExecutionContext) = this( - configuration.getMilliseconds("stream.longpolling.cache").get.millis, - configuration.getMilliseconds("stream.longpolling.refresh").get.millis, - configuration.getMilliseconds("stream.longpolling.nextItemMaxWait").get.millis, - configuration.getMilliseconds("stream.longpolling.globalMaxWait").get.millis, + configuration.getMillis("stream.longpolling.cache").millis, + configuration.getMillis("stream.longpolling.refresh").millis, + configuration.getMillis("stream.longpolling.nextItemMaxWait").millis, + configuration.getMillis("stream.longpolling.globalMaxWait").millis, authenticated, renderer, eventSrv, auxSrv, migrationSrv, + components, system, ec) - val log = Logger(getClass) + private[StreamCtrl] lazy val logger = Logger(getClass) /** * Create a new stream entry with the event head */ @Timed("controllers.StreamCtrl.create") - def create: Action[AnyContent] = authenticated(Role.read) { + def create: Action[AnyContent] = authenticated(Roles.read) { val id = generateStreamId() - val aref = system.actorOf(Props( + system.actorOf(Props( classOf[StreamActor], cacheExpiration, refresh, @@ -80,7 +81,7 @@ class StreamCtrl( Ok(id) } - val alphanumeric: immutable.IndexedSeq[Char] = (('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')) + val alphanumeric: immutable.IndexedSeq[Char] = ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9') private[controllers] def generateStreamId() = Seq.fill(10)(alphanumeric(Random.nextInt(alphanumeric.size))).mkString private[controllers] def isValidStreamId(streamId: String): Boolean = { streamId.length == 10 && streamId.forall(alphanumeric.contains) @@ -92,22 +93,23 @@ class StreamCtrl( */ @Timed("controllers.StreamCtrl.get") def get(id: String): Action[AnyContent] = Action.async { implicit request ⇒ - implicit val timeout = Timeout(refresh + globalMaxWait + 1.second) + implicit val timeout: Timeout = Timeout(refresh + globalMaxWait + 1.second) if (!isValidStreamId(id)) { Future.successful(BadRequest("Invalid stream id")) } else { - val status = authenticated.expirationStatus(request) match { - case ExpirationError if !migrationSrv.isMigrating ⇒ throw AuthenticationError("Not authenticated") - case _: ExpirationWarning ⇒ 220 - case _ ⇒ OK - + val futureStatus = authenticated.expirationStatus(request) match { + case ExpirationError if !migrationSrv.isMigrating ⇒ authenticated.getFromApiKey(request).map(_ ⇒ OK) + case _: ExpirationWarning ⇒ Future.successful(220) + case _ ⇒ Future.successful(OK) } - (system.actorSelection(s"/user/stream-$id") ? StreamActor.GetOperations) map { - case StreamMessages(operations) ⇒ renderer.toOutput(status, operations) - case m ⇒ InternalServerError(s"Unexpected message : $m (${m.getClass})") + futureStatus.flatMap { status ⇒ + (system.actorSelection(s"/user/stream-$id") ? StreamActor.GetOperations) map { + case StreamMessages(operations) ⇒ renderer.toOutput(status, operations) + case m ⇒ InternalServerError(s"Unexpected message : $m (${m.getClass})") + } } } } diff --git a/thehive-backend/app/controllers/TaskCtrl.scala b/thehive-backend/app/controllers/TaskCtrl.scala index 6f00e4beb9..ddadd22ffe 100644 --- a/thehive-backend/app/controllers/TaskCtrl.scala +++ b/thehive-backend/app/controllers/TaskCtrl.scala @@ -3,16 +3,18 @@ package controllers import javax.inject.{ Inject, Singleton } import scala.concurrent.ExecutionContext -import scala.reflect.runtime.universe + import play.api.http.Status -import play.api.mvc.{ Action, AnyContent, Controller } -import org.elastic4play.{ BadRequestError, Timed } +import play.api.mvc._ + +import models.Roles +import services.TaskSrv + import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites -import org.elastic4play.services.{ Agg, AuxSrv } -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } import org.elastic4play.services.JsonFormat.{ aggReads, queryReads } -import services.TaskSrv +import org.elastic4play.services._ +import org.elastic4play.{ BadRequestError, Timed } @Singleton class TaskCtrl @Inject() ( @@ -21,34 +23,35 @@ class TaskCtrl @Inject() ( authenticated: Authenticated, renderer: Renderer, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller with Status { + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { @Timed - def create(caseId: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def create(caseId: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ taskSrv.create(caseId, request.body) .map(task ⇒ renderer.toOutput(CREATED, task)) } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ taskSrv.get(id) .map(task ⇒ renderer.toOutput(OK, task)) } @Timed - def update(id: String): Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ taskSrv.update(id, request.body) .map(task ⇒ renderer.toOutput(OK, task)) } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.write).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ taskSrv.delete(id) .map(_ ⇒ NoContent) } @Timed - def findInCase(caseId: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def findInCase(caseId: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ import org.elastic4play.services.QueryDSL._ val childQuery = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val query = and(childQuery, "_parent" ~= caseId) @@ -60,7 +63,7 @@ class TaskCtrl @Inject() ( } @Timed - def find: Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -73,7 +76,7 @@ class TaskCtrl @Inject() ( } @Timed - def stats(): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def stats(): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val aggs = request.body.getValue("stats").getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]] taskSrv.stats(query, aggs).map(s ⇒ Ok(s)) diff --git a/thehive-backend/app/controllers/UserCtrl.scala b/thehive-backend/app/controllers/UserCtrl.scala index e6f4f82730..d7922402c9 100644 --- a/thehive-backend/app/controllers/UserCtrl.scala +++ b/thehive-backend/app/controllers/UserCtrl.scala @@ -2,22 +2,22 @@ package controllers import javax.inject.{ Inject, Singleton } -import scala.annotation.implicitNotFound import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Try + import play.api.Logger import play.api.http.Status -import play.api.mvc.{ Action, AnyContent, Controller, Result } -import org.elastic4play.{ AuthorizationError, MissingAttributeError, Timed } -import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } -import org.elastic4play.models.JsonFormat.baseModelEntityWrites -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } -import org.elastic4play.services.AuthSrv -import org.elastic4play.services.JsonFormat.{ authContextWrites, queryReads } +import play.api.libs.json.{ JsObject, Json } +import play.api.mvc._ + +import models.Roles import services.UserSrv -import play.api.libs.json.Json -import scala.util.Try -import play.api.libs.json.JsObject +import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } +import org.elastic4play.models.JsonFormat.baseModelEntityWrites +import org.elastic4play.services.JsonFormat.queryReads +import org.elastic4play.services.{ AuthSrv, QueryDSL, QueryDef } +import org.elastic4play.{ AuthorizationError, MissingAttributeError, Timed } @Singleton class UserCtrl @Inject() ( @@ -26,31 +26,29 @@ class UserCtrl @Inject() ( authenticated: Authenticated, renderer: Renderer, fieldsBodyParser: FieldsBodyParser, - implicit val ec: ExecutionContext) extends Controller with Status { + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) with Status { - lazy val logger = Logger(getClass) + private[UserCtrl] lazy val logger = Logger(getClass) @Timed - def create: Action[Fields] = authenticated(Role.admin).async(fieldsBodyParser) { implicit request ⇒ + def create: Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ userSrv.create(request.body) .map(user ⇒ renderer.toOutput(CREATED, user)) } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ userSrv.get(id) - .map { user ⇒ - val json = if (request.roles.contains(Role.admin)) user.toAdminJson else user.toJson - renderer.toOutput(OK, json) - } + .map { user ⇒ renderer.toOutput(OK, user) } } @Timed - def update(id: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ - if (id == request.authContext.userId || request.authContext.roles.contains(Role.admin)) { - if (request.body.contains("password")) - logger.warn("Change password attribute using update operation is deprecated. Please use dedicated API (setPassword and changePassword)") - userSrv.update(id, request.body.unset("password")).map { user ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ + if (id == request.authContext.userId || request.authContext.roles.contains(Roles.admin)) { + if (request.body.contains("password") || request.body.contains("key")) + logger.warn("Change password or key using update operation is deprecated. Please use dedicated API (setPassword, changePassword or renewKey)") + userSrv.update(id, request.body.unset("password").unset("key")).map { user ⇒ renderer.toOutput(OK, user) } } @@ -60,7 +58,7 @@ class UserCtrl @Inject() ( } @Timed - def setPassword(login: String): Action[Fields] = authenticated(Role.admin).async(fieldsBodyParser) { implicit request ⇒ + def setPassword(login: String): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ request.body.getString("password") .fold(Future.failed[Result](MissingAttributeError("password"))) { password ⇒ authSrv.setPassword(login, password).map(_ ⇒ NoContent) @@ -68,7 +66,7 @@ class UserCtrl @Inject() ( } @Timed - def changePassword(login: String): Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def changePassword(login: String): Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ if (login == request.authContext.userId) { val fields = request.body fields.getString("password").fold(Future.failed[Result](MissingAttributeError("password"))) { password ⇒ @@ -83,7 +81,7 @@ class UserCtrl @Inject() ( } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ userSrv.delete(id) .map(_ ⇒ NoContent) } @@ -95,7 +93,7 @@ class UserCtrl @Inject() ( user ← userSrv.get(authContext.userId) preferences = Try(Json.parse(user.preferences())) .recover { - case error ⇒ + case _ ⇒ logger.warn(s"User ${authContext.userId} has invalid preference format: ${user.preferences()}") JsObject(Nil) } @@ -105,11 +103,26 @@ class UserCtrl @Inject() ( } @Timed - def find: Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) val (users, total) = userSrv.find(query, range, sort) renderer.toOutput(OK, users, total) } + + @Timed + def getKey(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ + authSrv.getKey(id).map(Ok(_)) + } + + @Timed + def removeKey(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ + authSrv.removeKey(id).map(_ ⇒ Ok) + } + + @Timed + def renewKey(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ + authSrv.renewKey(id).map(Ok(_)) + } } \ No newline at end of file diff --git a/thehive-backend/app/global/Filters.scala b/thehive-backend/app/global/Filters.scala index 180f6d3020..b319722e37 100644 --- a/thehive-backend/app/global/Filters.scala +++ b/thehive-backend/app/global/Filters.scala @@ -2,15 +2,16 @@ package global import javax.inject.{ Inject, Provider, Singleton } -import akka.stream.Materializer +import scala.collection.immutable + import play.api.Logger -import play.api.http.HttpFilters +import play.api.http.{ HttpFilters, SessionConfiguration } import play.api.libs.crypto.CSRFTokenSigner import play.api.mvc.{ EssentialFilter, RequestHeader } import play.filters.csrf.CSRF.{ ErrorHandler, TokenProvider } import play.filters.csrf.CSRFConfig -import scala.collection.immutable +import akka.stream.Materializer @Singleton class TheHiveFilters @Inject() (injectedFilters: immutable.Set[EssentialFilter]) extends HttpFilters { @@ -35,10 +36,12 @@ object CSRFFilter { class CSRFFilter @Inject() ( config: Provider[CSRFConfig], tokenSignerProvider: Provider[CSRFTokenSigner], + sessionConfiguration: SessionConfiguration, tokenProvider: TokenProvider, errorHandler: ErrorHandler)(mat: Materializer) extends play.filters.csrf.CSRFFilter( config.get.copy(shouldProtect = CSRFFilter.shouldProtect), tokenSignerProvider.get, + sessionConfiguration, tokenProvider, errorHandler)(mat) \ No newline at end of file diff --git a/thehive-backend/app/global/TheHive.scala b/thehive-backend/app/global/TheHive.scala index f9f0f6f19d..a7d4b0c67a 100644 --- a/thehive-backend/app/global/TheHive.scala +++ b/thehive-backend/app/global/TheHive.scala @@ -1,6 +1,10 @@ package global -import java.net.{ URL, URLClassLoader } +import scala.collection.JavaConverters._ + +import play.api.libs.concurrent.AkkaGuiceSupport +import play.api.mvc.EssentialFilter +import play.api.{ Configuration, Environment, Logger, Mode } import com.google.inject.AbstractModule import com.google.inject.name.Names @@ -8,16 +12,14 @@ import connectors.Connector import controllers.{ AssetCtrl, AssetCtrlDev, AssetCtrlProd } import models.Migration import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder } -import org.elastic4play.models.BaseModelDef -import org.elastic4play.services.auth.MultiAuthSrv -import org.elastic4play.services.{ AuthSrv, AuthSrvFactory, MigrationOperations, TempFilter } import org.reflections.Reflections -import play.api.libs.concurrent.AkkaGuiceSupport -import play.api.mvc.EssentialFilter -import play.api.{ Configuration, Environment, Logger, Mode } -import services.{ AuditSrv, AuditedModel, StreamFilter, StreamMonitor } +import org.reflections.scanners.SubTypesScanner +import org.reflections.util.ConfigurationBuilder +import services._ -import scala.collection.JavaConversions.asScalaSet +import org.elastic4play.models.BaseModelDef +import org.elastic4play.services.auth.MultiAuthSrv +import org.elastic4play.services.{ AuthSrv, MigrationOperations, TempFilter } class TheHive( environment: Environment, @@ -31,17 +33,20 @@ class TheHive( val modelBindings = ScalaMultibinder.newSetBinder[BaseModelDef](binder) val auditedModelBindings = ScalaMultibinder.newSetBinder[AuditedModel](binder) val authBindings = ScalaMultibinder.newSetBinder[AuthSrv](binder) - val authFactoryBindings = ScalaMultibinder.newSetBinder[AuthSrvFactory](binder) - val packageUrls = Seq(getClass.getClassLoader, classOf[org.elastic4play.Timed].getClassLoader).flatMap { - case ucl: URLClassLoader ⇒ ucl.getURLs - case _ ⇒ Array.empty[URL] - } + val reflectionClasses = new Reflections(new ConfigurationBuilder() + .forPackages("org.elastic4play") + .forPackages("connectors.cortex") + .forPackages("connectors.misp") + .forPackages("connectors.metrics") + .addClassLoader(getClass.getClassLoader) + .addClassLoader(environment.getClass.getClassLoader) + .setExpandSuperTypes(false) + .setScanners(new SubTypesScanner(false))) - new Reflections(new org.reflections.util.ConfigurationBuilder() - .addUrls(packageUrls: _*) - .setScanners(new org.reflections.scanners.SubTypesScanner(false))) + reflectionClasses .getSubTypesOf(classOf[BaseModelDef]) + .asScala .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers)) .foreach { modelClass ⇒ logger.info(s"Loading model $modelClass") @@ -51,23 +56,13 @@ class TheHive( } } - new Reflections(new org.reflections.util.ConfigurationBuilder() - .addUrls(packageUrls: _*) - .setScanners(new org.reflections.scanners.SubTypesScanner(false))) + reflectionClasses .getSubTypesOf(classOf[AuthSrv]) + .asScala .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers) || c.isMemberClass) - .filterNot(_ == classOf[MultiAuthSrv]) - .foreach { modelClass ⇒ - authBindings.addBinding.to(modelClass) - } - - new Reflections(new org.reflections.util.ConfigurationBuilder() - .addUrls(packageUrls: _*) - .setScanners(new org.reflections.scanners.SubTypesScanner(false))) - .getSubTypesOf(classOf[AuthSrvFactory]) - .filterNot(c ⇒ java.lang.reflect.Modifier.isAbstract(c.getModifiers)) - .foreach { modelClass ⇒ - authFactoryBindings.addBinding.to(modelClass) + .filterNot(c ⇒ c == classOf[MultiAuthSrv] || c == classOf[TheHiveAuthSrv]) + .foreach { authSrvClass ⇒ + authBindings.addBinding.to(authSrvClass) } val filterBindings = ScalaMultibinder.newSetBinder[EssentialFilter](binder) @@ -76,9 +71,10 @@ class TheHive( filterBindings.addBinding.to[CSRFFilter] bind[MigrationOperations].to[Migration] - bind[AuthSrv].to[MultiAuthSrv] - bind[StreamMonitor].asEagerSingleton() - bind[AuditSrv].asEagerSingleton() + bind[AuthSrv].to[TheHiveAuthSrv] + + bindActor[AuditActor]("AuditActor") + bindActor[DeadLetterMonitoringActor]("DeadLetterMonitoringActor") if (environment.mode == Mode.Prod) bind[AssetCtrl].to[AssetCtrlProd] diff --git a/thehive-backend/app/models/Alert.scala b/thehive-backend/app/models/Alert.scala index b6cb770234..e2da83e9d4 100644 --- a/thehive-backend/app/models/Alert.scala +++ b/thehive-backend/app/models/Alert.scala @@ -3,18 +3,20 @@ package models import java.util.Date import javax.inject.{ Inject, Singleton } +import scala.concurrent.Future +import scala.util.Try + +import play.api.Logger +import play.api.libs.json._ + import models.JsonFormat.alertStatusFormat +import services.AuditedModel + import org.elastic4play.controllers.JsonInputValue import org.elastic4play.models.{ Attribute, AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, MultiAttributeFormat, OptionalAttributeFormat, AttributeFormat ⇒ F, AttributeOption ⇒ O } import org.elastic4play.services.DBLists import org.elastic4play.utils.Hasher import org.elastic4play.{ AttributeCheckingError, InvalidFormatAttributeError } -import play.api.Logger -import play.api.libs.json._ -import services.AuditedModel - -import scala.concurrent.Future -import scala.util.Try object AlertStatus extends Enumeration with HiveEnumeration { type Type = Value @@ -95,7 +97,7 @@ class AlertModel @Inject() (dblists: DBLists) val sourceRef = (attrs \ "sourceRef").asOpt[String].getOrElse("") val _id = hasher.fromString(s"$tpe|$source|$sourceRef").head.toString() attrs + ("_id" → JsString(_id)) - } - "lastSyncDate" - "case" - "status" - "follow" + } } } } diff --git a/thehive-backend/app/models/Artifact.scala b/thehive-backend/app/models/Artifact.scala index 379c2ec931..401c83542b 100644 --- a/thehive-backend/app/models/Artifact.scala +++ b/thehive-backend/app/models/Artifact.scala @@ -3,25 +3,24 @@ package models import java.util.Date import javax.inject.{ Inject, Provider, Singleton } -import akka.{ Done, NotUsed } - import scala.concurrent.{ ExecutionContext, Future } -import scala.language.postfixOps -import akka.stream.{ IOResult, Materializer } -import play.api.libs.json.{ JsArray, JsNull, JsObject, JsString, JsValue } +import scala.util.Success + +import play.api.Logger import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup import play.api.libs.json.JsValue.jsValueToJsLookup -import play.api.libs.json.Json import play.api.libs.json.Json.toJsFieldJsValueWrapper -import org.elastic4play.{ BadRequestError, InternalError } -import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O } -import org.elastic4play.services.{ Attachment, AttachmentSrv, DBLists } -import org.elastic4play.utils.MultiHash +import play.api.libs.json._ + +import akka.stream.{ IOResult, Materializer } +import akka.{ Done, NotUsed } import models.JsonFormat.artifactStatusFormat -import play.api.Logger import services.{ ArtifactSrv, AuditedModel } -import scala.util.Success +import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O } +import org.elastic4play.services.{ Attachment, AttachmentSrv, DBLists } +import org.elastic4play.utils.MultiHash +import org.elastic4play.{ BadRequestError, InternalError } object ArtifactStatus extends Enumeration with HiveEnumeration { type Type = Value @@ -75,14 +74,14 @@ class ArtifactModel @Inject() ( entity match { case artifact: Artifact ⇒ val removeMessage = (updateAttrs \ "message").toOption.exists { - case JsNull ⇒ true - case JsArray(Nil) ⇒ true - case _ ⇒ false + case JsNull ⇒ true + case JsArray(Seq()) ⇒ true + case _ ⇒ false } val removeTags = (updateAttrs \ "tags").toOption.exists { - case JsNull ⇒ true - case JsArray(Nil) ⇒ true - case _ ⇒ false + case JsNull ⇒ true + case JsArray(Seq()) ⇒ true + case _ ⇒ false } if ((removeMessage && removeTags) || (removeMessage && artifact.tags().isEmpty) || @@ -115,6 +114,7 @@ class ArtifactModel @Inject() ( entity match { case artifact: Artifact ⇒ val (_, total) = artifactSrv.get.findSimilar(artifact, Some("0-0"), Nil) + total.failed.foreach(t ⇒ logger.error("Artifact.getStats error", t)) total.map { t ⇒ Json.obj("seen" → t) } case _ ⇒ Future.successful(JsObject(Nil)) } diff --git a/thehive-backend/app/models/Audit.scala b/thehive-backend/app/models/Audit.scala index 3bb152c059..5c9d1c8860 100644 --- a/thehive-backend/app/models/Audit.scala +++ b/thehive-backend/app/models/Audit.scala @@ -4,12 +4,15 @@ import java.util.Date import javax.inject.{ Inject, Singleton } import scala.collection.immutable -import play.api.{ Configuration, Logger } + import play.api.libs.json.JsObject -import org.elastic4play.models.{ Attribute, AttributeFormat, AttributeDef, EntityDef, EnumerationAttributeFormat, ListEnumerationAttributeFormat, ModelDef, MultiAttributeFormat, ObjectAttributeFormat, OptionalAttributeFormat, StringAttributeFormat, AttributeOption ⇒ O } +import play.api.{ Configuration, Logger } + +import services.AuditedModel + +import org.elastic4play.models.{ Attribute, AttributeDef, AttributeFormat, EntityDef, EnumerationAttributeFormat, ListEnumerationAttributeFormat, ModelDef, MultiAttributeFormat, ObjectAttributeFormat, OptionalAttributeFormat, StringAttributeFormat, AttributeOption ⇒ O } import org.elastic4play.services.AuditableAction import org.elastic4play.services.JsonFormat.auditableActionFormat -import services.AuditedModel trait AuditAttributes { _: AttributeDef ⇒ def detailsAttributes: Seq[Attribute[_]] @@ -34,10 +37,10 @@ class AuditModel( configuration: Configuration, auditedModels: immutable.Set[AuditedModel]) = this( - configuration.getString("audit.name").get, + configuration.get[String]("audit.name"), auditedModels) - lazy val logger = Logger(getClass) + private[AuditModel] lazy val logger = Logger(getClass) def mergeAttributeFormat(context: String, format1: AttributeFormat[_], format2: AttributeFormat[_]): Option[AttributeFormat[_]] = { (format1, format2) match { diff --git a/thehive-backend/app/models/Case.scala b/thehive-backend/app/models/Case.scala index 8fe9d83d55..059bf97590 100644 --- a/thehive-backend/app/models/Case.scala +++ b/thehive-backend/app/models/Case.scala @@ -3,18 +3,20 @@ package models import java.util.Date import javax.inject.{ Inject, Provider, Singleton } -import models.JsonFormat.{ caseImpactStatusFormat, caseResolutionStatusFormat, caseStatusFormat } -import org.elastic4play.JsonFormat.dateFormat -import org.elastic4play.models.{ AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, AttributeFormat ⇒ F, AttributeOption ⇒ O } -import org.elastic4play.services.{ FindSrv, SequenceSrv } +import scala.concurrent.{ ExecutionContext, Future } +import scala.math.BigDecimal.{ int2bigDecimal, long2bigDecimal } + import play.api.Logger import play.api.libs.json.JsValue.jsValueToJsLookup import play.api.libs.json.Json.toJsFieldJsValueWrapper import play.api.libs.json._ + +import models.JsonFormat.{ caseImpactStatusFormat, caseResolutionStatusFormat, caseStatusFormat } import services.{ AuditedModel, CaseSrv } -import scala.concurrent.{ ExecutionContext, Future } -import scala.math.BigDecimal.{ int2bigDecimal, long2bigDecimal } +import org.elastic4play.JsonFormat.dateFormat +import org.elastic4play.models.{ AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, AttributeFormat ⇒ F, AttributeOption ⇒ O } +import org.elastic4play.services.{ FindSrv, SequenceSrv } object CaseStatus extends Enumeration with HiveEnumeration { type Type = Value @@ -57,11 +59,12 @@ class CaseModel @Inject() ( artifactModel: Provider[ArtifactModel], taskModel: Provider[TaskModel], caseSrv: Provider[CaseSrv], + alertModel: Provider[AlertModel], sequenceSrv: SequenceSrv, findSrv: FindSrv, implicit val ec: ExecutionContext) extends ModelDef[CaseModel, Case]("case") with CaseAttributes with AuditedModel { caseModel ⇒ - lazy val logger = Logger(getClass) + private[CaseModel] lazy val logger = Logger(getClass) override val defaultSortBy = Seq("-startDate") override val removeAttribute: JsObject = Json.obj("status" → CaseStatus.Deleted) @@ -140,6 +143,22 @@ class CaseModel @Inject() ( case _ ⇒ Json.obj() } } + + private[models] def buildAlertStats(caze: Case): Future[JsObject] = { + import org.elastic4play.services.QueryDSL._ + findSrv( + alertModel.get, + "case" ~= caze.id, + groupByField("type", groupByField("source", selectCount))) + .map { alertStatsJson ⇒ + val alertStats = for { + (tpe, JsObject(srcStats)) ← alertStatsJson.value + src ← srcStats.keys + } yield Json.obj("type" → tpe, "source" → src) + Json.obj("alerts" → alertStats) + } + } + override def getStats(entity: BaseEntity): Future[JsObject] = { entity match { @@ -147,9 +166,10 @@ class CaseModel @Inject() ( for { taskStats ← buildTaskStats(caze) artifactStats ← buildArtifactStats(caze) + alertStats ← buildAlertStats(caze) mergeIntoStats ← buildMergeIntoStats(caze) mergeFromStats ← buildMergeFromStats(caze) - } yield taskStats ++ artifactStats ++ mergeIntoStats ++ mergeFromStats + } yield taskStats ++ artifactStats ++ alertStats ++ mergeIntoStats ++ mergeFromStats case other ⇒ logger.warn(s"Request caseStats from a non-case entity ?! ${other.getClass}:$other") Future.successful(Json.obj()) diff --git a/thehive-backend/app/models/CaseTemplate.scala b/thehive-backend/app/models/CaseTemplate.scala index 833f58dcd2..f6775f64a9 100644 --- a/thehive-backend/app/models/CaseTemplate.scala +++ b/thehive-backend/app/models/CaseTemplate.scala @@ -3,9 +3,11 @@ package models import javax.inject.{ Inject, Singleton } import play.api.libs.json.{ JsObject, JsValue } -import org.elastic4play.models.{ Attribute, AttributeDef, EntityDef, HiveEnumeration, ModelDef, AttributeFormat ⇒ F } + import models.JsonFormat.caseTemplateStatusFormat +import org.elastic4play.models.{ Attribute, AttributeDef, EntityDef, HiveEnumeration, ModelDef, AttributeFormat ⇒ F } + object CaseTemplateStatus extends Enumeration with HiveEnumeration { type Type = Value val Ok, Deleted = Value diff --git a/thehive-backend/app/models/JsonFormat.scala b/thehive-backend/app/models/JsonFormat.scala index 824976b89d..3f79ed3ef5 100644 --- a/thehive-backend/app/models/JsonFormat.scala +++ b/thehive-backend/app/models/JsonFormat.scala @@ -2,8 +2,10 @@ package models import java.nio.file.Path +import play.api.libs.json._ + import org.elastic4play.models.JsonFormat.enumFormat -import play.api.libs.json.{ Format, JsString, Writes } +import org.elastic4play.services.Role object JsonFormat { implicit val userStatusFormat: Format[UserStatus.Type] = enumFormat(UserStatus) @@ -17,4 +19,11 @@ object JsonFormat { implicit val alertStatusFormat: Format[AlertStatus.Type] = enumFormat(AlertStatus) implicit val pathWrites: Writes[Path] = Writes((value: Path) ⇒ JsString(value.toString)) + + private val roleWrites: Writes[Role] = Writes((role: Role) ⇒ JsString(role.name.toLowerCase())) + private val roleReads: Reads[Role] = Reads { + case JsString(s) if Roles.isValid(s) ⇒ JsSuccess(Roles.withName(s).get) + case _ ⇒ JsError(Seq(JsPath → Seq(JsonValidationError(s"error.expected.role(${Roles.roleNames}")))) + } + implicit val roleFormat: Format[Role] = Format[Role](roleReads, roleWrites) } \ No newline at end of file diff --git a/thehive-backend/app/models/Log.scala b/thehive-backend/app/models/Log.scala index ea49125cd0..cf07277b62 100644 --- a/thehive-backend/app/models/Log.scala +++ b/thehive-backend/app/models/Log.scala @@ -1,17 +1,16 @@ package models import java.util.Date - import javax.inject.{ Inject, Singleton } -import play.api.libs.json.{ JsObject, Json } import play.api.libs.json.Json.toJsFieldJsValueWrapper - -import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, ChildModelDef, EntityDef, HiveEnumeration } +import play.api.libs.json.{ JsObject, Json } import models.JsonFormat.logStatusFormat import services.AuditedModel +import org.elastic4play.models.{ AttributeDef, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F, AttributeOption ⇒ O } + object LogStatus extends Enumeration with HiveEnumeration { type Type = Value val Ok, Deleted = Value diff --git a/thehive-backend/app/models/Migration.scala b/thehive-backend/app/models/Migration.scala index 80ec8ae3e5..d5467b7138 100644 --- a/thehive-backend/app/models/Migration.scala +++ b/thehive-backend/app/models/Migration.scala @@ -1,28 +1,31 @@ package models import java.util.Date -import javax.inject.Inject +import javax.inject.{ Inject, Singleton } + +import scala.collection.immutable.{ Set ⇒ ISet } +import scala.concurrent.{ ExecutionContext, Future } +import scala.math.BigDecimal.int2bigDecimal +import scala.util.Try + +import play.api.libs.json.JsValue.jsValueToJsLookup +import play.api.libs.json._ +import play.api.{ Configuration, Logger } import akka.NotUsed import akka.stream.Materializer import akka.stream.scaladsl.Source +import services.AlertSrv + import org.elastic4play.models.BaseModelDef import org.elastic4play.services.JsonFormat.attachmentFormat import org.elastic4play.services._ import org.elastic4play.utils import org.elastic4play.utils.{ Hasher, RichJson } -import play.api.libs.json.JsValue.jsValueToJsLookup -import play.api.libs.json._ -import play.api.{ Configuration, Logger } -import services.AlertSrv - -import scala.collection.immutable.{ Set ⇒ ISet } -import scala.concurrent.{ ExecutionContext, Future } -import scala.math.BigDecimal.int2bigDecimal -import scala.util.Try case class UpdateMispAlertArtifact() extends EventMessage +@Singleton class Migration( mispCaseTemplate: Option[String], mainHash: String, @@ -41,17 +44,17 @@ class Migration( ec: ExecutionContext, materializer: Materializer) = { this( - configuration.getString("misp.caseTemplate"), - configuration.getString("datastore.hash.main").get, - configuration.getStringSeq("datastore.hash.extra").get, - configuration.getString("datastore.name").get, + configuration.getOptional[String]("misp.caseTemplate"), + configuration.get[String]("datastore.hash.main"), + configuration.get[Seq[String]]("datastore.hash.extra"), + configuration.get[String]("datastore.name"), models, dblists, eventSrv, ec, materializer) } import org.elastic4play.services.Operation._ - val logger = Logger(getClass) + private[Migration] lazy val logger = Logger(getClass) private var requireUpdateMispAlertArtifact = false override def beginMigration(version: Int): Future[Unit] = Future.successful(()) diff --git a/thehive-backend/app/models/Roles.scala b/thehive-backend/app/models/Roles.scala new file mode 100644 index 0000000000..927b57e60f --- /dev/null +++ b/thehive-backend/app/models/Roles.scala @@ -0,0 +1,50 @@ +package models + +import play.api.libs.json.{ JsString, JsValue } + +import com.sksamuel.elastic4s.ElasticDsl.keywordField +import com.sksamuel.elastic4s.mappings.KeywordFieldDefinition +import org.scalactic.{ Every, Good, One, Or } +import models.JsonFormat.roleFormat + +import org.elastic4play.{ AttributeError, InvalidFormatAttributeError } +import org.elastic4play.controllers.{ InputValue, JsonInputValue, StringInputValue } +import org.elastic4play.models.AttributeFormat +import org.elastic4play.services.Role + +object Roles { + object read extends Role("read") + object write extends Role("write") + object admin extends Role("admin") + object alert extends Role("alert") + val roles: List[Role] = read :: write :: admin :: alert :: Nil + + val roleNames: List[String] = roles.map(_.name) + def isValid(roleName: String): Boolean = roleNames.contains(roleName.toLowerCase()) + def withName(roleName: String): Option[Role] = { + val lowerCaseRole = roleName.toLowerCase() + roles.find(_.name == lowerCaseRole) + } +} + +object RoleAttributeFormat extends AttributeFormat[Role]("role") { + + override def checkJson(subNames: Seq[String], value: JsValue): Or[JsValue, One[InvalidFormatAttributeError]] = value match { + case JsString(v) if subNames.isEmpty && Roles.isValid(v) ⇒ Good(value) + case _ ⇒ formatError(JsonInputValue(value)) + } + + override def fromInputValue(subNames: Seq[String], value: InputValue): Role Or Every[AttributeError] = { + if (subNames.nonEmpty) + formatError(value) + else + (value match { + case StringInputValue(Seq(v)) ⇒ Good(v) + case JsonInputValue(JsString(v)) ⇒ Good(v) + case _ ⇒ formatError(value) + }).flatMap(v ⇒ Roles.withName(v).fold[Role Or Every[AttributeError]](formatError(value))(role ⇒ Good(role))) + + } + + override def elasticType(attributeName: String): KeywordFieldDefinition = keywordField(attributeName) +} \ No newline at end of file diff --git a/thehive-backend/app/models/Task.scala b/thehive-backend/app/models/Task.scala index 4efa4b2433..0b99e25417 100644 --- a/thehive-backend/app/models/Task.scala +++ b/thehive-backend/app/models/Task.scala @@ -1,21 +1,20 @@ package models import java.util.Date - import javax.inject.{ Inject, Singleton } import scala.concurrent.Future -import play.api.libs.json.{ JsBoolean, JsObject } import play.api.libs.json.JsValue.jsValueToJsLookup - -import org.elastic4play.JsonFormat.dateFormat -import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration } -import org.elastic4play.utils.RichJson +import play.api.libs.json.{ JsBoolean, JsObject } import models.JsonFormat.taskStatusFormat import services.AuditedModel +import org.elastic4play.JsonFormat.dateFormat +import org.elastic4play.models.{ AttributeDef, BaseEntity, ChildModelDef, EntityDef, HiveEnumeration, AttributeFormat ⇒ F } +import org.elastic4play.utils.RichJson + object TaskStatus extends Enumeration with HiveEnumeration { type Type = Value val Waiting, InProgress, Completed, Cancel = Value diff --git a/thehive-backend/app/models/User.scala b/thehive-backend/app/models/User.scala index 673b3d02f9..f092b0801e 100644 --- a/thehive-backend/app/models/User.scala +++ b/thehive-backend/app/models/User.scala @@ -1,20 +1,15 @@ package models -import java.util.UUID - import scala.concurrent.Future -import scala.language.postfixOps -import play.api.libs.json.{ JsBoolean, JsObject, JsString, JsUndefined } import play.api.libs.json.JsValue.jsValueToJsLookup - -import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, BaseEntity, EntityDef, HiveEnumeration, ModelDef } -import org.elastic4play.services.JsonFormat.roleFormat -import org.elastic4play.services.Role +import play.api.libs.json.{ JsArray, JsBoolean, JsObject, JsString } import models.JsonFormat.userStatusFormat import services.AuditedModel +import org.elastic4play.models.{ AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef, AttributeFormat ⇒ F, AttributeOption ⇒ O } + object UserStatus extends Enumeration with HiveEnumeration { type Type = Value val Ok, Locked = Value @@ -23,10 +18,9 @@ object UserStatus extends Enumeration with HiveEnumeration { trait UserAttributes { _: AttributeDef ⇒ val login = attribute("login", F.stringFmt, "Login of the user", O.form) val userId = attribute("_id", F.stringFmt, "User id (login)", O.model) - val withKey = optionalAttribute("with-key", F.booleanFmt, "Generate an API key", O.form) - val key = optionalAttribute("key", F.uuidFmt, "API key", O.model, O.sensitive, O.unaudited) + val key = optionalAttribute("key", F.stringFmt, "API key", O.sensitive, O.unaudited) val userName = attribute("name", F.stringFmt, "Full name (Firstname Lastname)") - val roles = multiAttribute("roles", F.enumFmt(Role), "Comma separated role list (READ, WRITE and ADMIN)") + val roles = multiAttribute("roles", RoleAttributeFormat, "Comma separated role list (READ, WRITE and ADMIN)") val status = attribute("status", F.enumFmt(UserStatus), "Status of the user", UserStatus.Ok) val password = optionalAttribute("password", F.stringFmt, "Password", O.sensitive, O.unaudited) val avatar = optionalAttribute("avatar", F.stringFmt, "Base64 representation of user avatar image", O.unaudited) @@ -35,26 +29,18 @@ trait UserAttributes { _: AttributeDef ⇒ class UserModel extends ModelDef[UserModel, User]("user") with UserAttributes with AuditedModel { - private def addKey = (attrs: JsObject) ⇒ attrs \ "with-key" match { - case _: JsUndefined ⇒ attrs - case _ ⇒ attrs + ("key" → JsString(UUID.randomUUID.toString)) - "with-key" - } - - private def setUserId = (attrs: JsObject) ⇒ (attrs \ "login").asOpt[JsString].fold(attrs) { login ⇒ + private def setUserId(attrs: JsObject) = (attrs \ "login").asOpt[JsString].fold(attrs) { login ⇒ attrs - "login" + ("_id" → login) } - override def creationHook(parent: Option[BaseEntity], attrs: JsObject) = Future.successful(addKey.andThen(setUserId)(attrs)) - - override def updateHook(user: BaseEntity, updateAttrs: JsObject): Future[JsObject] = Future.successful(addKey(updateAttrs)) + override def creationHook(parent: Option[BaseEntity], attrs: JsObject): Future[JsObject] = Future.successful(setUserId(attrs)) } class User(model: UserModel, attributes: JsObject) extends EntityDef[UserModel, User](model, attributes) with UserAttributes with org.elastic4play.services.User { - override def toJson = super.toJson + - ("has-key" → JsBoolean(key().isDefined)) - - def toAdminJson = key().fold(toJson) { k ⇒ toJson + ("key" → JsString(k.toString)) } - override def getUserName = userName() override def getRoles = roles() + + override def toJson: JsObject = super.toJson + + ("roles" → JsArray(roles().map(r ⇒ JsString(r.name.toLowerCase())))) + + ("hasKey" → JsBoolean(key().isDefined)) } \ No newline at end of file diff --git a/thehive-backend/app/services/AlertSrv.scala b/thehive-backend/app/services/AlertSrv.scala index 2ee217216a..1cf512019e 100644 --- a/thehive-backend/app/services/AlertSrv.scala +++ b/thehive-backend/app/services/AlertSrv.scala @@ -1,25 +1,27 @@ package services import java.nio.file.Files -import javax.inject.Inject +import javax.inject.{ Inject, Singleton } + +import scala.collection.immutable +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.matching.Regex +import scala.util.{ Failure, Try } + +import play.api.libs.json._ +import play.api.{ Configuration, Logger } import akka.NotUsed import akka.stream.Materializer import akka.stream.scaladsl.{ Sink, Source } import connectors.ConnectorRouter import models._ + import org.elastic4play.InternalError import org.elastic4play.controllers.{ Fields, FileInputValue } import org.elastic4play.services.JsonFormat.attachmentFormat import org.elastic4play.services.QueryDSL.{ groupByField, parent, selectCount, withId } import org.elastic4play.services._ -import play.api.libs.json._ -import play.api.{ Configuration, Logger } - -import scala.collection.immutable -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.matching.Regex -import scala.util.{ Failure, Try } trait AlertTransformer { def createCase(alert: Alert, customCaseTemplate: Option[String])(implicit authContext: AuthContext): Future[Case] @@ -33,7 +35,9 @@ object AlertSrv { val dataExtractor: Regex = "^(.*);(.*);(.*)".r } +@Singleton class AlertSrv( + maxSimilarCases: Int, templates: Map[String, String], alertModel: AlertModel, createSrv: CreateSrv, @@ -65,6 +69,7 @@ class AlertSrv( connectors: ConnectorRouter, ec: ExecutionContext, mat: Materializer) = this( + configuration.getOptional[Int]("maxSimilarCases").getOrElse(100), Map.empty[String, String], alertModel: AlertModel, createSrv, @@ -77,7 +82,7 @@ class AlertSrv( caseTemplateSrv, attachmentSrv, connectors, - (configuration.getString("datastore.hash.main").get +: configuration.getStringSeq("datastore.hash.extra").get).distinct, + (configuration.get[String]("datastore.hash.main") +: configuration.get[Seq[String]]("datastore.hash.extra")).distinct, ec, mat) @@ -171,7 +176,7 @@ class AlertSrv( caseTemplate ← getCaseTemplate(alert, customCaseTemplate) caze ← caseSrv.create( Fields.empty - .set("title", s"#${alert.sourceRef()} " + alert.title()) + .set("title", alert.title()) .set("description", alert.description()) .set("severity", JsNumber(alert.severity())) .set("tags", JsArray(alert.tags().map(JsString))) @@ -300,7 +305,7 @@ class AlertSrv( caseSrv.get(caseId).map((_, similarIOCCount, similarArtifactCount)) } .filter { - case (caze, _, _) ⇒ caze.status() != CaseStatus.Deleted && caze.resolutionStatus != CaseResolutionStatus.Duplicated + case (caze, _, _) ⇒ caze.status() != CaseStatus.Deleted && !caze.resolutionStatus().contains(CaseResolutionStatus.Duplicated) } .mapAsyncUnordered(5) { case (caze, similarIOCCount, similarArtifactCount) ⇒ diff --git a/thehive-backend/app/services/ArtifactSrv.scala b/thehive-backend/app/services/ArtifactSrv.scala index e37daab702..9f957004ab 100644 --- a/thehive-backend/app/services/ArtifactSrv.scala +++ b/thehive-backend/app/services/ArtifactSrv.scala @@ -2,19 +2,21 @@ package services import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.{ Failure, Try } + +import play.api.Logger +import play.api.libs.json.JsObject +import play.api.libs.json.JsValue.jsValueToJsLookup + import akka.NotUsed import akka.stream.scaladsl.Source import models.{ CaseResolutionStatus, CaseStatus, _ } + import org.elastic4play.ConflictError import org.elastic4play.controllers.Fields import org.elastic4play.services._ import org.elastic4play.utils.{ RichFuture, RichOr } -import play.api.Logger -import play.api.libs.json.JsObject -import play.api.libs.json.JsValue.jsValueToJsLookup - -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.{ Failure, Try } @Singleton class ArtifactSrv @Inject() ( @@ -37,7 +39,7 @@ class ArtifactSrv @Inject() ( def create(caze: Case, fields: Fields)(implicit authContext: AuthContext): Future[Artifact] = { createSrv[ArtifactModel, Artifact, Case](artifactModel, caze, fields) .recoverWith { - case error ⇒ updateIfDeleted(caze, fields) // maybe the artifact already exists. If so, search it and update it + case _ ⇒ updateIfDeleted(caze, fields) // maybe the artifact already exists. If so, search it and update it } } @@ -71,7 +73,7 @@ class ArtifactSrv @Inject() ( case t ⇒ Future.successful(t) } - def get(id: String)(implicit authContext: AuthContext): Future[Artifact] = { + def get(id: String): Future[Artifact] = { getSrv[ArtifactModel, Artifact](artifactModel, id) } diff --git a/thehive-backend/app/services/AuditSrv.scala b/thehive-backend/app/services/AuditSrv.scala index ad3fb04555..4603969a3c 100644 --- a/thehive-backend/app/services/AuditSrv.scala +++ b/thehive-backend/app/services/AuditSrv.scala @@ -2,17 +2,18 @@ package services import javax.inject.{ Inject, Singleton } -import akka.actor.ActorDSL.{ Act, actor } -import akka.actor.{ ActorRef, ActorSystem } +import scala.concurrent.ExecutionContext + +import play.api.Logger +import play.api.libs.json.{ JsObject, Json } + +import akka.actor.Actor import models.{ Audit, AuditModel } + import org.elastic4play.controllers.Fields import org.elastic4play.models.{ Attribute, BaseEntity, BaseModelDef } import org.elastic4play.services._ import org.elastic4play.utils.Instance -import play.api.Logger -import play.api.libs.json.{ JsBoolean, JsObject, Json } - -import scala.concurrent.ExecutionContext trait AuditedModel { self: BaseModelDef ⇒ def attributes: Seq[Attribute[_]] @@ -34,39 +35,48 @@ trait AuditedModel { self: BaseModelDef ⇒ } @Singleton -class AuditSrv @Inject() ( +class AuditActor @Inject() ( auditModel: AuditModel, - eventSrv: EventSrv, createSrv: CreateSrv, - implicit val ec: ExecutionContext, - implicit val system: ActorSystem) { + eventSrv: EventSrv, + webHooks: WebHooks, + implicit val ec: ExecutionContext) extends Actor { object EntityExtractor { def unapply(e: BaseEntity) = Some((e.model, e.id, e.routing)) } + var currentRequestIds = Set.empty[String] + private[AuditActor] lazy val logger = Logger(getClass) + + override def preStart(): Unit = { + eventSrv.subscribe(self, classOf[EventMessage]) + super.preStart() + } - val auditActor: ActorRef = actor(new Act { + override def postStop(): Unit = { + eventSrv.unsubscribe(self) + super.postStop() + } - lazy val log = Logger(getClass) - var currentRequestIds = Set.empty[String] + override def receive: Receive = { + case RequestProcessEnd(request, _) ⇒ + currentRequestIds = currentRequestIds - Instance.getRequestId(request) + case AuditOperation(EntityExtractor(model: AuditedModel, id, routing), action, details, authContext, date) ⇒ + val requestId = authContext.requestId + val audit = Json.obj( + "operation" → action, + "details" → model.selectAuditedAttributes(details), + "objectType" → model.name, + "objectId" → id, + "base" → !currentRequestIds.contains(requestId), + "startDate" → date, + "rootId" → routing, + "requestId" → requestId) - become { - case RequestProcessEnd(request, _) ⇒ - currentRequestIds = currentRequestIds - Instance.getRequestId(request) - case AuditOperation(EntityExtractor(model: AuditedModel, id, routing), action, details, authContext, date) ⇒ - val requestId = authContext.requestId - createSrv[AuditModel, Audit](auditModel, Fields.empty - .set("operation", action.toString) - .set("details", model.selectAuditedAttributes(details)) - .set("objectType", model.name) - .set("objectId", id) - .set("base", JsBoolean(!currentRequestIds.contains(requestId))) - .set("startDate", Json.toJson(date)) - .set("rootId", routing) - .set("requestId", requestId))(authContext) - .onFailure { case t ⇒ log.error("Audit error", t) } - currentRequestIds = currentRequestIds + requestId - } - }) - eventSrv.subscribe(auditActor, classOf[EventMessage]) // need to unsubsribe ? + createSrv[AuditModel, Audit](auditModel, Fields(audit))(authContext) + .failed.foreach(t ⇒ logger.error("Audit error", t)) + currentRequestIds = currentRequestIds + requestId + + webHooks.send(audit) + } } \ No newline at end of file diff --git a/thehive-backend/app/services/CaseMergeSrv.scala b/thehive-backend/app/services/CaseMergeSrv.scala index 6e7fecb009..9bd9b7788d 100644 --- a/thehive-backend/app/services/CaseMergeSrv.scala +++ b/thehive-backend/app/services/CaseMergeSrv.scala @@ -3,20 +3,22 @@ package services import java.util.Date import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } +import scala.math.BigDecimal.long2bigDecimal +import scala.util.Failure + +import play.api.Logger +import play.api.libs.json.JsValue.jsValueToJsLookup +import play.api.libs.json._ + import akka.Done import akka.stream.Materializer import akka.stream.scaladsl.Sink import models._ + import org.elastic4play.controllers.{ AttachmentInputValue, Fields } import org.elastic4play.models.BaseEntity import org.elastic4play.services.{ AuthContext, EventMessage, EventSrv } -import play.api.Logger -import play.api.libs.json.JsValue.jsValueToJsLookup -import play.api.libs.json._ - -import scala.concurrent.{ ExecutionContext, Future } -import scala.math.BigDecimal.long2bigDecimal -import scala.util.Failure case class MergeArtifact(newArtifact: Artifact, artifacts: Seq[Artifact], authContext: AuthContext) extends EventMessage diff --git a/thehive-backend/app/services/CaseSrv.scala b/thehive-backend/app/services/CaseSrv.scala index 93db6a64e4..e802d77cab 100644 --- a/thehive-backend/app/services/CaseSrv.scala +++ b/thehive-backend/app/services/CaseSrv.scala @@ -2,21 +2,24 @@ package services import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Try + +import play.api.{ Configuration, Logger } +import play.api.libs.json.Json.toJsFieldJsValueWrapper +import play.api.libs.json._ + import akka.NotUsed import akka.stream.scaladsl.Source import models._ + import org.elastic4play.InternalError import org.elastic4play.controllers.Fields import org.elastic4play.services._ -import play.api.Logger -import play.api.libs.json.Json.toJsFieldJsValueWrapper -import play.api.libs.json._ - -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.Try @Singleton -class CaseSrv @Inject() ( +class CaseSrv( + maxSimilarCases: Int, caseModel: CaseModel, artifactModel: ArtifactModel, taskModel: TaskModel, @@ -28,7 +31,31 @@ class CaseSrv @Inject() ( findSrv: FindSrv, implicit val ec: ExecutionContext) { - lazy val log = Logger(getClass) + @Inject() def this( + configuration: Configuration, + caseModel: CaseModel, + artifactModel: ArtifactModel, + taskModel: TaskModel, + createSrv: CreateSrv, + artifactSrv: ArtifactSrv, + getSrv: GetSrv, + updateSrv: UpdateSrv, + deleteSrv: DeleteSrv, + findSrv: FindSrv, + ec: ExecutionContext) = this( + configuration.getOptional[Int]("maxSimilarCases").getOrElse(100), + caseModel, + artifactModel, + taskModel, + createSrv, + artifactSrv, + getSrv, + updateSrv, + deleteSrv, + findSrv, + ec) + + private[CaseSrv] lazy val logger = Logger(getClass) def applyTemplate(template: CaseTemplate, originalFields: Fields): Fields = { def getJsObjectOrEmpty(value: Option[JsValue]) = value.fold(JsObject(Nil)) { @@ -118,7 +145,7 @@ class CaseSrv @Inject() ( "status" ~= "Ok"), Some("all"), Nil) ._1 .flatMapConcat { artifact ⇒ artifactSrv.findSimilar(artifact, Some("all"), Nil)._1 } - .groupBy(100, _.parentId) + .groupBy(maxSimilarCases, _.parentId) .map { a ⇒ (a.parentId, Seq(a)) } .reduce((l, r) ⇒ (l._1, r._2 ++ l._2)) .mergeSubstreams diff --git a/thehive-backend/app/services/CaseTemplateSrv.scala b/thehive-backend/app/services/CaseTemplateSrv.scala index 62a08f0fe9..5a3405a155 100644 --- a/thehive-backend/app/services/CaseTemplateSrv.scala +++ b/thehive-backend/app/services/CaseTemplateSrv.scala @@ -7,12 +7,11 @@ import scala.concurrent.{ ExecutionContext, Future } import akka.NotUsed import akka.stream.Materializer import akka.stream.scaladsl.{ Sink, Source } +import models.{ CaseTemplate, CaseTemplateModel } import org.elastic4play.NotFoundError import org.elastic4play.controllers.Fields -import org.elastic4play.services.{ AuthContext, CreateSrv, DeleteSrv, FindSrv, GetSrv, QueryDSL, QueryDef, UpdateSrv } - -import models.{ CaseTemplate, CaseTemplateModel } +import org.elastic4play.services._ @Singleton class CaseTemplateSrv @Inject() ( @@ -28,7 +27,7 @@ class CaseTemplateSrv @Inject() ( def create(fields: Fields)(implicit authContext: AuthContext): Future[CaseTemplate] = createSrv[CaseTemplateModel, CaseTemplate](caseTemplateModel, fields) - def get(id: String)(implicit Context: AuthContext): Future[CaseTemplate] = + def get(id: String): Future[CaseTemplate] = getSrv[CaseTemplateModel, CaseTemplate](caseTemplateModel, id) def getByName(name: String): Future[CaseTemplate] = { diff --git a/thehive-backend/app/services/CustomWSAPI.scala b/thehive-backend/app/services/CustomWSAPI.scala index a35e237c68..64154b285f 100644 --- a/thehive-backend/app/services/CustomWSAPI.scala +++ b/thehive-backend/app/services/CustomWSAPI.scala @@ -1,39 +1,41 @@ package services -import javax.inject.Inject +import javax.inject.{ Inject, Singleton } -import akka.stream.Materializer import play.api.inject.ApplicationLifecycle -import play.api.libs.ws.ahc.{ AhcWSAPI, AhcWSClient, AhcWSClientConfig, AhcWSClientConfigParser } -import play.api.libs.ws.ssl.TrustStoreConfig import play.api.libs.ws._ +import play.api.libs.ws.ahc.{ AhcWSClient, AhcWSClientConfig, AhcWSClientConfigParser } import play.api.{ Configuration, Environment, Logger } +import akka.stream.Materializer +import com.typesafe.sslconfig.ssl.TrustStoreConfig + object CustomWSAPI { private[CustomWSAPI] lazy val logger = Logger(getClass) - def parseWSConfig(config: Configuration, environment: Environment): AhcWSClientConfig = { + def parseWSConfig(config: Configuration): AhcWSClientConfig = { new AhcWSClientConfigParser( - new WSConfigParser(config, environment).parse(), - config, - environment).parse() + new WSConfigParser(config.underlying, getClass.getClassLoader).parse(), + config.underlying, + getClass.getClassLoader).parse() } - def parseProxyConfig(config: Configuration): Option[WSProxyServer] = for { - proxyConfig ← config.getConfig("play.ws.proxy") - proxyHost ← proxyConfig.getString("host") - proxyPort ← proxyConfig.getInt("port") - proxyProtocol = proxyConfig.getString("protocol") - proxyPrincipal = proxyConfig.getString("user") - proxyPassword = proxyConfig.getString("password") - proxyNtlmDomain = proxyConfig.getString("ntlmDomain") - proxyEncoding = proxyConfig.getString("encoding") - proxyNonProxyHosts = proxyConfig.getStringSeq("nonProxyHosts") - } yield DefaultWSProxyServer(proxyHost, proxyPort, proxyProtocol, proxyPrincipal, proxyPassword, proxyNtlmDomain, proxyEncoding, proxyNonProxyHosts) + def parseProxyConfig(config: Configuration): Option[WSProxyServer] = + config.getOptional[Configuration]("play.ws.proxy").map { proxyConfig ⇒ + DefaultWSProxyServer( + proxyConfig.get[String]("host"), + proxyConfig.get[Int]("port"), + proxyConfig.getOptional[String]("protocol"), + proxyConfig.getOptional[String]("user"), + proxyConfig.getOptional[String]("password"), + proxyConfig.getOptional[String]("ntlmDomain"), + proxyConfig.getOptional[String]("encoding"), + proxyConfig.getOptional[Seq[String]]("nonProxyHosts")) + } - def getWS(config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer): AhcWSAPI = { - val clientConfig = parseWSConfig(config, environment) - val clientConfigWithTruststore = config.getString("play.cert") match { + def getWS(config: Configuration)(implicit mat: Materializer): AhcWSClient = { + val clientConfig = parseWSConfig(config) + val clientConfigWithTruststore = config.getOptional[String]("play.cert") match { case Some(p) ⇒ logger.warn( """Use of "cert" parameter in configuration file is deprecated. Please use: @@ -48,36 +50,45 @@ object CustomWSAPI { """.stripMargin) clientConfig.copy( wsClientConfig = clientConfig.wsClientConfig.copy( - ssl = clientConfig.wsClientConfig.ssl.copy( - trustManagerConfig = clientConfig.wsClientConfig.ssl.trustManagerConfig.copy( - trustStoreConfigs = clientConfig.wsClientConfig.ssl.trustManagerConfig.trustStoreConfigs :+ TrustStoreConfig(filePath = Some(p.toString), data = None))))) + ssl = clientConfig.wsClientConfig.ssl.withTrustManagerConfig( + clientConfig.wsClientConfig.ssl.trustManagerConfig.withTrustStoreConfigs( + clientConfig.wsClientConfig.ssl.trustManagerConfig.trustStoreConfigs :+ TrustStoreConfig(filePath = Some(p.toString), data = None))))) case None ⇒ clientConfig } - new AhcWSAPI(environment, clientConfigWithTruststore, lifecycle)(mat) + AhcWSClient(clientConfigWithTruststore, None) } def getConfig(config: Configuration, path: String): Configuration = { Configuration( - config.getConfig(s"play.$path").getOrElse(Configuration.empty).underlying.withFallback( - config.getConfig(path).getOrElse(Configuration.empty).underlying)) + config.getOptional[Configuration](s"play.$path").getOrElse(Configuration.empty).underlying.withFallback( + config.getOptional[Configuration](path).getOrElse(Configuration.empty).underlying)) } } -class CustomWSAPI(ws: AhcWSAPI, val proxy: Option[WSProxyServer], config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer) extends WSAPI { +@Singleton +class CustomWSAPI( + ws: AhcWSClient, + val proxy: Option[WSProxyServer], + config: Configuration, + environment: Environment, + lifecycle: ApplicationLifecycle, + mat: Materializer) extends WSClient { private[CustomWSAPI] lazy val logger = Logger(getClass) @Inject() def this(config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer) = this( - CustomWSAPI.getWS(config, environment, lifecycle, mat), + CustomWSAPI.getWS(config)(mat), CustomWSAPI.parseProxyConfig(config), config, environment, lifecycle, mat) + override def close(): Unit = ws.close() + override def url(url: String): WSRequest = { val req = ws.url(url) proxy.fold(req)(req.withProxyServer) } - override def client: AhcWSClient = ws.client + override def underlying[T]: T = ws.underlying[T] def withConfig(subConfig: Configuration): CustomWSAPI = { logger.debug(s"Override WS configuration using $subConfig") diff --git a/thehive-backend/app/services/FlowSrv.scala b/thehive-backend/app/services/FlowSrv.scala index 635a9f6411..ed4eb7ed84 100644 --- a/thehive-backend/app/services/FlowSrv.scala +++ b/thehive-backend/app/services/FlowSrv.scala @@ -2,28 +2,28 @@ package services import javax.inject.{ Inject, Singleton } -import scala.annotation.implicitNotFound import scala.concurrent.{ ExecutionContext, Future } +import play.api.Logger +import play.api.libs.json.JsValue.jsValueToJsLookup +import play.api.libs.json.{ JsObject, JsValue } + import akka.NotUsed import akka.stream.scaladsl.Source +import models.{ Audit, AuditModel } -import play.api.libs.json.{ JsObject, JsValue } -import play.api.libs.json.JsValue.jsValueToJsLookup - -import org.elastic4play.services.{ AuxSrv, FindSrv, ModelSrv, QueryDSL } +import org.elastic4play.services.{ AuxSrv, FindSrv, ModelSrv } import org.elastic4play.utils.RichJson -import models.{ Audit, AuditModel } -import play.api.Logger - +@Singleton class FlowSrv @Inject() ( auditModel: AuditModel, modelSrv: ModelSrv, auxSrv: AuxSrv, findSrv: FindSrv, implicit val ec: ExecutionContext) { - lazy val log = Logger(getClass) + + private[FlowSrv] lazy val logger = Logger(getClass) def apply(rootId: Option[String], count: Int): (Source[JsObject, NotUsed], Future[Long]) = { import org.elastic4play.services.QueryDSL._ diff --git a/thehive-backend/app/services/KeyAuthSrv.scala b/thehive-backend/app/services/KeyAuthSrv.scala new file mode 100644 index 0000000000..2bcc301aba --- /dev/null +++ b/thehive-backend/app/services/KeyAuthSrv.scala @@ -0,0 +1,59 @@ +package services + +import java.util.Base64 +import javax.inject.{ Inject, Singleton } + +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.Random + +import play.api.libs.json.JsArray +import play.api.mvc.RequestHeader + +import akka.stream.Materializer +import akka.stream.scaladsl.Sink + +import org.elastic4play.controllers.Fields +import org.elastic4play.services.{ AuthCapability, AuthContext, AuthSrv } +import org.elastic4play.{ AuthenticationError, BadRequestError } + +@Singleton +class KeyAuthSrv @Inject() ( + userSrv: UserSrv, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) extends AuthSrv { + override val name = "key" + + protected final def generateKey(): String = { + val bytes = Array.ofDim[Byte](24) + Random.nextBytes(bytes) + Base64.getEncoder.encodeToString(bytes) + } + + override val capabilities = Set(AuthCapability.authByKey) + + override def authenticate(key: String)(implicit request: RequestHeader): Future[AuthContext] = { + import org.elastic4play.services.QueryDSL._ + // key attribute is sensitive so it is not possible to search on that field + userSrv.find("status" ~= "Ok", Some("all"), Nil) + ._1 + .filter(_.key().contains(key)) + .runWith(Sink.headOption) + .flatMap { + case Some(user) ⇒ userSrv.getFromUser(request, user) + case None ⇒ Future.failed(AuthenticationError("Authentication failure")) + } + } + + override def renewKey(username: String)(implicit authContext: AuthContext): Future[String] = { + val newKey = generateKey() + userSrv.update(username, Fields.empty.set("key", newKey)).map(_ ⇒ newKey) + } + + override def getKey(username: String)(implicit authContext: AuthContext): Future[String] = { + userSrv.get(username).map(_.key().getOrElse(throw BadRequestError(s"User $username hasn't key"))) + } + + override def removeKey(username: String)(implicit authContext: AuthContext): Future[Unit] = { + userSrv.update(username, Fields.empty.set("key", JsArray())).map(_ ⇒ ()) + } +} diff --git a/thehive-backend/app/services/LocalAuthSrv.scala b/thehive-backend/app/services/LocalAuthSrv.scala index ad0641d3f3..fe96e383e5 100644 --- a/thehive-backend/app/services/LocalAuthSrv.scala +++ b/thehive-backend/app/services/LocalAuthSrv.scala @@ -2,29 +2,27 @@ package services import javax.inject.{ Inject, Singleton } -import scala.annotation.implicitNotFound import scala.concurrent.{ ExecutionContext, Future } import scala.util.Random -import play.api.libs.json.{ JsObject, JsString } import play.api.mvc.RequestHeader -import org.elastic4play.{ AuthenticationError, AuthorizationError } +import akka.stream.Materializer +import models.User + import org.elastic4play.controllers.Fields -import org.elastic4play.services.{ AuthCapability, AuthContext, AuthSrv, UpdateSrv } +import org.elastic4play.services.{ AuthCapability, AuthContext, AuthSrv } import org.elastic4play.utils.Hasher - -import models.{ User, UserModel } +import org.elastic4play.{ AuthenticationError, AuthorizationError } @Singleton class LocalAuthSrv @Inject() ( - userModel: UserModel, userSrv: UserSrv, - updateSrv: UpdateSrv, - implicit val ec: ExecutionContext) extends AuthSrv { + implicit val ec: ExecutionContext, + implicit val mat: Materializer) extends AuthSrv { val name = "local" - def capabilities = Set(AuthCapability.changePassword, AuthCapability.setPassword) + override val capabilities = Set(AuthCapability.changePassword, AuthCapability.setPassword) private[services] def doAuthenticate(user: User, password: String): Boolean = { user.password().map(_.split(",", 2)).fold(false) { @@ -35,24 +33,23 @@ class LocalAuthSrv @Inject() ( } } - def authenticate(username: String, password: String)(implicit request: RequestHeader): Future[AuthContext] = { + override def authenticate(username: String, password: String)(implicit request: RequestHeader): Future[AuthContext] = { userSrv.get(username).flatMap { user ⇒ if (doAuthenticate(user, password)) userSrv.getFromUser(request, user) else Future.failed(AuthenticationError("Authentication failure")) } } - def changePassword(username: String, oldPassword: String, newPassword: String)(implicit authContext: AuthContext): Future[Unit] = { + override def changePassword(username: String, oldPassword: String, newPassword: String)(implicit authContext: AuthContext): Future[Unit] = { userSrv.get(username).flatMap { user ⇒ if (doAuthenticate(user, oldPassword)) setPassword(username, newPassword) else Future.failed(AuthorizationError("Authentication failure")) } } - def setPassword(username: String, newPassword: String)(implicit authContext: AuthContext): Future[Unit] = { + override def setPassword(username: String, newPassword: String)(implicit authContext: AuthContext): Future[Unit] = { val seed = Random.nextString(10).replace(',', '!') val newHash = seed + "," + Hasher("SHA-256").fromString(seed + newPassword).head.toString - userSrv.update(username, Fields(JsObject(Seq("password" → JsString(newHash))))) - .map(_ ⇒ ()) + userSrv.update(username, Fields.empty.set("password", newHash)).map(_ ⇒ ()) } } \ No newline at end of file diff --git a/thehive-backend/app/services/LogSrv.scala b/thehive-backend/app/services/LogSrv.scala index 2d080edaaf..15a5c5636f 100644 --- a/thehive-backend/app/services/LogSrv.scala +++ b/thehive-backend/app/services/LogSrv.scala @@ -3,12 +3,15 @@ package services import javax.inject.{ Inject, Singleton } import scala.concurrent.{ ExecutionContext, Future } + +import play.api.libs.json.JsObject + import akka.NotUsed import akka.stream.scaladsl.Source -import org.elastic4play.controllers.Fields -import org.elastic4play.services.{ Agg, AuthContext, CreateSrv, DeleteSrv, FindSrv, GetSrv, QueryDef, UpdateSrv } import models.{ Log, LogModel, Task, TaskModel } -import play.api.libs.json.JsObject + +import org.elastic4play.controllers.Fields +import org.elastic4play.services._ @Singleton class LogSrv @Inject() ( @@ -28,7 +31,7 @@ class LogSrv @Inject() ( def create(task: Task, fields: Fields)(implicit authContext: AuthContext): Future[Log] = createSrv[LogModel, Log, Task](logModel, task, fields) - def get(id: String)(implicit Context: AuthContext): Future[Log] = + def get(id: String): Future[Log] = getSrv[LogModel, Log](logModel, id) def update(id: String, fields: Fields)(implicit Context: AuthContext): Future[Log] = diff --git a/thehive-backend/app/services/StreamMessage.scala b/thehive-backend/app/services/StreamMessage.scala index a210e6f1dc..b8dfb7de77 100644 --- a/thehive-backend/app/services/StreamMessage.scala +++ b/thehive-backend/app/services/StreamMessage.scala @@ -1,17 +1,12 @@ package services -import javax.inject.Singleton - -import scala.annotation.elidable -import scala.annotation.elidable.ASSERTION -import scala.annotation.implicitNotFound import scala.concurrent.{ ExecutionContext, Future } -import play.api.libs.json.{ JsObject, Json } +import play.api.Logger import play.api.libs.json.Json.toJsFieldJsValueWrapper +import play.api.libs.json.{ JsObject, Json } import org.elastic4play.services.{ AuditOperation, AuxSrv, MigrationEvent } -import play.api.Logger trait StreamMessageGroup[M] { def :+(message: M): StreamMessageGroup[M] @@ -54,7 +49,7 @@ case class AuditOperationGroup( } object AuditOperationGroup { - lazy val log = Logger(getClass) + private[AuditOperationGroup] lazy val logger = Logger(classOf[AuditOperationGroup]) def apply(auxSrv: AuxSrv, operation: AuditOperation)(implicit ec: ExecutionContext): AuditOperationGroup = { val auditedAttributes = JsObject { @@ -69,7 +64,7 @@ object AuditOperationGroup { val obj = auxSrv(operation.entity, 10, withStats = false, removeUnaudited = true) .recover { case error ⇒ - log.error("auxSrv fails", error) + logger.error("auxSrv fails", error) JsObject(Nil) } new AuditOperationGroup( diff --git a/thehive-backend/app/services/StreamSrv.scala b/thehive-backend/app/services/StreamSrv.scala index 61706ce738..d19f3f9a01 100644 --- a/thehive-backend/app/services/StreamSrv.scala +++ b/thehive-backend/app/services/StreamSrv.scala @@ -2,37 +2,43 @@ package services import javax.inject.{ Inject, Singleton } -import scala.concurrent.{ ExecutionContext, Future } import scala.concurrent.duration.FiniteDuration - -import akka.actor.{ ActorLogging, ActorRef, ActorSystem, Cancellable, DeadLetter, PoisonPill, actorRef2Scala } -import akka.actor.Actor -import akka.actor.ActorDSL.{ Act, actor } -import akka.stream.Materializer +import scala.concurrent.{ ExecutionContext, Future } import play.api.Logger import play.api.libs.json.JsObject import play.api.mvc.{ Filter, RequestHeader, Result } -import org.elastic4play.services.{ AuditOperation, AuxSrv, EndOfMigrationEvent, EventMessage, EventSrv, MigrationEvent } +import akka.actor.{ Actor, ActorLogging, ActorRef, ActorSystem, Cancellable, DeadLetter, PoisonPill, actorRef2Scala } +import akka.stream.Materializer + +import org.elastic4play.services._ import org.elastic4play.utils.Instance /** * This actor monitors dead messages and log them */ @Singleton -class StreamMonitor @Inject() (implicit val system: ActorSystem) { - lazy val logger = Logger(getClass) - val monitorActor: ActorRef = actor(new Act { - become { - case DeadLetter(StreamActor.GetOperations, sender, recipient) ⇒ - logger.warn(s"receive dead GetOperations message, $sender -> $recipient") - sender ! StreamActor.StreamNotFound - case other ⇒ - logger.error(s"receive dead message : $other") - } - }) - system.eventStream.subscribe(monitorActor, classOf[DeadLetter]) +class DeadLetterMonitoringActor @Inject() (system: ActorSystem) extends Actor { + private[DeadLetterMonitoringActor] lazy val logger = Logger(getClass) + + override def preStart(): Unit = { + system.eventStream.subscribe(self, classOf[DeadLetter]) + super.preStart() + } + + override def postStop(): Unit = { + system.eventStream.unsubscribe(self) + super.postStop() + } + + override def receive: Receive = { + case DeadLetter(StreamActor.GetOperations, sender, recipient) ⇒ + logger.warn(s"receive dead GetOperations message, $sender -> $recipient") + sender ! StreamActor.StreamNotFound + case other ⇒ + logger.error(s"receive dead message : $other") + } } object StreamActor { @@ -56,10 +62,10 @@ class StreamActor( globalMaxWait: FiniteDuration, eventSrv: EventSrv, auxSrv: AuxSrv) extends Actor with ActorLogging { - import services.StreamActor._ import context.dispatcher + import services.StreamActor._ - lazy val logger = Logger(getClass) + private[StreamActor] lazy val logger = Logger(getClass) private object FakeCancellable extends Cancellable { def cancel() = true @@ -204,7 +210,7 @@ class StreamFilter @Inject() ( implicit val mat: Materializer, implicit val ec: ExecutionContext) extends Filter { - val log = Logger(getClass) + private[StreamFilter] lazy val logger = Logger(getClass) def apply(nextFilter: RequestHeader ⇒ Future[Result])(requestHeader: RequestHeader): Future[Result] = { val requestId = Instance.getRequestId(requestHeader) eventSrv.publish(StreamActor.Initialize(requestId)) diff --git a/thehive-backend/app/services/TaskSrv.scala b/thehive-backend/app/services/TaskSrv.scala index b22120fd01..79be66344f 100644 --- a/thehive-backend/app/services/TaskSrv.scala +++ b/thehive-backend/app/services/TaskSrv.scala @@ -5,16 +5,15 @@ import javax.inject.{ Inject, Singleton } import scala.concurrent.{ ExecutionContext, Future } import scala.util.Try +import play.api.libs.json.{ JsBoolean, JsObject } + import akka.NotUsed import akka.stream.Materializer import akka.stream.scaladsl.{ Sink, Source } - -import play.api.libs.json.{ JsBoolean, JsObject } +import models._ import org.elastic4play.controllers.Fields -import org.elastic4play.services.{ Agg, AuthContext, CreateSrv, DeleteSrv, FindSrv, GetSrv, QueryDef, UpdateSrv } - -import models.{ Case, CaseModel, Task, TaskModel, TaskStatus } +import org.elastic4play.services._ @Singleton class TaskSrv @Inject() ( diff --git a/thehive-backend/app/services/TheHiveAuthSrv.scala b/thehive-backend/app/services/TheHiveAuthSrv.scala new file mode 100644 index 0000000000..77f33cf868 --- /dev/null +++ b/thehive-backend/app/services/TheHiveAuthSrv.scala @@ -0,0 +1,47 @@ +package services + +import javax.inject.{ Inject, Singleton } + +import scala.collection.immutable +import scala.concurrent.ExecutionContext + +import play.api.{ Configuration, Logger } + +import org.elastic4play.services.AuthSrv +import org.elastic4play.services.auth.MultiAuthSrv + +object TheHiveAuthSrv { + private[TheHiveAuthSrv] lazy val logger = Logger(getClass) + + def getAuthSrv(authTypes: Seq[String], authModules: immutable.Set[AuthSrv]): Seq[AuthSrv] = { + ("key" +: authTypes.filterNot(_ == "key")) + .flatMap { authType ⇒ + authModules.find(_.name == authType) + .orElse { + logger.error(s"Authentication module $authType not found") + None + } + } + } +} + +@Singleton +class TheHiveAuthSrv @Inject() ( + configuration: Configuration, + authModules: immutable.Set[AuthSrv], + userSrv: UserSrv, + override implicit val ec: ExecutionContext) extends MultiAuthSrv( + TheHiveAuthSrv.getAuthSrv( + configuration.getDeprecated[Option[Seq[String]]]("auth.provider", "auth.type").getOrElse(Seq("local")), + authModules), + ec) { + + // Uncomment the following lines if you want to prevent user with key to use password to authenticate + // override def authenticate(username: String, password: String)(implicit request: RequestHeader): Future[AuthContext] = + // userSrv.get(username) + // .transformWith { + // case Success(user) if user.key().isDefined ⇒ Future.failed(AuthenticationError("Authentication by password is not permitted for user with key")) + // case _: Success[_] ⇒ super.authenticate(username, password) + // case _: Failure[_] ⇒ Future.failed(AuthenticationError("Authentication failure")) + // } +} \ No newline at end of file diff --git a/thehive-backend/app/services/UserSrv.scala b/thehive-backend/app/services/UserSrv.scala index ec759bcb37..d9da66539c 100644 --- a/thehive-backend/app/services/UserSrv.scala +++ b/thehive-backend/app/services/UserSrv.scala @@ -2,17 +2,19 @@ package services import javax.inject.{ Inject, Provider, Singleton } +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.mvc.RequestHeader + import akka.NotUsed import akka.stream.scaladsl.Source -import models.{ User, UserModel, UserStatus } +import models.{ Roles, User, UserModel, UserStatus } + import org.elastic4play.controllers.Fields import org.elastic4play.database.DBIndex import org.elastic4play.services._ import org.elastic4play.utils.Instance import org.elastic4play.{ AuthenticationError, AuthorizationError } -import play.api.mvc.RequestHeader - -import scala.concurrent.{ ExecutionContext, Future } @Singleton class UserSrv @Inject() ( @@ -27,7 +29,7 @@ class UserSrv @Inject() ( dbIndex: DBIndex, implicit val ec: ExecutionContext) extends org.elastic4play.services.UserSrv { - private case class AuthContextImpl(userId: String, userName: String, requestId: String, roles: Seq[Role.Type]) extends AuthContext + private case class AuthContextImpl(userId: String, userName: String, requestId: String, roles: Seq[Role]) extends AuthContext override def getFromId(request: RequestHeader, userId: String): Future[AuthContext] = { getSrv[UserModel, User](userModel, userId) @@ -44,12 +46,12 @@ class UserSrv @Inject() ( override def getInitialUser(request: RequestHeader): Future[AuthContext] = dbIndex.getSize(userModel.name).map { - case size if size > 0 ⇒ throw AuthenticationError(s"Not authenticated") - case _ ⇒ AuthContextImpl("init", "", Instance.getRequestId(request), Seq(Role.admin, Role.read)) + case size if size > 0 ⇒ throw AuthenticationError(s"Use of initial user is forbidden because users exist in database") + case _ ⇒ AuthContextImpl("init", "", Instance.getRequestId(request), Seq(Roles.admin, Roles.read, Roles.alert)) } override def inInitAuthContext[A](block: AuthContext ⇒ Future[A]): Future[A] = { - val authContext = AuthContextImpl("init", "", Instance.getInternalId, Seq(Role.admin, Role.read)) + val authContext = AuthContextImpl("init", "", Instance.getInternalId, Seq(Roles.admin, Roles.read, Roles.alert)) eventSrv.publish(StreamActor.Initialize(authContext.requestId)) block(authContext).andThen { case _ ⇒ eventSrv.publish(StreamActor.Commit(authContext.requestId)) @@ -71,6 +73,10 @@ class UserSrv @Inject() ( updateSrv[UserModel, User](userModel, id, fields) } + def update(user: User, fields: Fields)(implicit Context: AuthContext): Future[User] = { + updateSrv(user, fields) + } + def delete(id: String)(implicit Context: AuthContext): Future[User] = deleteSrv[UserModel, User](userModel, id) diff --git a/thehive-backend/app/services/WebHook.scala b/thehive-backend/app/services/WebHook.scala new file mode 100644 index 0000000000..a96d2726f6 --- /dev/null +++ b/thehive-backend/app/services/WebHook.scala @@ -0,0 +1,58 @@ +package services + +import java.net.ConnectException +import javax.inject.Inject + +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.{ Failure, Success, Try } + +import play.api.{ Configuration, Logger } +import play.api.libs.json.JsObject +import play.api.libs.ws.WSRequest + +import org.elastic4play.services.AuxSrv + +case class WebHook(name: String, ws: WSRequest)(implicit ec: ExecutionContext) { + private[WebHook] lazy val logger = Logger(getClass.getName + "." + name) + + def send(obj: JsObject): Unit = ws.post(obj).onComplete { + case Success(resp) if resp.status / 100 != 2 ⇒ logger.error(s"WebHook returns status ${resp.status} ${resp.statusText}") + case Failure(_: ConnectException) ⇒ logger.error(s"Connection to WebHook $name error") + case Failure(error) ⇒ logger.error("WebHook call error", error) + case _ ⇒ + } +} + +class WebHooks( + webhooks: Seq[WebHook], + auxSrv: AuxSrv, + implicit val ec: ExecutionContext) { + @Inject() def this( + configuration: Configuration, + globalWS: CustomWSAPI, + auxSrv: AuxSrv, + ec: ExecutionContext) = { + this( + for { + cfg ← configuration.getOptional[Configuration]("webhooks").toSeq + whWS = globalWS.withConfig(cfg) + name ← cfg.subKeys + whConfig ← Try(cfg.get[Configuration](name)).toOption + url ← whConfig.getOptional[String]("url") + instanceWS = whWS.withConfig(whConfig).url(url) + } yield WebHook(name, instanceWS)(ec), + auxSrv, + ec) + } + + def send(obj: JsObject): Unit = { + (for { + objectType ← (obj \ "objectType").asOpt[String] + objectId ← (obj \ "objectId").asOpt[String] + } yield auxSrv(objectType, objectId, nparent = 0, withStats = false, removeUnaudited = false)) + .getOrElse(Future.successful(JsObject(Nil))) + .map(o ⇒ obj + ("object" → o)) + .fallbackTo(Future.successful(obj)) + .map(o ⇒ webhooks.foreach(_.send(o))) + } +} diff --git a/thehive-backend/build.sbt b/thehive-backend/build.sbt index 88036d9d43..a2fd132475 100644 --- a/thehive-backend/build.sbt +++ b/thehive-backend/build.sbt @@ -3,11 +3,13 @@ import Dependencies._ libraryDependencies ++= Seq( Library.Play.cache, Library.Play.ws, + Library.Play.ahc, Library.Play.filters, + Library.Play.guice, Library.scalaGuice, Library.elastic4play, Library.zip4j, - "org.reflections" % "reflections" % "0.9.10" + Library.reflections ) -enablePlugins(PlayScala) +play.sbt.routes.RoutesKeys.routesImport -= "controllers.Assets.Asset" \ No newline at end of file diff --git a/thehive-backend/conf/reference.conf b/thehive-backend/conf/reference.conf index a31791c084..f316ca5782 100644 --- a/thehive-backend/conf/reference.conf +++ b/thehive-backend/conf/reference.conf @@ -56,7 +56,7 @@ auth { # services.LocalAuthSrv : passwords are stored in user entity (in ElasticSearch). No configuration are required. # ad : use ActiveDirectory to authenticate users. Configuration is under "auth.ad" key # ldap : use LDAP to authenticate users. Configuration is under "auth.ldap" key - type = [local] + provider = [local] ad { # Domain Windows name using DNS format. This parameter is required. diff --git a/thehive-backend/conf/routes b/thehive-backend/conf/routes index 884d0dc7da..7ce66025ca 100644 --- a/thehive-backend/conf/routes +++ b/thehive-backend/conf/routes @@ -91,6 +91,10 @@ DELETE /api/user/:userId controllers.UserCtrl.delete(us PATCH /api/user/:userId controllers.UserCtrl.update(userId) POST /api/user/:userId/password/set controllers.UserCtrl.setPassword(userId) POST /api/user/:userId/password/change controllers.UserCtrl.changePassword(userId) +GET /api/user/:userId/key controllers.UserCtrl.getKey(userId) +DELETE /api/user/:userId/key controllers.UserCtrl.removeKey(userId) +POST /api/user/:userId/key/renew controllers.UserCtrl.renewKey(userId) + POST /api/stream controllers.StreamCtrl.create() GET /api/stream/status controllers.StreamCtrl.status diff --git a/thehive-cortex/app/connectors/cortex/CortexConnector.scala b/thehive-cortex/app/connectors/cortex/CortexConnector.scala index a1b0385578..9bf2cb6691 100644 --- a/thehive-cortex/app/connectors/cortex/CortexConnector.scala +++ b/thehive-cortex/app/connectors/cortex/CortexConnector.scala @@ -1,21 +1,24 @@ package connectors.cortex +import play.api.libs.concurrent.AkkaGuiceSupport import play.api.{ Configuration, Environment, Logger } import connectors.ConnectorModule -import connectors.cortex.controllers.CortextCtrl +import connectors.cortex.controllers.CortexCtrl +import connectors.cortex.services.JobReplicateActor class CortexConnector( environment: Environment, - configuration: Configuration) extends ConnectorModule { - val log = Logger(getClass) + configuration: Configuration) extends ConnectorModule with AkkaGuiceSupport { + private[CortexConnector] lazy val logger = Logger(getClass) def configure() { try { - registerController[CortextCtrl] + registerController[CortexCtrl] + bindActor[JobReplicateActor]("JobReplicateActor") } catch { - case t: Throwable ⇒ log.error("Corte connector is disabled because its configuration is invalid", t) + case t: Throwable ⇒ logger.error("Cortex connector is disabled because its configuration is invalid", t) } } } diff --git a/thehive-cortex/app/connectors/cortex/controllers/CortextCtrl.scala b/thehive-cortex/app/connectors/cortex/controllers/CortexCtrl.scala similarity index 80% rename from thehive-cortex/app/connectors/cortex/controllers/CortextCtrl.scala rename to thehive-cortex/app/connectors/cortex/controllers/CortexCtrl.scala index 5afb1708fb..a11bb6e7a7 100644 --- a/thehive-cortex/app/connectors/cortex/controllers/CortextCtrl.scala +++ b/thehive-cortex/app/connectors/cortex/controllers/CortexCtrl.scala @@ -3,23 +3,26 @@ package connectors.cortex.controllers import javax.inject.{ Inject, Singleton } import scala.concurrent.ExecutionContext + import play.api.Logger import play.api.http.Status import play.api.libs.json.{ JsObject, Json } -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc._ import play.api.routing.SimpleRouter import play.api.routing.sird.{ DELETE, GET, PATCH, POST, UrlContext } + import org.elastic4play.{ BadRequestError, NotFoundError, Timed } import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites -import org.elastic4play.services.{ AuxSrv, QueryDSL, QueryDef, Role } +import org.elastic4play.services.{ AuxSrv, QueryDSL, QueryDef } import org.elastic4play.services.JsonFormat.queryReads import connectors.Connector -import connectors.cortex.models.JsonFormat.{ analyzerFormats, cortexJobFormat } +import connectors.cortex.models.JsonFormat.analyzerFormats import connectors.cortex.services.{ CortexConfig, CortexSrv } +import models.Roles @Singleton -class CortextCtrl @Inject() ( +class CortexCtrl @Inject() ( reportTemplateCtrl: ReportTemplateCtrl, cortexConfig: CortexConfig, cortexSrv: CortexSrv, @@ -27,10 +30,14 @@ class CortextCtrl @Inject() ( authenticated: Authenticated, fieldsBodyParser: FieldsBodyParser, renderer: Renderer, - implicit val ec: ExecutionContext) extends Controller with Connector with Status { + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) with Connector with Status { + val name = "cortex" - val log = Logger(getClass) + private[CortexCtrl] lazy val logger = Logger(getClass) + override val status: JsObject = Json.obj("enabled" → true, "servers" → cortexConfig.instances.map(_.name)) + val router = SimpleRouter { case POST(p"/job") ⇒ createJob case GET(p"/job/$jobId<[^/]*>") ⇒ getJob(jobId) @@ -49,7 +56,7 @@ class CortextCtrl @Inject() ( } @Timed - def createJob: Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def createJob: Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ val analyzerId = request.body.getString("analyzerId").getOrElse(throw BadRequestError(s"analyzerId is missing")) val artifactId = request.body.getString("artifactId").getOrElse(throw BadRequestError(s"artifactId is missing")) val cortexId = request.body.getString("cortexId") @@ -59,14 +66,14 @@ class CortextCtrl @Inject() ( } @Timed - def getJob(jobId: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def getJob(jobId: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ cortexSrv.getJob(jobId).map { job ⇒ renderer.toOutput(OK, job) } } @Timed - def findJob: Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def findJob: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -77,21 +84,21 @@ class CortextCtrl @Inject() ( } @Timed - def getAnalyzer(analyzerId: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def getAnalyzer(analyzerId: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ cortexSrv.getAnalyzer(analyzerId).map { analyzer ⇒ renderer.toOutput(OK, analyzer) } } @Timed - def getAnalyzerFor(dataType: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def getAnalyzerFor(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ cortexSrv.getAnalyzersFor(dataType).map { analyzers ⇒ renderer.toOutput(OK, analyzers) } } @Timed - def listAnalyzer: Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def listAnalyzer: Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ cortexSrv.listAnalyzer.map { analyzers ⇒ renderer.toOutput(OK, analyzers) } diff --git a/thehive-cortex/app/connectors/cortex/controllers/ReportTemplateCtrl.scala b/thehive-cortex/app/connectors/cortex/controllers/ReportTemplateCtrl.scala index 144968d4b0..c669933998 100644 --- a/thehive-cortex/app/connectors/cortex/controllers/ReportTemplateCtrl.scala +++ b/thehive-cortex/app/connectors/cortex/controllers/ReportTemplateCtrl.scala @@ -2,23 +2,26 @@ package connectors.cortex.controllers import javax.inject.{ Inject, Singleton } -import scala.collection.JavaConversions.asScalaBuffer +import scala.collection.JavaConverters._ import scala.concurrent.{ ExecutionContext, Future } import scala.io.Source import scala.util.control.NonFatal + import akka.stream.Materializer import akka.stream.scaladsl.Sink import play.api.Logger import play.api.http.Status import play.api.libs.json.{ JsBoolean, JsObject } -import play.api.mvc.{ Action, AnyContent, Controller } +import play.api.mvc._ + import org.elastic4play.{ BadRequestError, Timed } -import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, FileInputValue, Renderer } +import org.elastic4play.controllers._ import org.elastic4play.models.JsonFormat.baseModelEntityWrites -import org.elastic4play.services.{ QueryDSL, QueryDef, Role } +import org.elastic4play.services.{ QueryDSL, QueryDef } import org.elastic4play.services.AuxSrv import org.elastic4play.services.JsonFormat.queryReads import connectors.cortex.services.ReportTemplateSrv +import models.Roles import net.lingala.zip4j.core.ZipFile import net.lingala.zip4j.model.FileHeader @@ -29,25 +32,26 @@ class ReportTemplateCtrl @Inject() ( authenticated: Authenticated, renderer: Renderer, fieldsBodyParser: FieldsBodyParser, + components: ControllerComponents, implicit val ec: ExecutionContext, - implicit val mat: Materializer) extends Controller with Status { + implicit val mat: Materializer) extends AbstractController(components) with Status { - lazy val logger = Logger(getClass) + private[ReportTemplateCtrl] lazy val logger = Logger(getClass) @Timed - def create: Action[Fields] = authenticated(Role.admin).async(fieldsBodyParser) { implicit request ⇒ + def create: Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ reportTemplateSrv.create(request.body) .map(reportTemplate ⇒ renderer.toOutput(CREATED, reportTemplate)) } @Timed - def get(id: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def get(id: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ reportTemplateSrv.get(id) .map(reportTemplate ⇒ renderer.toOutput(OK, reportTemplate)) } @Timed - def getContent(analyzerId: String, reportType: String): Action[AnyContent] = authenticated(Role.read).async { implicit request ⇒ + def getContent(analyzerId: String, reportType: String): Action[AnyContent] = authenticated(Roles.read).async { implicit request ⇒ import org.elastic4play.services.QueryDSL._ val (reportTemplates, total) = reportTemplateSrv.find(and("analyzerId" ~= analyzerId, "reportType" ~= reportType), Some("0-1"), Nil) total.foreach { t ⇒ @@ -62,19 +66,19 @@ class ReportTemplateCtrl @Inject() ( } @Timed - def update(id: String): Action[Fields] = authenticated(Role.admin).async(fieldsBodyParser) { implicit request ⇒ + def update(id: String): Action[Fields] = authenticated(Roles.admin).async(fieldsBodyParser) { implicit request ⇒ reportTemplateSrv.update(id, request.body) .map(reportTemplate ⇒ renderer.toOutput(OK, reportTemplate)) } @Timed - def delete(id: String): Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ + def delete(id: String): Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ reportTemplateSrv.delete(id) .map(_ ⇒ NoContent) } @Timed - def find: Action[Fields] = authenticated(Role.read).async(fieldsBodyParser) { implicit request ⇒ + def find: Action[Fields] = authenticated(Roles.read).async(fieldsBodyParser) { implicit request ⇒ val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef]) val range = request.body.getString("range") val sort = request.body.getStrings("sort").getOrElse(Nil) @@ -87,12 +91,12 @@ class ReportTemplateCtrl @Inject() ( } @Timed - def importTemplatePackage: Action[Fields] = authenticated(Role.write).async(fieldsBodyParser) { implicit request ⇒ + def importTemplatePackage: Action[Fields] = authenticated(Roles.write).async(fieldsBodyParser) { implicit request ⇒ val zipFile = request.body.get("templates") match { - case Some(FileInputValue(name, filepath, contentType)) ⇒ new ZipFile(filepath.toFile) - case _ ⇒ throw BadRequestError("") + case Some(FileInputValue(_, filepath, _)) ⇒ new ZipFile(filepath.toFile) + case _ ⇒ throw BadRequestError("") } - val importedReportTemplates: Seq[Future[(String, JsBoolean)]] = zipFile.getFileHeaders.toSeq.filter(_ != null).collect { + val importedReportTemplates: Seq[Future[(String, JsBoolean)]] = zipFile.getFileHeaders.asScala.filter(_ != null).collect { case fileHeader: FileHeader if !fileHeader.isDirectory ⇒ val Array(analyzerId, reportTypeHtml, _*) = (fileHeader.getFileName + "/").split("/", 3) val inputStream = zipFile.getInputStream(fileHeader) diff --git a/thehive-cortex/app/connectors/cortex/models/ReportTemplate.scala b/thehive-cortex/app/connectors/cortex/models/ReportTemplate.scala index b2cdf1b170..b2422a86f8 100644 --- a/thehive-cortex/app/connectors/cortex/models/ReportTemplate.scala +++ b/thehive-cortex/app/connectors/cortex/models/ReportTemplate.scala @@ -5,7 +5,6 @@ import javax.inject.{ Inject, Singleton } import play.api.libs.json.JsObject import org.elastic4play.models.{ AttributeDef, AttributeFormat ⇒ F, AttributeOption ⇒ O, EntityDef, ModelDef } -import org.elastic4play.BadRequestError import org.elastic4play.models.BaseEntity import play.api.libs.json.JsString import scala.concurrent.Future diff --git a/thehive-cortex/app/connectors/cortex/services/CortexClient.scala b/thehive-cortex/app/connectors/cortex/services/CortexClient.scala index a3ca683cc7..9a74b8943f 100644 --- a/thehive-cortex/app/connectors/cortex/services/CortexClient.scala +++ b/thehive-cortex/app/connectors/cortex/services/CortexClient.scala @@ -6,6 +6,7 @@ import connectors.cortex.models.{ Analyzer, CortexArtifact, DataArtifact, FileAr import play.api.Logger import play.api.libs.json.{ JsObject, JsValue, Json } import play.api.libs.ws.{ WSAuthScheme, WSRequest, WSResponse } +import play.api.libs.ws.WSBodyWritables.writeableOf_JsValue import play.api.mvc.MultipartFormData.{ DataPart, FilePart } import services.CustomWSAPI @@ -19,7 +20,7 @@ class CortexClient(val name: String, baseUrl: String, key: String, authenticatio logger.info(s"new Cortex($name, $baseUrl, $key) Basic Auth enabled: ${authentication.isDefined}") def request[A](uri: String, f: WSRequest ⇒ Future[WSResponse], t: WSResponse ⇒ A)(implicit ec: ExecutionContext): Future[A] = { - val requestBuilder = ws.url(s"$baseUrl/$uri").withHeaders("auth" → key) + val requestBuilder = ws.url(s"$baseUrl/$uri").withHttpHeaders("auth" → key) val authenticatedRequestBuilder = authentication.fold(requestBuilder) { case (username, password) ⇒ requestBuilder.withAuth(username, password, WSAuthScheme.BASIC) } @@ -70,6 +71,6 @@ class CortexClient(val name: String, baseUrl: String, key: String, authenticatio } def waitReport(jobId: String, atMost: Duration)(implicit ec: ExecutionContext): Future[JsObject] = { - request(s"api/job/$jobId/waitreport", _.withQueryString("atMost" → atMost.toString).get, r ⇒ r.json.as[JsObject]) + request(s"api/job/$jobId/waitreport", _.withQueryStringParameters("atMost" → atMost.toString).get, r ⇒ r.json.as[JsObject]) } } diff --git a/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala b/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala index 96b2e0e7e1..4d25208792 100644 --- a/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala +++ b/thehive-cortex/app/connectors/cortex/services/CortexSrv.scala @@ -4,13 +4,13 @@ import java.util.Date import javax.inject.{ Inject, Singleton } import akka.NotUsed -import akka.actor.ActorDSL.{ Act, actor } -import akka.actor.ActorSystem +import akka.actor.Actor import akka.stream.Materializer import akka.stream.scaladsl.{ Sink, Source } import connectors.cortex.models.JsonFormat._ import connectors.cortex.models._ import models.Artifact + import org.elastic4play.controllers.Fields import org.elastic4play.services._ import org.elastic4play.services.JsonFormat.attachmentFormat @@ -18,33 +18,32 @@ import org.elastic4play.{ InternalError, NotFoundError } import play.api.libs.json.{ JsObject, Json } import play.api.libs.ws.WSClient import play.api.{ Configuration, Logger } -import services.{ ArtifactSrv, CustomWSAPI, MergeArtifact } +import services.{ ArtifactSrv, CustomWSAPI, MergeArtifact } import scala.concurrent.duration.DurationInt import scala.concurrent.{ ExecutionContext, Future } -import scala.language.implicitConversions import scala.util.{ Failure, Success, Try } object CortexConfig { def getCortexClient(name: String, configuration: Configuration, ws: CustomWSAPI): Option[CortexClient] = { - val url = configuration.getString("url").getOrElse(sys.error("url is missing")).replaceFirst("/*$", "") + val url = configuration.getOptional[String]("url").getOrElse(sys.error("url is missing")).replaceFirst("/*$", "") val key = "" // configuration.getString("key").getOrElse(sys.error("key is missing")) val authentication = for { - basicEnabled ← configuration.getBoolean("basicAuth") + basicEnabled ← configuration.getOptional[Boolean]("basicAuth") if basicEnabled - username ← configuration.getString("username") - password ← configuration.getString("password") + username ← configuration.getOptional[String]("username") + password ← configuration.getOptional[String]("password") } yield username → password Some(new CortexClient(name, url, key, authentication, ws)) } def getInstances(configuration: Configuration, globalWS: CustomWSAPI): Seq[CortexClient] = { for { - cfg ← configuration.getConfig("cortex").toSeq + cfg ← configuration.getOptional[Configuration]("cortex").toSeq cortexWS = globalWS.withConfig(cfg) key ← cfg.subKeys if key != "ws" - c ← cfg.getConfig(key) + c ← cfg.getOptional[Configuration](key) instanceWS = cortexWS.withConfig(c) cic ← getCortexClient(key, c, instanceWS) } yield cic @@ -59,6 +58,34 @@ case class CortexConfig(instances: Seq[CortexClient]) { CortexConfig.getInstances(configuration, globalWS)) } +@Singleton +class JobReplicateActor @Inject() ( + cortexSrv: CortexSrv, + eventSrv: EventSrv, + implicit val mat: Materializer) extends Actor { + + override def preStart(): Unit = { + eventSrv.subscribe(self, classOf[MergeArtifact]) + super.preStart() + } + + override def postStop(): Unit = { + eventSrv.unsubscribe(self) + super.postStop() + } + + override def receive: Receive = { + case MergeArtifact(newArtifact, artifacts, authContext) ⇒ + import org.elastic4play.services.QueryDSL._ + cortexSrv.find(and(parent("case_artifact", withId(artifacts.map(_.id): _*)), "status" ~= JobStatus.Success), Some("all"), Nil)._1 + .mapAsyncUnordered(5) { job ⇒ + val baseFields = Fields(job.attributes - "_id" - "_routing" - "_parent" - "_type" - "createdBy" - "createdAt" - "updatedBy" - "updatedAt" - "user") + cortexSrv.create(newArtifact, baseFields)(authContext) + } + .runWith(Sink.ignore) + } +} + @Singleton class CortexSrv @Inject() ( cortexConfig: CortexConfig, @@ -70,10 +97,8 @@ class CortexSrv @Inject() ( updateSrv: UpdateSrv, findSrv: FindSrv, userSrv: UserSrv, - eventSrv: EventSrv, implicit val ws: WSClient, implicit val ec: ExecutionContext, - implicit val system: ActorSystem, implicit val mat: Materializer) { private[CortexSrv] lazy val logger = Logger(getClass) @@ -99,22 +124,7 @@ class CortexSrv @Inject() ( } } - private[CortexSrv] val mergeActor = actor(new Act { - become { - case MergeArtifact(newArtifact, artifacts, authContext) ⇒ - import org.elastic4play.services.QueryDSL._ - find(and(parent("case_artifact", withId(artifacts.map(_.id): _*)), "status" ~= JobStatus.Success), Some("all"), Nil)._1 - .mapAsyncUnordered(5) { job ⇒ - val baseFields = Fields(job.attributes - "_id" - "_routing" - "_parent" - "_type" - "createdBy" - "createdAt" - "updatedBy" - "updatedAt" - "user") - create(newArtifact, baseFields)(authContext) - } - .runWith(Sink.ignore) - } - }) - - eventSrv.subscribe(mergeActor, classOf[MergeArtifact]) // need to unsubsribe ? - - private[CortexSrv] def create(artifact: Artifact, fields: Fields)(implicit authContext: AuthContext): Future[Job] = { + private[services] def create(artifact: Artifact, fields: Fields)(implicit authContext: AuthContext): Future[Job] = { createSrv[JobModel, Job, Artifact](jobModel, artifact, fields.set("artifactId", artifact.id)) } diff --git a/thehive-cortex/app/connectors/cortex/services/ReportTemplateSrv.scala b/thehive-cortex/app/connectors/cortex/services/ReportTemplateSrv.scala index 16886a03c9..ee056b3cf0 100644 --- a/thehive-cortex/app/connectors/cortex/services/ReportTemplateSrv.scala +++ b/thehive-cortex/app/connectors/cortex/services/ReportTemplateSrv.scala @@ -28,7 +28,7 @@ class ReportTemplateSrv @Inject() ( findSrv: FindSrv, implicit val ec: ExecutionContext) { - lazy val log = Logger(getClass) + private[ReportTemplateSrv] lazy val logger = Logger(getClass) def create(fields: Fields)(implicit authContext: AuthContext): Future[ReportTemplate] = { createSrv[ReportTemplateModel, ReportTemplate](reportTemplateModel, fields) diff --git a/thehive-cortex/build.sbt b/thehive-cortex/build.sbt index d0bc335d4f..e6dc20cc15 100644 --- a/thehive-cortex/build.sbt +++ b/thehive-cortex/build.sbt @@ -2,8 +2,9 @@ import Dependencies._ libraryDependencies ++= Seq( Library.Play.ws, + Library.Play.guice, + Library.Play.ahc, Library.elastic4play, Library.zip4j ) -enablePlugins(PlayScala) diff --git a/thehive-metrics/app/connectors/metrics/Influxdb.scala b/thehive-metrics/app/connectors/metrics/Influxdb.scala index acf98c6015..ad57ed0426 100644 --- a/thehive-metrics/app/connectors/metrics/Influxdb.scala +++ b/thehive-metrics/app/connectors/metrics/Influxdb.scala @@ -1,11 +1,10 @@ package connectors.metrics import java.util -import java.util.SortedMap import java.util.concurrent.TimeUnit import javax.inject.{ Inject, Singleton } -import scala.collection.JavaConversions.{ iterableAsScalaIterable, mapAsScalaMap } +import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext import play.api.Logger import play.api.libs.ws.WSClient @@ -52,21 +51,21 @@ trait InfluxDBAPI { class InfluxDBFactory @Inject() ( ws: WSClient, implicit val ec: ExecutionContext) { - val log = Logger("InfluxDB") + private[InfluxDBFactory] lazy val logger = Logger(classOf[InfluxDB]) case class InfluxDB(url: String, user: String, password: String, database: String, retentionPolicy: String) extends InfluxDBAPI { def send(points: InfluxPoint*): Unit = { - val x = ws + ws .url(url.stripSuffix("/") + "/write") - .withQueryString( + .withQueryStringParameters( "u" → user, "p" → password, "db" → database, "rp" → retentionPolicy) - .withHeaders("Content-Type" → "text/plain") + .withHttpHeaders("Content-Type" → "text/plain") .post(points.map(_.lineProtocol).mkString("\n")) .map { response ⇒ if ((response.status / 100) != 2) - log.warn(s"Send metrics to InfluxDB error : ${response.body}") + logger.warn(s"Send metrics to InfluxDB error : ${response.body}") } () } @@ -90,11 +89,11 @@ class InfluxDBReporter( val now = System.currentTimeMillis() * 1000000 - val points = gauges.map { case (name, gauge) ⇒ pointGauge(now, name, gauge) } ++ - counters.map { case (name, counter) ⇒ pointCounter(now, tags, name, counter) } ++ - histograms.map { case (name, histogram) ⇒ pointHistogram(now, tags, name, histogram) } ++ - meters.map { case (name, meter) ⇒ pointMeter(now, tags, name, meter) } ++ - timers.map { case (name, timer) ⇒ pointTimer(now, tags, name, timer) } + val points = gauges.asScala.map { case (name, gauge) ⇒ pointGauge(now, name, gauge) } ++ + counters.asScala.map { case (name, counter) ⇒ pointCounter(now, tags, name, counter) } ++ + histograms.asScala.map { case (name, histogram) ⇒ pointHistogram(now, tags, name, histogram) } ++ + meters.asScala.map { case (name, meter) ⇒ pointMeter(now, tags, name, meter) } ++ + timers.asScala.map { case (name, timer) ⇒ pointTimer(now, tags, name, timer) } influxdb.send(points.toSeq: _*) } @@ -108,7 +107,7 @@ class InfluxDBReporter( def pointGauge(now: Long, name: String, gauge: Gauge[_]): InfluxPoint = { val value = gauge.getValue match { case s: String ⇒ InfluxString(s) - case i: java.lang.Iterable[_] ⇒ InfluxString(i.mkString(",")) + case i: java.lang.Iterable[_] ⇒ InfluxString(i.asScala.mkString(",")) case d: Double ⇒ InfluxFloat(d) case f: Float ⇒ InfluxFloat(f.toDouble) case l: Long ⇒ InfluxLong(l) diff --git a/thehive-metrics/app/connectors/metrics/MetricsCtrl.scala b/thehive-metrics/app/connectors/metrics/MetricsCtrl.scala index e7448e8b09..1fd0ac365d 100644 --- a/thehive-metrics/app/connectors/metrics/MetricsCtrl.scala +++ b/thehive-metrics/app/connectors/metrics/MetricsCtrl.scala @@ -1,19 +1,20 @@ package connectors.metrics import java.io.StringWriter - import javax.inject.{ Inject, Singleton } -import play.api.mvc.{ Action, Controller } +import play.api.mvc.{ AbstractController, ControllerComponents, DefaultActionBuilder } import play.api.routing.SimpleRouter import play.api.routing.sird.{ GET, UrlContext } import org.elastic4play.Timed - import connectors.Connector @Singleton -class MetricsCtrl @Inject() (metricsModule: Metrics) extends Controller with Connector { +class MetricsCtrl @Inject() ( + metricsModule: Metrics, + actionBuilder: DefaultActionBuilder, + components: ControllerComponents) extends AbstractController(components) with Connector { val name = "metrics" @@ -22,7 +23,7 @@ class MetricsCtrl @Inject() (metricsModule: Metrics) extends Controller with Con } @Timed("controllers.MetricsCtrl.stats") - def stats = Action { + def stats = actionBuilder { val writer = metricsModule.mapper.writerWithDefaultPrettyPrinter val stringWriter = new StringWriter() writer.writeValue(stringWriter, metricsModule.registry) diff --git a/thehive-metrics/app/connectors/metrics/MetricsModule.scala b/thehive-metrics/app/connectors/metrics/MetricsModule.scala index a1609e0028..82ac8b0314 100644 --- a/thehive-metrics/app/connectors/metrics/MetricsModule.scala +++ b/thehive-metrics/app/connectors/metrics/MetricsModule.scala @@ -3,15 +3,14 @@ package connectors.metrics import java.util.concurrent.TimeUnit import javax.inject.{ Inject, Provider, Singleton } -import akka.util.ByteString - -import scala.concurrent.Future +import scala.concurrent.{ ExecutionContext, Future } import scala.concurrent.duration.{ DurationLong, FiniteDuration } import scala.language.implicitConversions + import play.api.{ Configuration, Environment, Logger } import play.api.inject.ApplicationLifecycle -import play.api.libs.concurrent.Execution.Implicits.defaultContext import play.api.mvc._ + import org.aopalliance.intercept.{ MethodInterceptor, MethodInvocation } import net.codingwell.scalaguice.ScalaMultibinder import com.codahale.metrics._ @@ -21,26 +20,26 @@ import com.codahale.metrics.jvm.{ GarbageCollectorMetricSet, MemoryUsageGaugeSet import com.codahale.metrics.logback.InstrumentedAppender import com.fasterxml.jackson.databind.ObjectMapper import com.google.inject.matcher.Matchers + import org.elastic4play.Timed import ch.qos.logback.classic import connectors.ConnectorModule import info.ganglia.gmetric4j.gmetric.GMetric -import play.api.libs.streams.Accumulator trait UnitConverter { - val validUnits = Some(TimeUnit.values.map(_.toString).toSet) + val validUnits: Set[Option[String]] = TimeUnit.values.map(v ⇒ Some(v.toString)).toSet + None implicit def stringToTimeUnit(s: String): TimeUnit = TimeUnit.valueOf(s) } case class GraphiteMetricConfig(host: String, port: Int, prefix: String, rateUnit: TimeUnit, durationUnit: TimeUnit, interval: FiniteDuration) object GraphiteMetricConfig extends UnitConverter { def apply(configuration: Configuration): GraphiteMetricConfig = { - val host = configuration.getString("metrics.graphite.host").getOrElse("127.0.0.1") - val port = configuration.getInt("metrics.graphite.port").getOrElse(2003) - val prefix = configuration.getString("metrics.graphite.prefix").getOrElse("thehive") - val rateUnit = configuration.getString("metrics.graphite.rateUnit", validUnits).getOrElse("SECONDS") - val durationUnit = configuration.getString("metrics.graphite.durationUnit", validUnits).getOrElse("MILLISECONDS") - val interval = configuration.getMilliseconds("metrics.graphite.period").getOrElse(10000L).millis // 10 seconds + val host = configuration.getOptional[String]("metrics.graphite.host").getOrElse("127.0.0.1") + val port = configuration.getOptional[Int]("metrics.graphite.port").getOrElse(2003) + val prefix = configuration.getOptional[String]("metrics.graphite.prefix").getOrElse("thehive") + val rateUnit = configuration.getAndValidate[Option[String]]("metrics.graphite.rateUnit", validUnits).getOrElse("SECONDS") + val durationUnit = configuration.getAndValidate[Option[String]]("metrics.graphite.durationUnit", validUnits).getOrElse("MILLISECONDS") + val interval = configuration.getOptional[FiniteDuration]("metrics.graphite.period").getOrElse(10.seconds) GraphiteMetricConfig(host, port, prefix, rateUnit, durationUnit, interval) } } @@ -48,18 +47,18 @@ object GraphiteMetricConfig extends UnitConverter { case class GangliaMetricConfig(host: String, port: Int, prefix: String, mode: String, ttl: Int, version: Boolean, rateUnit: TimeUnit, durationUnit: TimeUnit, tMax: Int, dMax: Int, interval: FiniteDuration) object GangliaMetricConfig extends UnitConverter { def apply(configuration: Configuration): GangliaMetricConfig = { - val host = configuration.getString("metrics.ganglia.host").getOrElse("127.0.0.1") - val port = configuration.getInt("metrics.ganglia.port").getOrElse(8649) - val prefix = configuration.getString("metrics.ganglia.prefix").getOrElse("thehive") - val validMode = Some(GMetric.UDPAddressingMode.values().map(_.toString).toSet) - val mode = configuration.getString("metrics.ganglia.mode", validMode).getOrElse("UNICAST") - val ttl = configuration.getInt("metrics.ganglia.ttl").getOrElse(1) - val version = configuration.getString("metrics.ganglia.version", Some(Set("3.0", "3.1"))).forall(_ == 3.1) - val rateUnit = configuration.getString("metrics.ganglia.rateUnit", validUnits).getOrElse("SECONDS") - val durationUnit = configuration.getString("metrics.ganglia.durationUnit", validUnits).getOrElse("MILLISECONDS") - val tMax = configuration.getInt("metrics.ganglia.tmax").getOrElse(60) - val dMax = configuration.getInt("metrics.ganglia.dmax").getOrElse(0) - val interval = configuration.getMilliseconds("metrics.ganglia.period").getOrElse(10000L).millis // 10 seconds + val host = configuration.getOptional[String]("metrics.ganglia.host").getOrElse("127.0.0.1") + val port = configuration.getOptional[Int]("metrics.ganglia.port").getOrElse(8649) + val prefix = configuration.getOptional[String]("metrics.ganglia.prefix").getOrElse("thehive") + val validMode = GMetric.UDPAddressingMode.values().map(v ⇒ Some(v.toString)).toSet + None + val mode = configuration.getAndValidate[Option[String]]("metrics.ganglia.mode", validMode).getOrElse("UNICAST") + val ttl = configuration.getOptional[Int]("metrics.ganglia.ttl").getOrElse(1) + val version = configuration.getAndValidate[Option[String]]("metrics.ganglia.version", Set(Some("3.0"), Some("3.1"), None)).forall(_ == 3.1) + val rateUnit = configuration.getAndValidate[Option[String]]("metrics.ganglia.rateUnit", validUnits).getOrElse("SECONDS") + val durationUnit = configuration.getAndValidate[Option[String]]("metrics.ganglia.durationUnit", validUnits).getOrElse("MILLISECONDS") + val tMax = configuration.getOptional[Int]("metrics.ganglia.tmax").getOrElse(60) + val dMax = configuration.getOptional[Int]("metrics.ganglia.dmax").getOrElse(0) + val interval = configuration.getOptional[FiniteDuration]("metrics.ganglia.period").getOrElse(10.seconds) GangliaMetricConfig(host, port, prefix, mode, ttl, version, rateUnit, durationUnit, tMax, dMax, interval) } } @@ -67,15 +66,15 @@ object GangliaMetricConfig extends UnitConverter { case class InfluxMetricConfig(url: String, user: String, password: String, database: String, retention: String, tags: Map[String, String], interval: FiniteDuration) object InfluxMetricConfig { def apply(configuration: Configuration): InfluxMetricConfig = { - val url = configuration.getString("metrics.influx.url").getOrElse("http://127.0.0.1:8086") - val user = configuration.getString("meorg.aopalliance.intercept.MethodInterceptortrics.influx.user").getOrElse("root") - val password = configuration.getString("metrics.influx.password").getOrElse("root") - val database = configuration.getString("metrics.influx.database").getOrElse("thehive") - val retention = configuration.getString("metrics.influx.retention").getOrElse("default") - val tags = configuration.getConfig("metrics.influx.tags").fold(Map.empty[String, String]) { cfg ⇒ + val url = configuration.getOptional[String]("metrics.influx.url").getOrElse("http://127.0.0.1:8086") + val user = configuration.getOptional[String]("meorg.aopalliance.intercept.MethodInterceptortrics.influx.user").getOrElse("root") + val password = configuration.getOptional[String]("metrics.influx.password").getOrElse("root") + val database = configuration.getOptional[String]("metrics.influx.database").getOrElse("thehive") + val retention = configuration.getOptional[String]("metrics.influx.retention").getOrElse("default") + val tags = configuration.getOptional[Configuration]("metrics.influx.tags").fold(Map.empty[String, String]) { cfg ⇒ cfg.entrySet.toMap.mapValues(_.render) } - val interval = configuration.getMilliseconds("metrics.influx.period").getOrElse(10000L).millis // 10 seconds + val interval = configuration.getOptional[FiniteDuration]("metrics.influx.period").getOrElse(10.seconds) InfluxMetricConfig(url, user, password, database, retention, tags, interval) } } @@ -84,20 +83,26 @@ case class MetricConfig(registryName: String, rateUnit: TimeUnit, durationUnit: object MetricConfig extends UnitConverter { def apply(configuration: Configuration): MetricConfig = { - val registryName = configuration.getString("metrics.name").getOrElse("default") - val rateUnit = configuration.getString("metrics.rateUnit", validUnits).getOrElse("SECONDS") - val durationUnit = configuration.getString("metrics.durationUnit", validUnits).getOrElse("SECONDS") - val jvm = configuration.getBoolean("metrics.jvm").getOrElse(true) - val logback = configuration.getBoolean("metrics.logback").getOrElse(true) - - val graphiteMetricConfig = configuration.getBoolean("metrics.graphite.enabled").filter(identity).map(_ ⇒ GraphiteMetricConfig(configuration)) - val gangliaMetricConfig = configuration.getBoolean("metrics.ganglia.enabled").filter(identity).map(_ ⇒ GangliaMetricConfig(configuration)) - val influxMetricConfig = configuration.getBoolean("metrics.influx.enabled").filter(identity).map(_ ⇒ InfluxMetricConfig(configuration)) + val registryName = configuration.getOptional[String]("metrics.name").getOrElse("default") + val rateUnit = configuration.getAndValidate[Option[String]]("metrics.rateUnit", validUnits).getOrElse("SECONDS") + val durationUnit = configuration.getAndValidate[Option[String]]("metrics.durationUnit", validUnits).getOrElse("SECONDS") + val jvm = configuration.getOptional[Boolean]("metrics.jvm").getOrElse(true) + val logback = configuration.getOptional[Boolean]("metrics.logback").getOrElse(true) + + val graphiteMetricConfig = configuration.getOptional[Boolean]("metrics.graphite.enabled").filter(identity).map(_ ⇒ GraphiteMetricConfig(configuration)) + val gangliaMetricConfig = configuration.getOptional[Boolean]("metrics.ganglia.enabled").filter(identity).map(_ ⇒ GangliaMetricConfig(configuration)) + val influxMetricConfig = configuration.getOptional[Boolean]("metrics.influx.enabled").filter(identity).map(_ ⇒ InfluxMetricConfig(configuration)) MetricConfig(registryName, rateUnit, durationUnit, jvm, logback, graphiteMetricConfig, gangliaMetricConfig, influxMetricConfig) } } -class TimedInterceptor @Inject() (metricsProvider: Provider[Metrics]) extends MethodInterceptor { +@Singleton +class TimedInterceptor @Inject() ( + actionBuilderProvider: Provider[DefaultActionBuilder], + metricsProvider: Provider[Metrics], + ecProvider: Provider[ExecutionContext]) extends MethodInterceptor { + implicit lazy val ec = ecProvider.get + lazy val actionBuilder = actionBuilderProvider.get override def invoke(invocation: MethodInvocation): AnyRef = { val timerName = invocation.getStaticPart.getAnnotation(classOf[Timed]).value match { case "" ⇒ @@ -110,14 +115,12 @@ class TimedInterceptor @Inject() (metricsProvider: Provider[Metrics]) extends Me case f: Future[_] ⇒ f.onComplete { _ ⇒ timer.stop() } f - case action: Action[x] ⇒ new Action[x] { - def apply(request: Request[x]) = { - val result = action.apply(request) - result.onComplete { _ ⇒ timer.stop() } - result - } - lazy val parser = action.parser + case action: Action[x] ⇒ actionBuilder.async(action.parser) { (request: Request[x]) ⇒ + val result: Future[Result] = action.apply(request) + result.onComplete { _ ⇒ timer.stop() } + result } + case o ⇒ timer.stop() o @@ -129,13 +132,15 @@ class MetricsModule( environment: Environment, configuration: Configuration) extends ConnectorModule { def configure(): Unit = { - if (configuration.getBoolean("metrics.enabled").getOrElse(false)) { + if (configuration.getOptional[Boolean]("metrics.enabled").getOrElse(false)) { bind[MetricConfig].toInstance(MetricConfig(configuration)) bind[Metrics].asEagerSingleton() val filterBindings = ScalaMultibinder.newSetBinder[Filter](binder) filterBindings.addBinding.to[MetricsFilterImpl] - - bindInterceptor(Matchers.any, Matchers.annotatedWith(classOf[org.elastic4play.Timed]), new TimedInterceptor(getProvider[Metrics])) + bindInterceptor( + Matchers.any, + Matchers.annotatedWith(classOf[org.elastic4play.Timed]), + new TimedInterceptor(getProvider[DefaultActionBuilder], getProvider[Metrics], getProvider[ExecutionContext])) registerController[MetricsCtrl] } () @@ -158,10 +163,16 @@ class Metrics @Inject() (configuration: Configuration, metricConfig: MetricConfi if (metricConfig.logback) { val appender: InstrumentedAppender = new InstrumentedAppender(registry) - val logger = Logger.logger.asInstanceOf[classic.Logger] - appender.setContext(logger.getLoggerContext) - appender.start() - logger.addAppender(appender) + val logger = Logger(getClass) + + logger.underlyingLogger match { + case cl: classic.Logger ⇒ + appender.setContext(cl.getLoggerContext) + appender.start() + cl.addAppender(appender) + case l ⇒ + logger.error(s"Can't initialize logger metrics. Logger (${l.getClass} is not a logback classic logger).") + } } mapper.registerModule(new json.MetricsModule(metricConfig.rateUnit, metricConfig.durationUnit, false)) diff --git a/thehive-metrics/build.sbt b/thehive-metrics/build.sbt index e696b189f4..445fca1da2 100644 --- a/thehive-metrics/build.sbt +++ b/thehive-metrics/build.sbt @@ -14,5 +14,3 @@ libraryDependencies ++= Seq( "io.dropwizard.metrics" % "metrics-ganglia" % "3.1.2", "info.ganglia.gmetric4j" % "gmetric4j" % "1.0.10" ) - -enablePlugins(PlayScala) diff --git a/thehive-misp/app/connectors/misp/JsonFormat.scala b/thehive-misp/app/connectors/misp/JsonFormat.scala index 5a27bb80d5..2b7234e013 100644 --- a/thehive-misp/app/connectors/misp/JsonFormat.scala +++ b/thehive-misp/app/connectors/misp/JsonFormat.scala @@ -6,6 +6,8 @@ import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup import play.api.libs.json.JsValue.jsValueToJsLookup import play.api.libs.json._ +import org.elastic4play.services.JsonFormat.attachmentFormat + object JsonFormat { implicit val mispAlertReads: Reads[MispAlert] = Reads[MispAlert] { json ⇒ @@ -29,7 +31,8 @@ object JsonFormat { date = new Date(timestamp.toLong * 1000) publishTimestamp ← (json \ "publish_timestamp").validate[String] publishDate = new Date(publishTimestamp.toLong * 1000) - threatLevel ← (json \ "threat_level_id").validate[String] + threatLevelString ← (json \ "threat_level_id").validate[String] + threatLevel = threatLevelString.toLong isPublished ← (json \ "published").validate[Boolean] } yield MispAlert( org, @@ -39,7 +42,7 @@ object JsonFormat { isPublished, s"#$eventId ${info.trim}", s"Imported from MISP Event #$eventId, created at $date", - threatLevel.toLong, + if (0 < threatLevel && threatLevel < 4) 4 - threatLevel else 2, alertTags, tlp, "") @@ -64,5 +67,26 @@ object JsonFormat { date, comment, value, - tags :+ s"MISP:category$category" :+ s"MISP:type=$tpe")) -} + tags)) + + implicit val exportedAttributeWrites: Writes[ExportedMispAttribute] = Writes[ExportedMispAttribute] { attribute ⇒ + Json.obj( + "category" → attribute.category, + "type" → attribute.tpe, + "value" → attribute.value.fold[String](identity, _.name), + "comment" → attribute.comment) + } + + implicit val mispArtifactWrites: Writes[MispArtifact] = OWrites[MispArtifact] { artifact ⇒ + Json.obj( + "dataType" → artifact.dataType, + "message" → artifact.message, + "tlp" → artifact.tlp, + "tags" → artifact.tags, + "startDate" → artifact.startDate) + (artifact.value match { + case SimpleArtifactData(data) ⇒ "data" → JsString(data) + case RemoteAttachmentArtifact(filename, reference, tpe) ⇒ "remoteAttachment" → Json.obj("filename" → filename, "reference" → reference, "type" → tpe) + case AttachmentArtifact(attachment) ⇒ "attachment" → Json.toJson(attachment) + }) + } +} \ No newline at end of file diff --git a/thehive-misp/app/connectors/misp/MispConfig.scala b/thehive-misp/app/connectors/misp/MispConfig.scala new file mode 100644 index 0000000000..28350d045e --- /dev/null +++ b/thehive-misp/app/connectors/misp/MispConfig.scala @@ -0,0 +1,40 @@ +package connectors.misp + +import javax.inject.{ Inject, Singleton } + +import scala.concurrent.duration.{ DurationInt, FiniteDuration } +import scala.util.Try + +import play.api.Configuration + +import services.CustomWSAPI + +@Singleton +class MispConfig(val interval: FiniteDuration, val connections: Seq[MispConnection]) { + + def this(configuration: Configuration, defaultCaseTemplate: Option[String], globalWS: CustomWSAPI) = this( + configuration.getOptional[FiniteDuration]("misp.interval").getOrElse(1.hour), + + for { + cfg ← configuration.getOptional[Configuration]("misp").toSeq + mispWS = globalWS.withConfig(cfg) + + defaultArtifactTags = cfg.getOptional[Seq[String]]("tags").getOrElse(Nil) + name ← cfg.subKeys + + mispConnectionConfig ← Try(cfg.get[Configuration](name)).toOption.toSeq + url ← mispConnectionConfig.getOptional[String]("url") + key ← mispConnectionConfig.getOptional[String]("key") + instanceWS = mispWS.withConfig(mispConnectionConfig) + artifactTags = mispConnectionConfig.getOptional[Seq[String]]("tags").getOrElse(defaultArtifactTags) + caseTemplate = mispConnectionConfig.getOptional[String]("caseTemplate").orElse(defaultCaseTemplate) + } yield MispConnection(name, url, key, instanceWS, caseTemplate, artifactTags)) + + @Inject def this(configuration: Configuration, httpSrv: CustomWSAPI) = + this( + configuration, + configuration.getOptional[String]("misp.caseTemplate"), + httpSrv) + + def getConnection(name: String): Option[MispConnection] = connections.find(_.name == name) +} diff --git a/thehive-misp/app/connectors/misp/MispConnection.scala b/thehive-misp/app/connectors/misp/MispConnection.scala new file mode 100644 index 0000000000..4bc4db9c21 --- /dev/null +++ b/thehive-misp/app/connectors/misp/MispConnection.scala @@ -0,0 +1,30 @@ +package connectors.misp + +import play.api.Logger + +import services.CustomWSAPI + +case class MispConnection( + name: String, + baseUrl: String, + key: String, + ws: CustomWSAPI, + caseTemplate: Option[String], + artifactTags: Seq[String]) { + + private[MispConnection] lazy val logger = Logger(getClass) + + logger.info( + s"""Add MISP connection $name + |\turl: $baseUrl + |\tproxy: ${ws.proxy} + |\tcase template: ${caseTemplate.getOrElse("")} + |\tartifact tags: ${artifactTags.mkString}""".stripMargin) + + private[misp] def apply(url: String) = + ws.url(s"$baseUrl/$url") + .withHttpHeaders( + "Authorization" → key, + "Accept" → "application/json") + +} \ No newline at end of file diff --git a/thehive-misp/app/connectors/misp/MispConnector.scala b/thehive-misp/app/connectors/misp/MispConnector.scala index bab80ac1b3..d560f51a19 100644 --- a/thehive-misp/app/connectors/misp/MispConnector.scala +++ b/thehive-misp/app/connectors/misp/MispConnector.scala @@ -2,6 +2,7 @@ package connectors.misp import javax.inject.Singleton +import play.api.libs.concurrent.AkkaGuiceSupport import play.api.{ Configuration, Environment, Logger } import connectors.ConnectorModule @@ -9,18 +10,17 @@ import connectors.ConnectorModule @Singleton class MispConnector( environment: Environment, - configuration: Configuration) extends ConnectorModule { - val log = Logger(getClass) + configuration: Configuration) extends ConnectorModule with AkkaGuiceSupport { + private[MispConnector] lazy val logger = Logger(getClass) def configure() { try { - // val mispConfig = MispConfig(configuration) - // bind[MispConfig].toInstance(mispConfig) bind[MispSrv].asEagerSingleton() + bindActor[UpdateMispAlertArtifactActor]("UpdateMispAlertArtifactActor") registerController[MispCtrl] } catch { - case t: Throwable ⇒ log.error("MISP connector is disabled because its configuration is invalid", t) + case t: Throwable ⇒ logger.error("MISP connector is disabled because its configuration is invalid", t) } } } \ No newline at end of file diff --git a/thehive-misp/app/connectors/misp/MispConverter.scala b/thehive-misp/app/connectors/misp/MispConverter.scala new file mode 100644 index 0000000000..c4e5446bde --- /dev/null +++ b/thehive-misp/app/connectors/misp/MispConverter.scala @@ -0,0 +1,217 @@ +package connectors.misp + +trait MispConverter { + def convertAttribute(mispAttribute: MispAttribute): Seq[MispArtifact] = { + val tags = Seq(s"MISP:type=${mispAttribute.tpe}", s"MISP:category=${mispAttribute.category}") + if (mispAttribute.tpe == "attachment" || mispAttribute.tpe == "malware-sample") { + Seq( + MispArtifact( + value = RemoteAttachmentArtifact(mispAttribute.value.split("\\|").head, mispAttribute.id, mispAttribute.tpe), + dataType = "file", + message = mispAttribute.comment, + tlp = 0, + tags = tags ++ mispAttribute.tags, + startDate = mispAttribute.date)) + } + else { + val dataType = toArtifact(mispAttribute.tpe) + val artifact = + MispArtifact( + value = SimpleArtifactData(mispAttribute.value), + dataType = dataType, + message = mispAttribute.comment, + tlp = 0, + tags = tags ++ mispAttribute.tags, + startDate = mispAttribute.date) + + val types = mispAttribute.tpe.split('|').toSeq + if (types.length > 1) { + val values = mispAttribute.value.split('|').toSeq + val typesValues = types.zipAll(values, "noType", "noValue") + val additionnalMessage = typesValues + .map { + case (t, v) ⇒ s"$t: $v" + } + .mkString("\n") + typesValues.map { + case (tpe, value) ⇒ + artifact.copy( + dataType = toArtifact(tpe), + value = SimpleArtifactData(value), + message = mispAttribute.comment + "\n" + additionnalMessage) + } + } + else { + Seq(artifact) + } + } + } + + def fromArtifact(dataType: String, data: Option[String]): (String, String) = { + dataType match { + case "filename" ⇒ "Payload delivery" → "filename" + case "fqdn" ⇒ "Network activity" → "hostname" + case "url" ⇒ "External analysis" → "url" + case "user-agent" ⇒ "Network activity" → "user-agent" + case "domain" ⇒ "Network activity" → "domain" + case "ip" ⇒ "Network activity" → "ip-src" + case "mail_subject" ⇒ "Payload delivery" → "email-subject" + case "hash" ⇒ data.fold(0)(_.length) match { + case 32 ⇒ "Payload delivery" → "md5" + case 40 ⇒ "Payload delivery" → "sha1" + case 64 ⇒ "Payload delivery" → "sha256" + case 56 ⇒ "Payload delivery" → "sha224" + case 71 ⇒ "Payload delivery" → "sha384" + case 128 ⇒ "Payload delivery" → "sha512" + case _ ⇒ "Payload delivery" → "other" + } + case "mail" ⇒ "Payload delivery" → "email-src" + case "registry" ⇒ "Persistence mechanism" → "regkey" + case "uri_path" ⇒ "Network activity" → "uri" + case "regexp" ⇒ "Other" → "other" + case "other" ⇒ "Other" → "other" + case "file" ⇒ "Payload delivery" → "malware-sample" + case _ ⇒ "Other" → "other" + } + } + + def toArtifact(tpe: String): String = attribute2artifactLookup.getOrElse(tpe, "other") + + private lazy val attribute2artifactLookup = Map( + "md5" → "hash", + "sha1" → "hash", + "sha256" → "hash", + "filename" → "filename", + "pdb" → "other", + "filename|md5" → "other", + "filename|sha1" → "other", + "filename|sha256" → "other", + "ip-src" → "ip", + "ip-dst" → "ip", + "hostname" → "fqdn", + "domain" → "domain", + "domain|ip" → "other", + "email-src" → "mail", + "email-dst" → "mail", + "email-subject" → "mail_subject", + "email-attachment" → "other", + "float" → "other", + "url" → "url", + "http-method" → "other", + "user-agent" → "user-agent", + "regkey" → "registry", + "regkey|value" → "registry", + "AS" → "other", + "snort" → "other", + "pattern-in-file" → "other", + "pattern-in-traffic" → "other", + "pattern-in-memory" → "other", + "yara" → "other", + "sigma" → "other", + "vulnerability" → "other", + "attachment" → "file", + "malware-sample" → "file", + "link" → "other", + "comment" → "other", + "text" → "other", + "hex" → "other", + "other" → "other", + "named" → "other", + "mutex" → "other", + "target-user" → "other", + "target-email" → "mail", + "target-machine" → "fqdn", + "target-org" → "other", + "target-location" → "other", + "target-external" → "other", + "btc" → "other", + "iban" → "other", + "bic" → "other", + "bank-account-nr" → "other", + "aba-rtn" → "other", + "bin" → "other", + "cc-number" → "other", + "prtn" → "other", + "threat-actor" → "other", + "campaign-name" → "other", + "campaign-id" → "other", + "malware-type" → "other", + "uri" → "uri_path", + "authentihash" → "other", + "ssdeep" → "hash", + "imphash" → "hash", + "pehash" → "hash", + "impfuzzy" → "hash", + "sha224" → "hash", + "sha384" → "hash", + "sha512" → "hash", + "sha512/224" → "hash", + "sha512/256" → "hash", + "tlsh" → "other", + "filename|authentihash" → "other", + "filename|ssdeep" → "other", + "filename|imphash" → "other", + "filename|impfuzzy" → "other", + "filename|pehash" → "other", + "filename|sha224" → "other", + "filename|sha384" → "other", + "filename|sha512" → "other", + "filename|sha512/224" → "other", + "filename|sha512/256" → "other", + "filename|tlsh" → "other", + "windows-scheduled-task" → "other", + "windows-service-name" → "other", + "windows-service-displayname" → "other", + "whois-registrant-email" → "mail", + "whois-registrant-phone" → "other", + "whois-registrant-name" → "other", + "whois-registrar" → "other", + "whois-creation-date" → "other", + "x509-fingerprint-sha1" → "other", + "dns-soa-email" → "other", + "size-in-bytes" → "other", + "counter" → "other", + "datetime" → "other", + "cpe" → "other", + "port" → "other", + "ip-dst|port" → "other", + "ip-src|port" → "other", + "hostname|port" → "other", + "email-dst-display-name" → "other", + "email-src-display-name" → "other", + "email-header" → "other", + "email-reply-to" → "other", + "email-x-mailer" → "other", + "email-mime-boundary" → "other", + "email-thread-index" → "other", + "email-message-id" → "other", + "github-username" → "other", + "github-repository" → "other", + "github-organisation" → "other", + "jabber-id" → "other", + "twitter-id" → "other", + "first-name" → "other", + "middle-name" → "other", + "last-name" → "other", + "date-of-birth" → "other", + "place-of-birth" → "other", + "gender" → "other", + "passport-number" → "other", + "passport-country" → "other", + "passport-expiration" → "other", + "redress-number" → "other", + "nationality" → "other", + "visa-number" → "other", + "issue-date-of-the-visa" → "other", + "primary-residence" → "other", + "country-of-residence" → "other", + "special-service-request" → "other", + "frequent-flyer-number" → "other", + "travel-details" → "other", + "payment-details" → "other", + "place-port-of-original-embarkation" → "other", + "place-port-of-clearance" → "other", + "place-port-of-onward-foreign-destination" → "other", + "passenger-name-record-locator-number" → "other", + "mobile-application-id" → "other") +} \ No newline at end of file diff --git a/thehive-misp/app/connectors/misp/MispCtrl.scala b/thehive-misp/app/connectors/misp/MispCtrl.scala index b1f3c9179b..315d79995c 100644 --- a/thehive-misp/app/connectors/misp/MispCtrl.scala +++ b/thehive-misp/app/connectors/misp/MispCtrl.scala @@ -2,58 +2,80 @@ package connectors.misp import javax.inject.{ Inject, Singleton } +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.Logger +import play.api.http.Status +import play.api.libs.json.{ JsObject, Json } +import play.api.mvc._ +import play.api.routing.SimpleRouter +import play.api.routing.sird.{ GET, POST, UrlContext } + import connectors.Connector -import models.{ Alert, Case, UpdateMispAlertArtifact } +import models.{ Alert, Case, Roles, UpdateMispAlertArtifact } +import services.{ AlertTransformer, CaseSrv } + import org.elastic4play.JsonFormat.tryWrites -import org.elastic4play.controllers.Authenticated +import org.elastic4play.controllers.{ Authenticated, Renderer } import org.elastic4play.models.JsonFormat.baseModelEntityWrites import org.elastic4play.services._ import org.elastic4play.{ NotFoundError, Timed } -import play.api.Logger -import play.api.http.Status -import play.api.libs.json.Json -import play.api.mvc.{ Action, AnyContent, Controller } -import play.api.routing.SimpleRouter -import play.api.routing.sird.{ GET, UrlContext } -import services.AlertTransformer - -import scala.concurrent.{ ExecutionContext, Future } @Singleton class MispCtrl @Inject() ( + mispSynchro: MispSynchro, mispSrv: MispSrv, + mispExport: MispExport, + mispConfig: MispConfig, + caseSrv: CaseSrv, authenticated: Authenticated, + renderer: Renderer, eventSrv: EventSrv, - implicit val ec: ExecutionContext) extends Controller with Connector with Status with AlertTransformer { + components: ControllerComponents, + implicit val ec: ExecutionContext) extends AbstractController(components) with Connector with Status with AlertTransformer { override val name: String = "misp" + override val status: JsObject = Json.obj("enabled" → true, "servers" → mispConfig.connections.map(_.name)) + private[MispCtrl] lazy val logger = Logger(getClass) val router = SimpleRouter { - case GET(p"/_syncAlerts") ⇒ syncAlerts - case GET(p"/_syncAllAlerts") ⇒ syncAllAlerts - case GET(p"/_syncArtifacts") ⇒ syncArtifacts - case r ⇒ throw NotFoundError(s"${r.uri} not found") + case GET(p"/_syncAlerts") ⇒ syncAlerts + case GET(p"/_syncAllAlerts") ⇒ syncAllAlerts + case GET(p"/_syncArtifacts") ⇒ syncArtifacts + case POST(p"/export/$caseId/$mispName") ⇒ exportCase(mispName, caseId) + case r ⇒ throw NotFoundError(s"${r.uri} not found") } @Timed - def syncAlerts: Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ - mispSrv.synchronize() + def syncAlerts: Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ + mispSynchro.synchronize() .map { m ⇒ Ok(Json.toJson(m)) } } @Timed - def syncAllAlerts: Action[AnyContent] = authenticated(Role.admin).async { implicit request ⇒ - mispSrv.fullSynchronize() + def syncAllAlerts: Action[AnyContent] = authenticated(Roles.admin).async { implicit request ⇒ + mispSynchro.fullSynchronize() .map { m ⇒ Ok(Json.toJson(m)) } } @Timed - def syncArtifacts: Action[AnyContent] = authenticated(Role.admin) { + def syncArtifacts: Action[AnyContent] = authenticated(Roles.admin) { eventSrv.publish(UpdateMispAlertArtifact()) Ok("") } + @Timed + def exportCase(mispName: String, caseId: String): Action[AnyContent] = authenticated(Roles.write).async { implicit request ⇒ + caseSrv + .get(caseId) + .flatMap { caze ⇒ mispExport.export(mispName, caze) } + .map { + case (_, exportedAttributes) ⇒ + renderer.toMultiOutput(CREATED, exportedAttributes) + } + } + override def createCase(alert: Alert, customCaseTemplate: Option[String])(implicit authContext: AuthContext): Future[Case] = { mispSrv.createCase(alert, customCaseTemplate) } @@ -61,4 +83,4 @@ class MispCtrl @Inject() ( override def mergeWithCase(alert: Alert, caze: Case)(implicit authContext: AuthContext): Future[Case] = { mispSrv.mergeWithCase(alert, caze) } -} \ No newline at end of file +} diff --git a/thehive-misp/app/connectors/misp/MispExport.scala b/thehive-misp/app/connectors/misp/MispExport.scala new file mode 100644 index 0000000000..d6d4060cfe --- /dev/null +++ b/thehive-misp/app/connectors/misp/MispExport.scala @@ -0,0 +1,186 @@ +package connectors.misp + +import java.text.SimpleDateFormat +import java.util.Date +import javax.inject.{ Inject, Provider, Singleton } + +import scala.concurrent.{ ExecutionContext, Future } +import scala.util.{ Success, Try } + +import play.api.libs.json.{ JsObject, Json } + +import akka.stream.scaladsl.Sink +import models.{ Artifact, Case } +import services.{ AlertSrv, ArtifactSrv } +import JsonFormat.exportedAttributeWrites +import akka.stream.Materializer + +import org.elastic4play.InternalError +import org.elastic4play.controllers.Fields +import org.elastic4play.services.{ Attachment, AttachmentSrv, AuthContext } +import org.elastic4play.services.JsonFormat.attachmentFormat +import org.elastic4play.utils.RichFuture + +@Singleton +class MispExport @Inject() ( + mispConfig: MispConfig, + mispSrv: MispSrv, + artifactSrv: ArtifactSrv, + alertSrvProvider: Provider[AlertSrv], + attachmentSrv: AttachmentSrv, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) extends MispConverter { + + lazy val dateFormat = new SimpleDateFormat("yy-MM-dd") + private[misp] lazy val alertSrv = alertSrvProvider.get + + def relatedMispEvent(mispName: String, caseId: String): Future[(Option[String], Option[String])] = { + import org.elastic4play.services.QueryDSL._ + alertSrv.find(and("type" ~= "misp", "case" ~= caseId, "source" ~= mispName), Some("0-1"), Nil) + ._1 + .map { alert ⇒ alert.id → alert.sourceRef() } + .runWith(Sink.headOption) + .map(alertIdSource ⇒ alertIdSource.map(_._1) → alertIdSource.map(_._2)) + } + + def removeDuplicateAttributes(attributes: Seq[ExportedMispAttribute]): Seq[ExportedMispAttribute] = { + val attrIndex = attributes.zipWithIndex + + attrIndex + .filter { + case (ExportedMispAttribute(_, category, tpe, value, _), index) ⇒ attrIndex.exists { + case (ExportedMispAttribute(_, `category`, `tpe`, `value`, _), otherIndex) ⇒ otherIndex >= index + case _ ⇒ true + } + } + .map(_._1) + } + + def createEvent(mispConnection: MispConnection, title: String, severity: Long, date: Date, attributes: Seq[ExportedMispAttribute]): Future[(String, Seq[ExportedMispAttribute])] = { + val mispEvent = Json.obj( + "Event" → Json.obj( + "distribution" → 0, + "threat_level_id" → (4 - severity), + "analysis" → 0, + "info" → title, + "date" → dateFormat.format(date), + "published" → false, + "Attribute" → attributes)) + mispConnection("events") + .post(mispEvent) + .map { mispResponse ⇒ + val eventId = (mispResponse.json \ "Event" \ "id") + .asOpt[String] + .getOrElse(throw InternalError(s"Unexpected MISP response: ${mispResponse.status} ${mispResponse.statusText}\n${mispResponse.body}")) + val messages = (mispResponse.json \ "errors" \ "Attribute") + .asOpt[JsObject] + .getOrElse(JsObject(Nil)) + .fields + .toMap + .mapValues { m ⇒ + (m \ "value") + .asOpt[Seq[String]] + .flatMap(_.headOption) + .getOrElse(s"Unexpected message format: $m") + } + val exportedAttributes = attributes.zipWithIndex.collect { + case (attr, index) if !messages.contains(index.toString) ⇒ attr + } + eventId → exportedAttributes + } + } + + def exportAttribute(mispConnection: MispConnection, eventId: String, attribute: ExportedMispAttribute): Future[Artifact] = { + val mispResponse = attribute match { + case ExportedMispAttribute(_, _, _, Right(attachment), comment) ⇒ + attachmentSrv + .source(attachment.id) + .runReduce(_ ++ _) + .flatMap { data ⇒ + val b64data = java.util.Base64.getEncoder.encodeToString(data.toArray[Byte]) + val body = Json.obj( + "request" → Json.obj( + "event_id" → eventId.toInt, + "category" → "Payload delivery", + "type" → "malware-sample", + "comment" → comment, + "files" → Json.arr( + Json.obj( + "filename" → attachment.name, + "data" → b64data)))) + mispConnection("events/upload_sample").post(body) + } + case attr ⇒ mispConnection(s"attributes/add/$eventId").post(Json.toJson(attr)) + + } + + mispResponse.map { + case response if response.status / 100 == 2 ⇒ attribute.artifact + case response ⇒ + val json = response.json + val message = (json \ "message").asOpt[String] + val error = (json \ "errors" \ "value").head.asOpt[String] + val errorMessage = for (m ← message; e ← error) yield s"$m $e" + throw MispExportError(errorMessage orElse message orElse error getOrElse s"Unexpected MISP response: ${response.status} ${response.statusText}\n${response.body}", attribute.artifact) + } + } + + def export(mispName: String, caze: Case)(implicit authContext: AuthContext): Future[(String, Seq[Try[Artifact]])] = { + val mispConnection = mispConfig.getConnection(mispName).getOrElse(sys.error("MISP instance not found")) + + for { + (maybeAlertId, maybeEventId) ← relatedMispEvent(mispName, caze.id) + attributes ← mispSrv.getAttributesFromCase(caze) + uniqueAttributes = removeDuplicateAttributes(attributes) + (eventId, initialExportesArtifacts, existingAttributes) ← maybeEventId.fold { + val simpleAttributes = uniqueAttributes.filter(_.value.isLeft) + // if no event is associated to this case, create a new one + createEvent(mispConnection, caze.title(), caze.severity(), caze.startDate(), simpleAttributes).map { + case (eventId, exportedAttributes) ⇒ (eventId, exportedAttributes.map(a ⇒ Success(a.artifact)), exportedAttributes.map(_.value.map(_.name))) + } + } { eventId ⇒ // if an event already exists, retrieve its attributes in order to export only new one + mispSrv.getAttributesFromMisp(mispConnection, eventId, None).map { attributes ⇒ + (eventId, Nil, attributes.map { + case MispArtifact(SimpleArtifactData(data), _, _, _, _, _) ⇒ Left(data) + case MispArtifact(RemoteAttachmentArtifact(filename, _, _), _, _, _, _, _) ⇒ Right(filename) + case MispArtifact(AttachmentArtifact(Attachment(filename, _, _, _, _)), _, _, _, _, _) ⇒ Right(filename) + }) + } + } + newAttributes = uniqueAttributes.filterNot(attr ⇒ existingAttributes.contains(attr.value.map(_.name))) + exportedArtifact ← Future.traverse(newAttributes)(attr ⇒ exportAttribute(mispConnection, eventId, attr).toTry) + artifacts = uniqueAttributes.map { a ⇒ + Json.obj( + "data" → a.artifact.data(), + "dataType" → a.artifact.dataType(), + "message" → a.artifact.message(), + "startDate" → a.artifact.startDate(), + "attachment" → a.artifact.attachment(), + "tlp" → a.artifact.tlp(), + "tags" → a.artifact.tags(), + "ioc" → a.artifact.ioc()) + } + alert ← maybeAlertId.fold { + alertSrv.create(Fields(Json.obj( + "type" → "misp", + "source" → mispName, + "sourceRef" → eventId, + "date" → caze.startDate(), + "lastSyncDate" → new Date(0), + "case" → caze.id, + "title" → caze.title(), + "description" → "Case have been exported to MISP", + "severity" → caze.severity(), + "tags" → caze.tags(), + "tlp" → caze.tlp(), + "artifacts" → artifacts, + "status" → "Imported", + "follow" → true))) + } { alertId ⇒ + alertSrv.update(alertId, Fields(Json.obj( + "artifacts" → artifacts, + "status" → "Imported"))) + } + } yield alert.id → (initialExportesArtifacts ++ exportedArtifact) + } +} diff --git a/thehive-misp/app/connectors/misp/MispModel.scala b/thehive-misp/app/connectors/misp/MispModel.scala index c7d472eeab..37f6c8dc2a 100644 --- a/thehive-misp/app/connectors/misp/MispModel.scala +++ b/thehive-misp/app/connectors/misp/MispModel.scala @@ -2,6 +2,23 @@ package connectors.misp import java.util.Date +import models.Artifact + +import org.elastic4play.ErrorWithObject +import org.elastic4play.services.Attachment +import org.elastic4play.utils.Hash + +sealed trait ArtifactData +case class SimpleArtifactData(data: String) extends ArtifactData +case class AttachmentArtifact(attachment: Attachment) extends ArtifactData { + def name: String = attachment.name + def hashes: Seq[Hash] = attachment.hashes + def size: Long = attachment.size + def contentType: String = attachment.contentType + def id: String = attachment.id +} +case class RemoteAttachmentArtifact(filename: String, reference: String, tpe: String) extends ArtifactData + case class MispAlert( source: String, sourceRef: String, @@ -22,4 +39,21 @@ case class MispAttribute( date: Date, comment: String, value: String, - tags: Seq[String]) \ No newline at end of file + tags: Seq[String]) + +case class ExportedMispAttribute( + artifact: Artifact, + tpe: String, + category: String, + value: Either[String, Attachment], + comment: Option[String]) + +case class MispArtifact( + value: ArtifactData, + dataType: String, + message: String, + tlp: Long, + tags: Seq[String], + startDate: Date) + +case class MispExportError(message: String, artifact: Artifact) extends ErrorWithObject(message, artifact.attributes) \ No newline at end of file diff --git a/thehive-misp/app/connectors/misp/MispSrv.scala b/thehive-misp/app/connectors/misp/MispSrv.scala index 6a5f4636d4..5eba30ed0b 100644 --- a/thehive-misp/app/connectors/misp/MispSrv.scala +++ b/thehive-misp/app/connectors/misp/MispSrv.scala @@ -3,9 +3,16 @@ package connectors.misp import java.util.Date import javax.inject.{ Inject, Provider, Singleton } +import scala.concurrent.{ ExecutionContext, Future } + +import play.api.Logger +import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup +import play.api.libs.json.JsValue.jsValueToJsLookup +import play.api.libs.json.Json.toJsFieldJsValueWrapper +import play.api.libs.json._ +import play.api.libs.ws.WSBodyWritables.writeableOf_JsValue + import akka.NotUsed -import akka.actor.ActorDSL._ -import akka.actor.ActorSystem import akka.stream.Materializer import akka.stream.scaladsl.{ FileIO, Sink, Source } import connectors.misp.JsonFormat._ @@ -13,74 +20,11 @@ import models._ import net.lingala.zip4j.core.ZipFile import net.lingala.zip4j.exception.ZipException import net.lingala.zip4j.model.FileHeader -import org.elastic4play.controllers.{ Fields, FileInputValue } -import org.elastic4play.services.{ UserSrv ⇒ _, _ } -import org.elastic4play.utils.RichJson -import org.elastic4play.{ InternalError, NotFoundError } -import play.api.inject.ApplicationLifecycle -import play.api.libs.json.JsLookupResult.jsLookupResultToJsLookup -import play.api.libs.json.JsValue.jsValueToJsLookup -import play.api.libs.json.Json.toJsFieldJsValueWrapper -import play.api.libs.json._ -import play.api.{ Configuration, Environment, Logger } import services._ -import scala.collection.immutable -import scala.concurrent.duration.{ DurationInt, DurationLong, FiniteDuration } -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.{ Failure, Success, Try } - -class MispConfig(val interval: FiniteDuration, val connections: Seq[MispConnection]) { - - def this(configuration: Configuration, defaultCaseTemplate: Option[String], globalWS: CustomWSAPI) = this( - configuration.getMilliseconds("misp.interval").fold(1.hour)(_.millis), - - for { - cfg ← configuration.getConfig("misp").toSeq - mispWS = globalWS.withConfig(cfg) - - defaultArtifactTags = cfg.getStringSeq("tags").getOrElse(Nil) - name ← cfg.subKeys - - mispConnectionConfig ← Try(cfg.getConfig(name)).toOption.flatten.toSeq - url ← mispConnectionConfig.getString("url") - key ← mispConnectionConfig.getString("key") - instanceWS = mispWS.withConfig(mispConnectionConfig) - artifactTags = mispConnectionConfig.getStringSeq("tags").getOrElse(defaultArtifactTags) - caseTemplate = mispConnectionConfig.getString("caseTemplate").orElse(defaultCaseTemplate) - } yield MispConnection(name, url, key, instanceWS, caseTemplate, artifactTags)) - - @Inject def this(configuration: Configuration, httpSrv: CustomWSAPI) = - this( - configuration, - configuration.getString("misp.caseTemplate"), - httpSrv) -} - -case class MispConnection( - name: String, - baseUrl: String, - key: String, - ws: CustomWSAPI, - caseTemplate: Option[String], - artifactTags: Seq[String]) { - - private[MispConnection] lazy val logger = Logger(getClass) - - logger.info( - s"""Add MISP connection $name - |\turl: $baseUrl - |\tproxy: ${ws.proxy} - |\tcase template: ${caseTemplate.getOrElse("")} - |\tartifact tags: ${artifactTags.mkString}""".stripMargin) - - private[misp] def apply(url: String) = - ws.url(s"$baseUrl/$url") - .withHeaders( - "Authorization" → key, - "Accept" → "application/json") - -} +import org.elastic4play.controllers.{ Fields, FileInputValue } +import org.elastic4play.services.{ Attachment, AuthContext, TempSrv } +import org.elastic4play.{ InternalError, NotFoundError } @Singleton class MispSrv @Inject() ( @@ -88,19 +32,11 @@ class MispSrv @Inject() ( alertSrvProvider: Provider[AlertSrv], caseSrv: CaseSrv, artifactSrv: ArtifactSrv, - userSrv: UserSrv, - attachmentSrv: AttachmentSrv, tempSrv: TempSrv, - eventSrv: EventSrv, - migrationSrv: MigrationSrv, - httpSrv: CustomWSAPI, - environment: Environment, - lifecycle: ApplicationLifecycle, - implicit val system: ActorSystem, - implicit val materializer: Materializer, - implicit val ec: ExecutionContext) { + implicit val ec: ExecutionContext, + implicit val mat: Materializer) extends MispConverter { - private[misp] val logger = Logger(getClass) + private[misp] lazy val logger = Logger(getClass) private[misp] lazy val alertSrv = alertSrvProvider.get private[misp] def getInstanceConfig(name: String): Future[MispConnection] = mispConfig.connections @@ -109,144 +45,6 @@ class MispSrv @Inject() ( Future.successful(instanceConfig) } - private[misp] def initScheduler() = { - val task = system.scheduler.schedule(0.seconds, mispConfig.interval) { - if (migrationSrv.isReady) { - logger.info("Update of MISP events is starting ...") - userSrv - .inInitAuthContext { implicit authContext ⇒ - synchronize().andThen { case _ ⇒ tempSrv.releaseTemporaryFiles() } - } - .onComplete { - case Success(a) ⇒ - logger.info("Misp synchronization completed") - a.collect { - case Failure(t) ⇒ logger.warn(s"Update MISP error", t) - } - case Failure(t) ⇒ logger.info("Misp synchronization failed", t) - } - } - else { - logger.info("MISP synchronization cancel, database is not ready") - } - } - lifecycle.addStopHook { () ⇒ - logger.info("Stopping MISP fetching ...") - task.cancel() - Future.successful(()) - } - } - - initScheduler() - eventSrv.subscribe(actor(new Act { - become { - case UpdateMispAlertArtifact() ⇒ - logger.info("UpdateMispAlertArtifact") - userSrv - .inInitAuthContext { implicit authContext ⇒ - updateMispAlertArtifact() - } - .onComplete { - case Success(_) ⇒ logger.info("Artifacts in MISP alerts updated") - case Failure(error) ⇒ logger.error("Update MISP alert artifacts error :", error) - } - () - case msg ⇒ - logger.info(s"Receiving unexpected message: $msg (${msg.getClass})") - } - }), classOf[UpdateMispAlertArtifact]) - logger.info("subscribe actor") - - def synchronize()(implicit authContext: AuthContext): Future[Seq[Try[Alert]]] = { - import org.elastic4play.services.QueryDSL._ - - // for each MISP server - Source(mispConfig.connections.toList) - // get last synchronization - .mapAsyncUnordered(1) { mispConnection ⇒ - alertSrv.stats(and("type" ~= "misp", "source" ~= mispConnection.name), Seq(selectMax("lastSyncDate"))) - .map { maxLastSyncDate ⇒ mispConnection → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) } - .recover { case _ ⇒ mispConnection → new Date(0) } - } - .flatMapConcat { - case (mispConnection, lastSyncDate) ⇒ - synchronize(mispConnection, lastSyncDate) - } - .runWith(Sink.seq) - } - - def fullSynchronize()(implicit authContext: AuthContext): Future[immutable.Seq[Try[Alert]]] = { - Source(mispConfig.connections.toList) - .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, new Date(1))) - .runWith(Sink.seq) - } - - def synchronize(mispConnection: MispConnection, lastSyncDate: Date)(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = { - logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate") - val fullSynchro = if (lastSyncDate.getTime == 1) Some(lastSyncDate) else None - // get events that have been published after the last synchronization - getEventsFromDate(mispConnection, lastSyncDate) - // get related alert - .mapAsyncUnordered(1) { event ⇒ - logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") - alertSrv.get("misp", event.source, event.sourceRef) - .map((event, _)) - } - .mapAsyncUnordered(1) { - case (event, alert) ⇒ - logger.trace(s"MISP synchro ${mispConnection.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}") - logger.info(s"getting MISP event ${event.source}:${event.sourceRef}") - getAttributes(mispConnection, event.sourceRef, fullSynchro.orElse(alert.map(_.lastSyncDate()))) - .map((event, alert, _)) - } - .mapAsyncUnordered(1) { - // if there is no related alert, create a new one - case (event, None, attrs) ⇒ - logger.info(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)") - val alertJson = Json.toJson(event).as[JsObject] + - ("type" → JsString("misp")) + - ("caseTemplate" → mispConnection.caseTemplate.fold[JsValue](JsNull)(JsString)) + - ("artifacts" → JsArray(attrs)) - alertSrv.create(Fields(alertJson)) - .map(Success(_)) - .recover { case t ⇒ Failure(t) } - - // if a related alert exists, update it - case (event, Some(alert), attrs) ⇒ - logger.info(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)") - val alertJson = Json.toJson(event).as[JsObject] - - "type" - - "source" - - "sourceRef" - - "caseTemplate" - - "date" + - ("artifacts" → JsArray(attrs)) + - // if this is a full synchronization, don't update alert status - ("status" → (if (!alert.follow() || fullSynchro.isDefined) Json.toJson(alert.status()) - else alert.status() match { - case AlertStatus.New ⇒ Json.toJson(AlertStatus.New) - case _ ⇒ Json.toJson(AlertStatus.Updated) - })) - logger.debug(s"Update alert ${alert.id} with\n$alertJson") - val fAlert = alertSrv.update(alert.id, Fields(alertJson)) - // if a case have been created, update it - (alert.caze() match { - case None ⇒ fAlert - case Some(caze) ⇒ - for { - a ← fAlert - // if this is a full synchronization, don't update case status - caseFields = if (fullSynchro.isDefined) Fields(alert.toCaseJson).unset("status") - else Fields(alert.toCaseJson) - _ ← caseSrv.update(caze, caseFields) - _ ← artifactSrv.create(caze, attrs.map(Fields.apply)) - } yield a - }) - .map(Success(_)) - .recover { case t ⇒ Failure(t) } - } - } - def getEventsFromDate(mispConnection: MispConnection, fromDate: Date): Source[MispAlert, NotUsed] = { val date = fromDate.getTime / 1000 Source @@ -275,14 +73,33 @@ class MispSrv @Inject() ( val eventsSize = events.size if (eventJsonSize != eventsSize) logger.warn(s"MISP returns $eventJsonSize events but only $eventsSize contain valid data") - events.toList + events.filter(_.lastSyncDate after fromDate).toList + } + } + + def getAttributesFromCase(caze: Case): Future[Seq[ExportedMispAttribute]] = { + import org.elastic4play.services.QueryDSL._ + artifactSrv + .find(and(withParent(caze), "status" ~= "Ok", "ioc" ~= true), Some("all"), Nil) + ._1 + .map { artifact ⇒ + val (category, tpe) = fromArtifact(artifact.dataType(), artifact.data()) + val value = (artifact.data(), artifact.attachment()) match { + case (Some(data), None) ⇒ Left(data) + case (None, Some(attachment)) ⇒ Right(attachment) + case _ ⇒ + logger.error(s"Artifact $artifact has neither data nor attachment") + sys.error("???") + } + ExportedMispAttribute(artifact, tpe, category, value, artifact.message()) } + .runWith(Sink.seq) } - def getAttributes( + def getAttributesFromMisp( mispConnection: MispConnection, eventId: String, - fromDate: Option[Date]): Future[Seq[JsObject]] = { + fromDate: Option[Date]): Future[Seq[MispArtifact]] = { val date = fromDate.fold(0L)(_.getTime / 1000) @@ -295,34 +112,30 @@ class MispSrv @Inject() ( // add ("deleted" → "only") to see only deleted attributes .map { response ⇒ val refDate = fromDate.getOrElse(new Date(0)) - val artifactTags = JsString(s"src:${mispConnection.name}") +: JsArray(mispConnection.artifactTags.map(JsString)) + val artifactTags = s"src:${mispConnection.name}" +: mispConnection.artifactTags (Json.parse(response.body) \ "response" \\ "Attribute") .flatMap(_.as[Seq[MispAttribute]]) .filter(_.date after refDate) - .flatMap { - case a if a.tpe == "attachment" || a.tpe == "malware-sample" ⇒ - Seq( - Json.obj( - "dataType" → "file", - "message" → a.comment, - "tags" → (artifactTags.value ++ a.tags.map(JsString)), - "remoteAttachment" → Json.obj( - "filename" → a.value, - "reference" → a.id, - "type" → a.tpe), - "startDate" → a.date)) - case a ⇒ convertAttribute(a).map { j ⇒ - val tags = artifactTags ++ (j \ "tags").asOpt[JsArray].getOrElse(JsArray(Nil)) - j.setIfAbsent("tlp", 2L) + ("tags" → tags) - } + .flatMap(convertAttribute) + .groupBy { + case MispArtifact(SimpleArtifactData(data), dataType, _, _, _, _) ⇒ dataType → Right(data) + case MispArtifact(RemoteAttachmentArtifact(filename, _, _), dataType, _, _, _, _) ⇒ dataType → Left(filename) + case MispArtifact(AttachmentArtifact(Attachment(filename, _, _, _, _)), dataType, _, _, _, _) ⇒ dataType → Left(filename) } + .values + .map { mispArtifact ⇒ + mispArtifact.head.copy( + tags = (mispArtifact.head.tags ++ artifactTags).distinct, + tlp = 2L) + } + .toSeq } } def attributeToArtifact( mispConnection: MispConnection, - alert: Alert, - attr: JsObject)(implicit authContext: AuthContext): Option[Future[Fields]] = { + attr: JsObject, + defaultTlp: Long)(implicit authContext: AuthContext): Option[Future[Fields]] = { (for { dataType ← (attr \ "dataType").validate[String] data ← (attr \ "data").validateOpt[String] @@ -347,7 +160,7 @@ class MispSrv @Inject() ( case "tlp:amber" ⇒ JsNumber(2) case "tlp:red" ⇒ JsNumber(3) } - .getOrElse(JsNumber(alert.tlp())) + .getOrElse(JsNumber(defaultTlp)) fields = Fields.empty .set("dataType", dataType) .set("message", message) @@ -363,7 +176,7 @@ class MispSrv @Inject() ( })) match { case JsSuccess(r, _) ⇒ Some(r) case e: JsError ⇒ - logger.warn(s"Invalid attribute in alert ${alert.id}: $e\n$attr") + logger.warn(s"Invalid attribute format: $e\n$attr") None } } @@ -383,7 +196,7 @@ class MispSrv @Inject() ( def importArtifacts(alert: Alert, caze: Case)(implicit authContext: AuthContext): Future[Case] = { for { instanceConfig ← getInstanceConfig(alert.source()) - artifacts ← Future.sequence(alert.artifacts().flatMap(attributeToArtifact(instanceConfig, alert, _))) + artifacts ← Future.sequence(alert.artifacts().flatMap(attributeToArtifact(instanceConfig, _, alert.tlp()))) _ ← artifactSrv.create(caze, artifacts) } yield caze } @@ -408,7 +221,7 @@ class MispSrv @Inject() ( else { getInstanceConfig(alert.source()) .flatMap { mcfg ⇒ - getAttributes(mcfg, alert.sourceRef(), None) + getAttributesFromMisp(mcfg, alert.sourceRef(), None) } .map(alert → _) .recover { @@ -425,7 +238,7 @@ class MispSrv @Inject() ( .mapAsyncUnordered(5) { case (alert, artifacts) ⇒ logger.info(s"Updating alert ${alert.id}") - alertSrv.update(alert.id, Fields.empty.set("artifacts", JsArray(artifacts))) + alertSrv.update(alert.id, Fields.empty.set("artifacts", Json.toJson(artifacts))) .recover { case t ⇒ logger.error(s"Update alert ${alert.id} fail", t) } @@ -435,7 +248,7 @@ class MispSrv @Inject() ( } def extractMalwareAttachment(file: FileInputValue)(implicit authContext: AuthContext): FileInputValue = { - import scala.collection.JavaConversions._ + import scala.collection.JavaConverters._ try { val zipFile = new ZipFile(file.filepath.toFile) @@ -443,7 +256,7 @@ class MispSrv @Inject() ( zipFile.setPassword("infected") // Get the list of file headers from the zip file - val fileHeaders = zipFile.getFileHeaders.toList.asInstanceOf[List[FileHeader]] + val fileHeaders = zipFile.getFileHeaders.asScala.toList.asInstanceOf[List[FileHeader]] val (fileNameHeaders, contentFileHeaders) = fileHeaders.partition { fileHeader ⇒ fileHeader.getFileName.endsWith(".filename.txt") } @@ -477,31 +290,28 @@ class MispSrv @Inject() ( } } + private[MispSrv] val fileNameExtractor = """attachment; filename="(.*)"""".r def downloadAttachment( mispConnection: MispConnection, attachmentId: String)(implicit authContext: AuthContext): Future[FileInputValue] = { - val fileNameExtractor = """attachment; filename="(.*)"""".r mispConnection(s"attributes/download/$attachmentId") .withMethod("GET") .stream() .flatMap { - case response if response.headers.status != 200 ⇒ - val status = response.headers.status - response.body.runWith(Sink.headOption).flatMap { body ⇒ - val message = body.fold("")(_.decodeString("UTF-8")) - logger.warn(s"MISP attachment $attachmentId can't be downloaded (status $status) : $message") - Future.failed(InternalError(s"MISP attachment $attachmentId can't be downloaded (status $status)")) - } + case response if response.status != 200 ⇒ + val status = response.status + logger.warn(s"MISP attachment $attachmentId can't be downloaded (status $status) : ${response.body}") + Future.failed(InternalError(s"MISP attachment $attachmentId can't be downloaded (status $status)")) case response ⇒ val tempFile = tempSrv.newTemporaryFile("misp_attachment", attachmentId) - response.body + response.bodyAsSource .runWith(FileIO.toPath(tempFile)) .map { ioResult ⇒ if (!ioResult.wasSuccessful) // throw an exception if transfer failed throw ioResult.getError - val contentType = response.headers.headers.getOrElse("Content-Type", Seq("application/octet-stream")).head - val filename = response.headers.headers + val contentType = response.headers.getOrElse("Content-Type", Seq("application/octet-stream")).head + val filename = response.headers .get("Content-Disposition") .flatMap(_.collectFirst { case fileNameExtractor(name) ⇒ name }) .getOrElse("noname") @@ -509,171 +319,4 @@ class MispSrv @Inject() ( } } } - - def convertAttribute(mispAttribute: MispAttribute): Seq[JsObject] = { - val dataType = typeLookup.getOrElse(mispAttribute.tpe, "other") - val fields = Json.obj( - "data" → mispAttribute.value, - "dataType" → dataType, - "message" → mispAttribute.comment, - "startDate" → mispAttribute.date, - "tags" → Json.arr(s"MISP:type=${mispAttribute.tpe}", s"MISP:category=${mispAttribute.category}")) - - val types = mispAttribute.tpe.split('|').toSeq - if (types.length > 1) { - val values = mispAttribute.value.split('|').toSeq - val typesValues = types.zipAll(values, "noType", "noValue") - val additionnalMessage = typesValues - .map { case (t, v) ⇒ s"$t: $v" } - .mkString("\n") - typesValues.map { - case (tpe, value) ⇒ - fields + - ("dataType" → JsString(typeLookup.getOrElse(tpe, "other"))) + - ("data" → JsString(value)) + - ("message" → JsString(mispAttribute.comment + "\n" + additionnalMessage)) - } - } - else { - Seq(fields) - } - } - - private val typeLookup = Map( - "md5" → "hash", - "sha1" → "hash", - "sha256" → "hash", - "filename" → "filename", - "pdb" → "other", - "filename|md5" → "other", - "filename|sha1" → "other", - "filename|sha256" → "other", - "ip-src" → "ip", - "ip-dst" → "ip", - "hostname" → "fqdn", - "domain" → "domain", - "domain|ip" → "other", - "email-src" → "mail", - "email-dst" → "mail", - "email-subject" → "mail_subject", - "email-attachment" → "other", - "float" → "other", - "url" → "url", - "http-method" → "other", - "user-agent" → "user-agent", - "regkey" → "registry", - "regkey|value" → "registry", - "AS" → "other", - "snort" → "other", - "pattern-in-file" → "other", - "pattern-in-traffic" → "other", - "pattern-in-memory" → "other", - "yara" → "other", - "sigma" → "other", - "vulnerability" → "other", - "attachment" → "file", - "malware-sample" → "file", - "link" → "other", - "comment" → "other", - "text" → "other", - "hex" → "other", - "other" → "other", - "named" → "other", - "mutex" → "other", - "target-user" → "other", - "target-email" → "mail", - "target-machine" → "fqdn", - "target-org" → "other", - "target-location" → "other", - "target-external" → "other", - "btc" → "other", - "iban" → "other", - "bic" → "other", - "bank-account-nr" → "other", - "aba-rtn" → "other", - "bin" → "other", - "cc-number" → "other", - "prtn" → "other", - "threat-actor" → "other", - "campaign-name" → "other", - "campaign-id" → "other", - "malware-type" → "other", - "uri" → "uri_path", - "authentihash" → "other", - "ssdeep" → "hash", - "imphash" → "hash", - "pehash" → "hash", - "impfuzzy" → "hash", - "sha224" → "hash", - "sha384" → "hash", - "sha512" → "hash", - "sha512/224" → "hash", - "sha512/256" → "hash", - "tlsh" → "other", - "filename|authentihash" → "other", - "filename|ssdeep" → "other", - "filename|imphash" → "other", - "filename|impfuzzy" → "other", - "filename|pehash" → "other", - "filename|sha224" → "other", - "filename|sha384" → "other", - "filename|sha512" → "other", - "filename|sha512/224" → "other", - "filename|sha512/256" → "other", - "filename|tlsh" → "other", - "windows-scheduled-task" → "other", - "windows-service-name" → "other", - "windows-service-displayname" → "other", - "whois-registrant-email" → "mail", - "whois-registrant-phone" → "other", - "whois-registrant-name" → "other", - "whois-registrar" → "other", - "whois-creation-date" → "other", - "x509-fingerprint-sha1" → "other", - "dns-soa-email" → "other", - "size-in-bytes" → "other", - "counter" → "other", - "datetime" → "other", - "cpe" → "other", - "port" → "other", - "ip-dst|port" → "other", - "ip-src|port" → "other", - "hostname|port" → "other", - "email-dst-display-name" → "other", - "email-src-display-name" → "other", - "email-header" → "other", - "email-reply-to" → "other", - "email-x-mailer" → "other", - "email-mime-boundary" → "other", - "email-thread-index" → "other", - "email-message-id" → "other", - "github-username" → "other", - "github-repository" → "other", - "github-organisation" → "other", - "jabber-id" → "other", - "twitter-id" → "other", - "first-name" → "other", - "middle-name" → "other", - "last-name" → "other", - "date-of-birth" → "other", - "place-of-birth" → "other", - "gender" → "other", - "passport-number" → "other", - "passport-country" → "other", - "passport-expiration" → "other", - "redress-number" → "other", - "nationality" → "other", - "visa-number" → "other", - "issue-date-of-the-visa" → "other", - "primary-residence" → "other", - "country-of-residence" → "other", - "special-service-request" → "other", - "frequent-flyer-number" → "other", - "travel-details" → "other", - "payment-details" → "other", - "place-port-of-original-embarkation" → "other", - "place-port-of-clearance" → "other", - "place-port-of-onward-foreign-destination" → "other", - "passenger-name-record-locator-number" → "other", - "mobile-application-id" → "other") } diff --git a/thehive-misp/app/connectors/misp/MispSynchro.scala b/thehive-misp/app/connectors/misp/MispSynchro.scala new file mode 100644 index 0000000000..d9d4139774 --- /dev/null +++ b/thehive-misp/app/connectors/misp/MispSynchro.scala @@ -0,0 +1,186 @@ +package connectors.misp + +import java.util.Date +import javax.inject.{ Inject, Provider, Singleton } + +import scala.collection.immutable +import scala.concurrent.{ ExecutionContext, Future } +import scala.concurrent.duration._ +import scala.util.{ Failure, Success, Try } + +import play.api.Logger +import play.api.inject.ApplicationLifecycle +import play.api.libs.json._ + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.Materializer +import akka.stream.scaladsl.{ Sink, Source } +import connectors.misp.JsonFormat.mispArtifactWrites +import models.{ Alert, AlertStatus, Artifact, CaseStatus } +import services.{ AlertSrv, ArtifactSrv, CaseSrv, UserSrv } +import JsonFormat.mispAlertWrites + +import org.elastic4play.controllers.Fields +import org.elastic4play.services.{ Attachment, AuthContext, MigrationSrv, TempSrv } + +@Singleton +class MispSynchro @Inject() ( + mispConfig: MispConfig, + migrationSrv: MigrationSrv, + mispSrv: MispSrv, + caseSrv: CaseSrv, + artifactSrv: ArtifactSrv, + alertSrvProvider: Provider[AlertSrv], + userSrv: UserSrv, + tempSrv: TempSrv, + lifecycle: ApplicationLifecycle, + system: ActorSystem, + implicit val ec: ExecutionContext, + implicit val mat: Materializer) { + + private[misp] lazy val logger = Logger(getClass) + private[misp] lazy val alertSrv = alertSrvProvider.get + + private[misp] def initScheduler(): Unit = { + val task = system.scheduler.schedule(0.seconds, mispConfig.interval) { + if (migrationSrv.isReady) { + logger.info("Update of MISP events is starting ...") + userSrv + .inInitAuthContext { implicit authContext ⇒ + synchronize().andThen { case _ ⇒ tempSrv.releaseTemporaryFiles() } + } + .onComplete { + case Success(a) ⇒ + logger.info("Misp synchronization completed") + a.collect { + case Failure(t) ⇒ logger.warn(s"Update MISP error", t) + } + case Failure(t) ⇒ logger.info("Misp synchronization failed", t) + } + } + else { + logger.info("MISP synchronization cancel, database is not ready") + } + } + lifecycle.addStopHook { () ⇒ + logger.info("Stopping MISP fetching ...") + task.cancel() + Future.successful(()) + } + } + + initScheduler() + + def synchronize()(implicit authContext: AuthContext): Future[Seq[Try[Alert]]] = { + import org.elastic4play.services.QueryDSL._ + + // for each MISP server + Source(mispConfig.connections.toList) + // get last synchronization + .mapAsyncUnordered(1) { mispConnection ⇒ + alertSrv.stats(and("type" ~= "misp", "source" ~= mispConnection.name), Seq(selectMax("lastSyncDate"))) + .map { maxLastSyncDate ⇒ mispConnection → new Date((maxLastSyncDate \ "max_lastSyncDate").as[Long]) } + .recover { case _ ⇒ mispConnection → new Date(0) } + } + .flatMapConcat { + case (mispConnection, lastSyncDate) ⇒ + synchronize(mispConnection, Some(lastSyncDate)) + } + .runWith(Sink.seq) + } + + def fullSynchronize()(implicit authContext: AuthContext): Future[immutable.Seq[Try[Alert]]] = { + Source(mispConfig.connections.toList) + .flatMapConcat(mispConnection ⇒ synchronize(mispConnection, None)) + .runWith(Sink.seq) + } + + def updateArtifacts(mispConnection: MispConnection, caseId: String, mispArtifacts: Seq[MispArtifact])(implicit authContext: AuthContext): Future[Seq[Try[Artifact]]] = { + import org.elastic4play.services.QueryDSL._ + + for { + // Either data or filename + existingArtifacts: Seq[Either[String, String]] ← artifactSrv.find(and(withParent("case", caseId), "status" ~= "Ok"), Some("all"), Nil)._1.map { artifact ⇒ + artifact.data().map(Left.apply).getOrElse(Right(artifact.attachment().get.name)) + } + .runWith(Sink.seq) + newAttributes ← Future.traverse(mispArtifacts) { + case artifact @ MispArtifact(SimpleArtifactData(data), _, _, _, _, _) if !existingArtifacts.contains(Right(data)) ⇒ Future.successful(Fields(Json.toJson(artifact).as[JsObject])) + case artifact @ MispArtifact(AttachmentArtifact(Attachment(filename, _, _, _, _)), _, _, _, _, _) if !existingArtifacts.contains(Left(filename)) ⇒ Future.successful(Fields(Json.toJson(artifact).as[JsObject])) + case artifact @ MispArtifact(RemoteAttachmentArtifact(filename, reference, tpe), _, _, _, _, _) if !existingArtifacts.contains(Left(filename)) ⇒ + mispSrv.downloadAttachment(mispConnection, reference) + .map { + case fiv if tpe == "malware-sample" ⇒ mispSrv.extractMalwareAttachment(fiv) + case fiv ⇒ fiv + } + .map(fiv ⇒ Fields(Json.toJson(artifact).as[JsObject]).unset("remoteAttachment").set("attachment", fiv)) + case _ ⇒ Future.successful(Fields.empty) + } + createdArtifacts ← artifactSrv.create(caseId, newAttributes.filterNot(_.isEmpty)) + } yield createdArtifacts + } + + def synchronize(mispConnection: MispConnection, lastSyncDate: Option[Date])(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = { + logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate") + // get events that have been published after the last synchronization + mispSrv.getEventsFromDate(mispConnection, lastSyncDate.getOrElse(new Date(0))) + // get related alert + .mapAsyncUnordered(1) { event ⇒ + logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}") + alertSrv.get("misp", event.source, event.sourceRef) + .map((event, _)) + } + .mapAsyncUnordered(1) { + case (event, alert) ⇒ + logger.trace(s"MISP synchro ${mispConnection.name}, event ${event.sourceRef}, alert ${alert.fold("no alert")(a ⇒ "alert " + a.alertId() + "last sync at " + a.lastSyncDate())}") + logger.info(s"getting MISP event ${event.source}:${event.sourceRef}") + mispSrv.getAttributesFromMisp(mispConnection, event.sourceRef, lastSyncDate.flatMap(_ ⇒ alert.map(_.lastSyncDate()))) + .map((event, alert, _)) + } + .mapAsyncUnordered(1) { + // if there is no related alert, create a new one + case (event, None, attrs) ⇒ + logger.info(s"MISP event ${event.source}:${event.sourceRef} has no related alert, create it with ${attrs.size} observable(s)") + val alertJson = Json.toJson(event).as[JsObject] + + ("type" → JsString("misp")) + + ("caseTemplate" → mispConnection.caseTemplate.fold[JsValue](JsNull)(JsString)) + + ("artifacts" → Json.toJson(attrs)) + alertSrv.create(Fields(alertJson)) + .map(Success(_)) + .recover { case t ⇒ Failure(t) } + + case (event, Some(alert), attrs) ⇒ + logger.info(s"MISP event ${event.source}:${event.sourceRef} has related alert, update it with ${attrs.size} observable(s)") + + alert.caze().fold[Future[Boolean]](Future.successful(lastSyncDate.isDefined && attrs.nonEmpty && alert.follow())) { + case caze if alert.follow() ⇒ + for { + addedArtifacts ← updateArtifacts(mispConnection, caze, attrs) + updateStatus = lastSyncDate.nonEmpty && addedArtifacts.exists(_.isSuccess) + _ ← if (updateStatus) caseSrv.update(caze, Fields.empty.set("status", CaseStatus.Open.toString)) else Future.successful(()) + } yield updateStatus + case _ ⇒ Future.successful(false) + } + .flatMap { updateStatus ⇒ + val artifacts = JsArray(alert.artifacts() ++ attrs.map(Json.toJson(_))) + val alertJson = Json.toJson(event).as[JsObject] - + "type" - + "source" - + "sourceRef" - + "caseTemplate" - + "date" + + ("artifacts" → artifacts) + + ("status" → (if (!updateStatus) Json.toJson(alert.status()) + else alert.status() match { + case AlertStatus.New ⇒ Json.toJson(AlertStatus.New) + case _ ⇒ Json.toJson(AlertStatus.Updated) + })) + logger.debug(s"Update alert ${alert.id} with\n$alertJson") + alertSrv.update(alert.id, Fields(alertJson)) + } + .map(Success(_)) + .recover { case t ⇒ Failure(t) } + } + } +} diff --git a/thehive-misp/app/connectors/misp/UpdateMispAlertArtifactActor.scala b/thehive-misp/app/connectors/misp/UpdateMispAlertArtifactActor.scala new file mode 100644 index 0000000000..ba6989cb15 --- /dev/null +++ b/thehive-misp/app/connectors/misp/UpdateMispAlertArtifactActor.scala @@ -0,0 +1,58 @@ +package connectors.misp + +import javax.inject.{ Inject, Singleton } + +import scala.concurrent.ExecutionContext +import scala.util.{ Failure, Success } + +import play.api.Logger + +import akka.actor.Actor +import models.UpdateMispAlertArtifact +import services.UserSrv + +import org.elastic4play.services.EventSrv + +/** + * This actor listens message from migration (message UpdateMispAlertArtifact) which indicates that artifacts in + * MISP event must be retrieved in inserted in alerts. + * + * @param eventSrv event bus used to receive migration message + * @param userSrv user service used to do operations on database without real user request + * @param mispSrv misp service to invoke artifact update action + * @param ec execution context + */ +@Singleton +class UpdateMispAlertArtifactActor @Inject() ( + eventSrv: EventSrv, + userSrv: UserSrv, + mispSrv: MispSrv, + implicit val ec: ExecutionContext) extends Actor { + + private[UpdateMispAlertArtifactActor] lazy val logger = Logger(getClass) + override def preStart(): Unit = { + eventSrv.subscribe(self, classOf[UpdateMispAlertArtifact]) + super.preStart() + } + + override def postStop(): Unit = { + eventSrv.unsubscribe(self) + super.postStop() + } + + override def receive: Receive = { + case UpdateMispAlertArtifact() ⇒ + logger.info("UpdateMispAlertArtifact") + userSrv + .inInitAuthContext { implicit authContext ⇒ + mispSrv.updateMispAlertArtifact() + } + .onComplete { + case Success(_) ⇒ logger.info("Artifacts in MISP alerts updated") + case Failure(error) ⇒ logger.error("Update MISP alert artifacts error :", error) + } + () + case msg ⇒ + logger.info(s"Receiving unexpected message: $msg (${msg.getClass})") + } +} \ No newline at end of file diff --git a/thehive-misp/build.sbt b/thehive-misp/build.sbt index f7064e3fc1..969ef9384d 100644 --- a/thehive-misp/build.sbt +++ b/thehive-misp/build.sbt @@ -2,8 +2,8 @@ import Dependencies._ libraryDependencies ++= Seq( Library.Play.ws, + Library.Play.guice, + Library.Play.ahc, Library.zip4j, Library.elastic4play -) - -enablePlugins(PlayScala) +) \ No newline at end of file diff --git a/ui/app/index.html b/ui/app/index.html index cf277ef6a5..74a0f6a452 100644 --- a/ui/app/index.html +++ b/ui/app/index.html @@ -140,6 +140,7 @@ + @@ -147,6 +148,7 @@ + @@ -161,6 +163,7 @@ + diff --git a/ui/app/scripts/controllers/admin/AdminUserDialogCtrl.js b/ui/app/scripts/controllers/admin/AdminUserDialogCtrl.js new file mode 100644 index 0000000000..f4c234e0d2 --- /dev/null +++ b/ui/app/scripts/controllers/admin/AdminUserDialogCtrl.js @@ -0,0 +1,68 @@ +(function() { + 'use strict'; + + angular.module('theHiveControllers').controller('AdminUserDialogCtrl', function($scope, $uibModalInstance, UserSrv, NotificationSrv, user) { + var self = this; + + self.user = user; + self.isEdit = user.id; + + var formData = _.defaults(_.pick(self.user, 'id', 'name', 'roles'), { + id: null, + name: null, + roles: [], + alert: false + }); + formData.alert = formData.roles.indexOf('alert') !== -1; + formData.roles = _.without(formData.roles, 'alert'); + + self.formData = formData; + + var onSuccess = function(data) { + $uibModalInstance.close(data); + }; + + var onFailure = function(response) { + NotificationSrv.error('AdminUserDialogCtrl', response.data, response.status); + }; + + var buildRoles = function(roles, alert) { + var result = angular.copy(roles) || []; + + if(alert && roles.indexOf('alert') === -1) { + result.push('alert'); + } else if (!alert && roles.indexOf('alert') !== -1) { + result = _.omit(result, 'alert'); + } + + return result; + }; + + self.saveUser = function(form) { + if (!form.$valid) { + return; + } + + var postData = {}; + + if (self.user.id) { + postData = { + name: self.formData.name, + roles: buildRoles(self.formData.roles, self.formData.alert) + }; + UserSrv.update({'userId': self.user.id}, postData, onSuccess, onFailure); + } else { + postData = { + login: angular.lowercase(self.formData.id), + name: self.formData.name, + roles: buildRoles(self.formData.roles, self.formData.alert) + }; + UserSrv.save(postData, onSuccess, onFailure); + } + }; + + self.cancel = function() { + $uibModalInstance.dismiss(); + } + }); +})(); diff --git a/ui/app/scripts/controllers/admin/AdminUsersCtrl.js b/ui/app/scripts/controllers/admin/AdminUsersCtrl.js index 26773bf493..3c0a0e8182 100644 --- a/ui/app/scripts/controllers/admin/AdminUsersCtrl.js +++ b/ui/app/scripts/controllers/admin/AdminUsersCtrl.js @@ -2,7 +2,7 @@ 'use strict'; angular.module('theHiveControllers').controller('AdminUsersCtrl', - function($scope, PSearchSrv, UserSrv, NotificationSrv, appConfig) { + function($scope, $uibModal, PSearchSrv, UserSrv, NotificationSrv, clipboard, appConfig) { $scope.appConfig = appConfig; $scope.canSetPass = appConfig.config.capabilities.indexOf('setPassword') !== -1; $scope.newUser = { @@ -48,15 +48,35 @@ $scope.usrKey = {}; $scope.getKey = function(user) { - UserSrv.get({ + UserSrv.getKey(user.id) + .then(function(key) { + $scope.usrKey[user.id] = key; + }); + + }; + $scope.createKey = function(user) { + UserSrv.setKey({ userId: user.id - }, function(usr) { - $scope.usrKey[user.id] = usr.key; + },{}, function() { + delete $scope.usrKey[user.id]; + }, function(response) { + NotificationSrv.error('AdminUsersCtrl', response.data, response.status); }); + }; + $scope.revokeKey = function(user) { + UserSrv.revokeKey({ + userId: user.id + },{}, function() { + delete $scope.usrKey[user.id]; + }, function(response) { + NotificationSrv.error('AdminUsersCtrl', response.data, response.status); + }); }; - $scope.createKey = function(user) { - $scope.updateField(user, 'with-key', true); + + $scope.copyKey = function(user) { + clipboard.copyText($scope.usrKey[user.id]); + delete $scope.usrKey[user.id]; }; $scope.updateField = function(user, fieldName, newValue) { @@ -86,6 +106,24 @@ }); }; + $scope.showUserDialog = function(user) { + var modalInstance = $uibModal.open({ + templateUrl: 'views/partials/admin/user-dialog.html', + controller: 'AdminUserDialogCtrl', + controllerAs: '$vm', + size: 'lg', + resolve: { + user: angular.copy(user) || {} + } + }); + + modalInstance.result.then(function(data) { + //self.initCustomfields(); + //CustomFieldsCacheSrv.clearCache(); + //$scope.$emit('custom-fields:refresh'); + }); + } + }); })(); diff --git a/ui/app/scripts/controllers/alert/AlertListCtrl.js b/ui/app/scripts/controllers/alert/AlertListCtrl.js index bb047aa2fa..843b956fbc 100644 --- a/ui/app/scripts/controllers/alert/AlertListCtrl.js +++ b/ui/app/scripts/controllers/alert/AlertListCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers') - .controller('AlertListCtrl', function($scope, $q, $state, $uibModal, TemplateSrv, AlertingSrv, NotificationSrv, FilteringSrv, Severity) { + .controller('AlertListCtrl', function($scope, $q, $state, $uibModal, TagSrv, TemplateSrv, AlertingSrv, NotificationSrv, FilteringSrv, Severity) { var self = this; self.list = []; @@ -93,6 +93,7 @@ self.searchForm = { searchQuery: self.filtering.buildQuery() || '' }; + self.lastSearch = null; $scope.$watch('$vm.list.pageSize', function (newValue) { self.filtering.setPageSize(newValue); @@ -275,7 +276,11 @@ this.applyFilters = function () { self.searchForm.searchQuery = self.filtering.buildQuery(); - self.search(); + + if(self.lastSearch !== self.searchForm.searchQuery) { + self.lastSearch = self.searchForm.searchQuery; + self.search(); + } }; this.clearFilters = function () { @@ -372,6 +377,10 @@ return AlertingSrv.sources(query); }; + this.getTags = function(query) { + return TagSrv.fromAlerts(query); + }; + self.load(); }); })(); diff --git a/ui/app/scripts/controllers/case/CaseExportDialogCtrl.js b/ui/app/scripts/controllers/case/CaseExportDialogCtrl.js new file mode 100644 index 0000000000..eeaaa4c4e6 --- /dev/null +++ b/ui/app/scripts/controllers/case/CaseExportDialogCtrl.js @@ -0,0 +1,88 @@ +(function() { + 'use strict'; + + angular + .module('theHiveControllers') + .controller('CaseExportDialogCtrl', function(MispSrv, NotificationSrv, clipboard, $uibModalInstance, caze, config) { + var self = this; + + this.caze = caze; + this.mode = ''; + this.servers = config.servers; + this.failures = []; + + this.existingExports = {}; + this.loading = false; + + _.each(_.filter(this.caze.stats.alerts || [], function(item) { + return item.type === 'misp'; + }), function(item) { + self.existingExports[item.source] = true; + }); + + var extractExportErrors = function (errors) { + var result = []; + + result = errors.map(function(item) { + return { + data: item.object.dataType === 'file' ? item.object.attachment.name : item.object.data, + message: item.message + }; + }); + + return result; + } + + this.copyToClipboard = function() { + clipboard.copyText(_.pluck(self.failures, 'data').join('\n')); + $uibModalInstance.dismiss(); + } + + this.cancel = function() { + $uibModalInstance.dismiss(); + }; + + this.confirm = function() { + $uibModalInstance.close(); + }; + + this.export = function(server) { + self.loading = true; + self.failures = []; + + MispSrv.export(self.caze.id, server) + .then(function(response){ + var success = 0, + failure = 0; + + if (response.status === 207) { + success = response.data.success.length; + failure = response.data.failure.length; + + self.mode = 'error'; + self.failures = extractExportErrors(response.data.failure); + + NotificationSrv.log('The case has been successfully exported, but '+ failure +' observable(s) failed', 'warning'); + } else { + success = angular.isObject(response.data) ? 1 : response.data.length; + NotificationSrv.log('The case has been successfully exported with ' + success+ ' observable(s)', 'success'); + $uibModalInstance.close(); + } + self.loading = false; + + }, function(err) { + if(!err) { + return; + } + + if (err.status === 400) { + self.mode = 'error'; + self.failures = extractExportErrors(err.data); + } else { + NotificationSrv.error('CaseExportCtrl', 'An unexpected error occurred while exporting case', err.status); + } + self.loading = false; + }); + } + }); +})(); diff --git a/ui/app/scripts/controllers/case/CaseListCtrl.js b/ui/app/scripts/controllers/case/CaseListCtrl.js index f1d46a3bfa..6c5c1879de 100644 --- a/ui/app/scripts/controllers/case/CaseListCtrl.js +++ b/ui/app/scripts/controllers/case/CaseListCtrl.js @@ -15,6 +15,7 @@ this.searchForm = { searchQuery: this.uiSrv.buildQuery() || '' }; + this.lastQuery = null; this.list = PSearchSrv(undefined, 'case', { scope: $scope, @@ -36,7 +37,6 @@ field: 'status' }); - $scope.$watch('$vm.list.pageSize', function (newValue) { self.uiSrv.setPageSize(newValue); }); @@ -55,7 +55,12 @@ this.applyFilters = function () { self.searchForm.searchQuery = self.uiSrv.buildQuery(); - self.search(); + + if(self.lastQuery !== self.searchForm.searchQuery) { + self.lastQuery = self.searchForm.searchQuery; + self.search(); + } + }; this.clearFilters = function () { diff --git a/ui/app/scripts/controllers/case/CaseMainCtrl.js b/ui/app/scripts/controllers/case/CaseMainCtrl.js index f8ab031bcf..dc97fff5a8 100644 --- a/ui/app/scripts/controllers/case/CaseMainCtrl.js +++ b/ui/app/scripts/controllers/case/CaseMainCtrl.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveControllers').controller('CaseMainCtrl', - function($scope, $rootScope, $state, $stateParams, $q, $uibModal, CaseTabsSrv, CaseSrv, MetricsCacheSrv, UserInfoSrv, StreamStatSrv, NotificationSrv, UtilsSrv, CaseResolutionStatus, CaseImpactStatus, caze) { + function($scope, $rootScope, $state, $stateParams, $q, $uibModal, CaseTabsSrv, CaseSrv, MetricsCacheSrv, UserInfoSrv, MispSrv, StreamStatSrv, NotificationSrv, UtilsSrv, CaseResolutionStatus, CaseImpactStatus, caze) { $scope.CaseResolutionStatus = CaseResolutionStatus; $scope.CaseImpactStatus = CaseImpactStatus; @@ -26,6 +26,13 @@ $scope.caze = caze; $rootScope.title = 'Case #' + caze.caseId + ': ' + caze.title; + $scope.initExports = function() { + $scope.existingExports = _.filter($scope.caze.stats.alerts || [], function(item) { + return item.type === 'misp'; + }).length; + }; + $scope.initExports(); + $scope.updateMetricsList = function() { MetricsCacheSrv.all().then(function(metrics) { $scope.allMetrics = _.omit(metrics, _.keys($scope.caze.metrics)); @@ -198,6 +205,33 @@ }); }; + $scope.shareCase = function() { + var modalInstance = $uibModal.open({ + templateUrl: 'views/partials/misp/case.export.confirm.html', + controller: 'CaseExportDialogCtrl', + controllerAs: 'dialog', + size: 'lg', + resolve: { + caze: function() { + return $scope.caze; + }, + config: function() { + return $scope.appConfig.connectors.misp; + } + } + }); + + modalInstance.result.then(function() { + return CaseSrv.get({ + 'caseId': $scope.caseId, + 'nstats': true + }).$promise; + }).then(function(data) { + $scope.caze = data.toJSON(); + $scope.initExports(); + }) + }; + /** * A workaround filter to make sure the ngRepeat doesn't order the * object keys diff --git a/ui/app/scripts/controllers/case/ObservableCreationCtrl.js b/ui/app/scripts/controllers/case/ObservableCreationCtrl.js index 3fc95327e4..41665bfecc 100644 --- a/ui/app/scripts/controllers/case/ObservableCreationCtrl.js +++ b/ui/app/scripts/controllers/case/ObservableCreationCtrl.js @@ -102,7 +102,7 @@ return _.map(failures, function(observable) { return { - data: observable.object.data, + data: observable.object.dataType === 'file' ? observable.object.attachment.name : observable.object.data, type: observable.type }; }); diff --git a/ui/app/scripts/controllers/misc/ServerInstanceDialogCtrl.js b/ui/app/scripts/controllers/misc/ServerInstanceDialogCtrl.js new file mode 100644 index 0000000000..1e86095a85 --- /dev/null +++ b/ui/app/scripts/controllers/misc/ServerInstanceDialogCtrl.js @@ -0,0 +1,20 @@ +(function() { + 'use strict'; + angular.module('theHiveControllers') + .controller('ServerInstanceDialogCtrl', ServerInstanceDialogCtrl); + + function ServerInstanceDialogCtrl($uibModalInstance, servers) { + var self = this; + + this.servers = servers; + this.selected = null; + + this.ok = function() { + $uibModalInstance.close(this.selected); + }; + + this.cancel = function() { + $uibModalInstance.dismiss(); + }; + } +})(); diff --git a/ui/app/scripts/services/CortexSrv.js b/ui/app/scripts/services/CortexSrv.js index 0dd722c419..71b5108c4a 100644 --- a/ui/app/scripts/services/CortexSrv.js +++ b/ui/app/scripts/services/CortexSrv.js @@ -66,7 +66,7 @@ promptForInstance: function(servers) { var modalInstance = $uibModal.open({ templateUrl: 'views/partials/cortex/choose-instance-dialog.html', - controller: 'CortexInstanceDialogCtrl', + controller: 'ServerInstanceDialogCtrl', controllerAs: 'vm', size: '', resolve: { diff --git a/ui/app/scripts/services/MispSrv.js b/ui/app/scripts/services/MispSrv.js index 2c38651928..1779e99cc5 100644 --- a/ui/app/scripts/services/MispSrv.js +++ b/ui/app/scripts/services/MispSrv.js @@ -1,7 +1,7 @@ (function() { 'use strict'; angular.module('theHiveServices') - .factory('MispSrv', function($q, $http, $rootScope, StatSrv, StreamSrv, PSearchSrv) { + .factory('MispSrv', function($q, $http, $rootScope, $uibModal, StatSrv, StreamSrv, PSearchSrv) { var baseUrl = './api/connector/misp'; @@ -116,6 +116,10 @@ }); return defer.promise; + }, + + export: function(caseId, server) { + return $http.post(baseUrl + '/export/' + caseId + '/' + server, {}); } }; diff --git a/ui/app/scripts/services/TagSrv.js b/ui/app/scripts/services/TagSrv.js index 2572c3097e..2d60d2959f 100644 --- a/ui/app/scripts/services/TagSrv.js +++ b/ui/app/scripts/services/TagSrv.js @@ -3,27 +3,40 @@ angular.module('theHiveServices') .service('TagSrv', function(StatSrv, $q) { - this.fromCases = function(query) { - var defer = $q.defer(); - - StatSrv.getPromise({ - objectType: 'case', + var getPromiseFor = function(objectType) { + return StatSrv.getPromise({ + objectType: objectType, field: 'tags', limit: 1000 - }).then(function(response) { - var tags = []; + }); + }; + + var mapTags = function(collection, term) { + return _.map(_.filter(_.keys(collection), function(tag) { + var regex = new RegExp(term, 'gi'); + return regex.test(tag); + }), function(tag) { + return {text: tag}; + }); + }; - tags = _.map(_.filter(_.keys(response.data), function(tag) { - var regex = new RegExp(query, 'gi'); - return regex.test(tag); - }), function(tag) { - return {text: tag}; - }); + var getTags = function(objectType, term) { + var defer = $q.defer(); - defer.resolve(tags); + getPromiseFor(objectType).then(function(response) { + defer.resolve(mapTags(response.data, term) || []); }); return defer.promise; + } + + + this.fromCases = function(term) { + return getTags('case', term); + }; + + this.fromAlerts = function(term) { + return getTags('alert', term); }; }); diff --git a/ui/app/scripts/services/UserSrv.js b/ui/app/scripts/services/UserSrv.js index 3e2296b0e2..e5ce4b6fb0 100644 --- a/ui/app/scripts/services/UserSrv.js +++ b/ui/app/scripts/services/UserSrv.js @@ -17,6 +17,18 @@ angular.module('theHiveServices') setPass: { method: 'POST', url: './api/user/:userId/password/set' + }, + // getKey: { + // method: 'GET', + // url: './api/user/:userId/key' + // }, + setKey: { + method: 'POST', + url: './api/user/:userId/key/renew' + }, + revokeKey: { + method: 'DELETE', + url: './api/user/:userId/key' } }); /** @@ -70,6 +82,17 @@ angular.module('theHiveServices') return defer.promise; }; + res.getKey = function(userId) { + var defer = $q.defer(); + + $http.get('./api/user/'+userId+'/key') + .then(function(response) { + defer.resolve(response.data); + }); + + return defer.promise; + }; + res.list = function(query) { var defer = $q.defer(); diff --git a/ui/app/views/directives/flow/alert.html b/ui/app/views/directives/flow/alert.html index d7c477572e..daffaa1c24 100644 --- a/ui/app/views/directives/flow/alert.html +++ b/ui/app/views/directives/flow/alert.html @@ -2,7 +2,7 @@
Alert updates - {{base.object.title}} + [{{base.object.source}}] {{base.object.title}}
@@ -18,14 +18,31 @@
- +
{{k}}: {{v.length}}
+
+ {{k}}: +
+
+ {{k}}: + + {{tag}} + + + None + +
+
+ {{k}}: +
{{k}}: - {{v}} + {{v | limitTo: 250}}
diff --git a/ui/app/views/partials/admin/user-dialog.html b/ui/app/views/partials/admin/user-dialog.html new file mode 100644 index 0000000000..9cc4f2a303 --- /dev/null +++ b/ui/app/views/partials/admin/user-dialog.html @@ -0,0 +1,75 @@ +
+
+

{{$vm.formData.id ? 'Update' : 'Add'}} user

+
+
+
+ +
+ +

This field is required.

+
+
+ +
+ +
+ +

This field is required.

+
+
+ +
+ +
+
+ + +
+
+
+ +
+ +
+
+ +
+
+
+
+
+ + +
+
diff --git a/ui/app/views/partials/admin/users.html b/ui/app/views/partials/admin/users.html index 0eabd04488..547badb5c6 100644 --- a/ui/app/views/partials/admin/users.html +++ b/ui/app/views/partials/admin/users.html @@ -3,116 +3,81 @@

User management

-
-
-
-
- -
-
- -
-
- - API key -
-
- -
-
- Roles -
- - -
-
-
-
-
- Add user -
-
-
-
+ + + +
- +
- - - - - + + + + + + - - - - + + - - diff --git a/ui/app/views/partials/alert/event.dialog.html b/ui/app/views/partials/alert/event.dialog.html index 587256bdde..d3ba65256a 100644 --- a/ui/app/views/partials/alert/event.dialog.html +++ b/ui/app/views/partials/alert/event.dialog.html @@ -35,7 +35,7 @@

- None + None {{tag}}
diff --git a/ui/app/views/partials/alert/list.html b/ui/app/views/partials/alert/list.html index e083799f1e..6864d17e71 100644 --- a/ui/app/views/partials/alert/list.html +++ b/ui/app/views/partials/alert/list.html @@ -44,7 +44,7 @@

List of alerts ({{$vm.list.total || 0}} of {{alertEvents.c

- + @@ -61,7 +61,7 @@

List of alerts ({{$vm.list.total || 0}} of {{alertEvents.c

-
LoginFull NameRolesPassword / API keyLockLoginFull NameRolesPasswordAPI keyActions
- {{user.id}} + + {{user.id}} + -
- - -
-
- -
+ 		+ + New password
-
- - Create API Key - Show API Key - {{usrKey[user.id]}} + 		+ + Create API Key + + +
+ + Renew + Revoke + Reveal + + + + + +
+ 		-
- - - - - -
+ 		+ + +
ReferenceReference Type Status Title + {{::event.sourceRef}} diff --git a/ui/app/views/partials/alert/list/filters.html b/ui/app/views/partials/alert/list/filters.html index 44a5bba987..b087c197d2 100644 --- a/ui/app/views/partials/alert/list/filters.html +++ b/ui/app/views/partials/alert/list/filters.html @@ -31,8 +31,9 @@

Filters

min-length="2" ng-model="$vm.filtering.activeFilters.status.value" placeholder="ex: New, Updated, Ignored, Imported" - replace-spaces-with-dashes="false"> - + replace-spaces-with-dashes="false" + add-from-autocomplete-only="true"> + @@ -55,8 +56,9 @@

Filters

min-length="2" ng-model="$vm.filtering.activeFilters.source.value" placeholder="ex: CIRCL, OSINT" - replace-spaces-with-dashes="false"> - + replace-spaces-with-dashes="false" + add-from-autocomplete-only="true"> + @@ -69,8 +71,9 @@

Filters

min-length="2" ng-model="$vm.filtering.activeFilters.severity.value" placeholder="ex: High, Medium, Low" - replace-spaces-with-dashes="false"> - + replace-spaces-with-dashes="false" + add-from-autocomplete-only="true"> + diff --git a/ui/app/views/partials/case/case.export.html b/ui/app/views/partials/case/case.export.html new file mode 100644 index 0000000000..21267a5e1b --- /dev/null +++ b/ui/app/views/partials/case/case.export.html @@ -0,0 +1,79 @@ +
+
+
No records.
+
+
+ + +
+
+ + + +
+
+ + +
+
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + TypeData/FilenameTagsMISP CategoryMISP Type 
+ + + + {{(artifact.data | fang) || (artifact.attachment.name | fang)}} + + + + + + + Not specified + Not specified + + Not specified + Not specified + + + +
+ +
+
diff --git a/ui/app/views/partials/case/case.panelinfo.html b/ui/app/views/partials/case/case.panelinfo.html index 88e4d41a60..46d815771d 100644 --- a/ui/app/views/partials/case/case.panelinfo.html +++ b/ui/app/views/partials/case/case.panelinfo.html @@ -37,6 +37,13 @@

+ + + + Share ({{existingExports}}) + + + diff --git a/ui/app/views/partials/case/list/filters.html b/ui/app/views/partials/case/list/filters.html index 2751cc7cde..d1db14b3b1 100644 --- a/ui/app/views/partials/case/list/filters.html +++ b/ui/app/views/partials/case/list/filters.html @@ -31,8 +31,9 @@

Filters

min-length="2" ng-model="$vm.uiSrv.activeFilters.status.value" placeholder="ex: Open, Resolved" - replace-spaces-with-dashes="false"> - + replace-spaces-with-dashes="false" + add-from-autocomplete-only="true"> + @@ -56,6 +57,7 @@

Filters

ng-model="$vm.uiSrv.activeFilters.owner.value" placeholder="ex: Firstname Lastname" replace-spaces-with-dashes="false" + add-from-autocomplete-only="true" display-property="label"> @@ -70,8 +72,9 @@

Filters

min-length="2" ng-model="$vm.uiSrv.activeFilters.severity.value" placeholder="ex: High, Medium, Low" - replace-spaces-with-dashes="false"> - + replace-spaces-with-dashes="false" + add-from-autocomplete-only="true"> + diff --git a/ui/app/views/partials/misp/case.export.confirm.html b/ui/app/views/partials/misp/case.export.confirm.html new file mode 100644 index 0000000000..b971b586c7 --- /dev/null +++ b/ui/app/views/partials/misp/case.export.confirm.html @@ -0,0 +1,48 @@ +
+

MISP Export

+
+
+
+

+ You are about to export the case {{dialog.caze.title}} to on of the following MISP servers: +

+ + + + + + + + +

{{server}}

 + +
+
+ +
+
+ Failed to export the following observables +
+ + + + + + + + + + + +
ObservableReason
{{o.data | fang}}{{o.message}}
+
+
+
+ + +
diff --git a/ui/app/views/partials/misp/choose-instance-dialog.html b/ui/app/views/partials/misp/choose-instance-dialog.html new file mode 100644 index 0000000000..1315ad0a04 --- /dev/null +++ b/ui/app/views/partials/misp/choose-instance-dialog.html @@ -0,0 +1,22 @@ +
+
+

Select MISP server

+
+
+
+ +
+ +
+
+
+
+ + +
+
diff --git a/ui/bower.json b/ui/bower.json index f5b2636fcb..747d0c8017 100644 --- a/ui/bower.json +++ b/ui/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "2.12.1", + "version": "2.13.0", "license": "AGPL-3.0", "dependencies": { "angular": "1.5.8", diff --git a/ui/package.json b/ui/package.json index f32f835bb9..5d500c67fc 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "2.12.1", + "version": "2.13.0", "license": "AGPL-3.0", "repository": { "type": "git", diff --git a/version.sbt b/version.sbt index e0fc553286..b6764391ec 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "2.12.1" +version in ThisBuild := "2.13.0"