diff --git a/ScalliGraph b/ScalliGraph index c889571c81..2b4d10fd55 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit c889571c8184ecb3966c8edc5b33e046c824b98b +Subproject commit 2b4d10fd55bb73e48087c2a21471651eebc9d25b diff --git a/build.sbt b/build.sbt index d3fb4daeb0..13d0ce7654 100644 --- a/build.sbt +++ b/build.sbt @@ -62,7 +62,7 @@ libraryDependencies in ThisBuild ++= { } dependencyOverrides in ThisBuild ++= Seq( // "org.locationtech.spatial4j" % "spatial4j" % "0.6", - "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2" +// "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2" ) PlayKeys.includeDocumentationInBinary := false milestoneFilter := ((milestone: Milestone) => milestone.title.startsWith("4")) @@ -337,12 +337,13 @@ lazy val thehiveMigration = (project in file("migration")) libraryDependencies ++= Seq( elastic4sCore, elastic4sHttpStreams, - elastic4sHttp, + elastic4sClient, // jts, ehcache, scopt, specs % Test ), + dependencyOverrides += akkaActor, fork := true, normalizedName := "migrate" ) diff --git a/migration/src/main/scala/org/thp/thehive/migration/dto/InputAlert.scala b/migration/src/main/scala/org/thp/thehive/migration/dto/InputAlert.scala index 1228b0cd48..7ab3dc1484 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/dto/InputAlert.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/dto/InputAlert.scala @@ -7,7 +7,6 @@ case class InputAlert( alert: Alert, caseId: Option[String], organisation: String, - tags: Set[String], customFields: Map[String, Option[Any]], caseTemplate: Option[String] ) { diff --git a/migration/src/main/scala/org/thp/thehive/migration/dto/InputCase.scala b/migration/src/main/scala/org/thp/thehive/migration/dto/InputCase.scala index bb87636698..23b035b2cb 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/dto/InputCase.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/dto/InputCase.scala @@ -4,12 +4,7 @@ import org.thp.thehive.models.Case case class InputCase( `case`: Case, - user: Option[String], organisations: Map[String, String], - tags: Set[String], customFields: Map[String, Option[Any]], - caseTemplate: Option[String], - resolutionStatus: Option[String], - impactStatus: Option[String], metaData: MetaData ) diff --git a/migration/src/main/scala/org/thp/thehive/migration/dto/InputObservable.scala b/migration/src/main/scala/org/thp/thehive/migration/dto/InputObservable.scala index 55975ec460..d3f2e4460f 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/dto/InputObservable.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/dto/InputObservable.scala @@ -6,7 +6,5 @@ case class InputObservable( metaData: MetaData, observable: Observable, organisations: Seq[String], - `type`: String, - tags: Set[String], dataOrAttachment: Either[String, InputAttachment] ) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala index ee2d66e808..a41d68475a 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala @@ -1,11 +1,8 @@ package org.thp.thehive.migration.th3 -import java.util.{Base64, Date} - import akka.NotUsed import akka.stream.scaladsl.Source import akka.util.ByteString -import org.thp.scalligraph.EntityId import org.thp.scalligraph.utils.Hash import org.thp.thehive.connector.cortex.models.{Action, Job, JobStatus} import org.thp.thehive.controllers.v0 @@ -15,6 +12,8 @@ import org.thp.thehive.models._ import play.api.libs.functional.syntax._ import play.api.libs.json._ +import java.util.{Base64, Date} + case class Attachment(name: String, hashes: Seq[Hash], size: Long, contentType: String, id: String) trait Conversion { @@ -64,7 +63,7 @@ trait Conversion { status <- (json \ "status").validate[CaseStatus.Value] summary <- (json \ "summary").validateOpt[String] user <- (json \ "owner").validateOpt[String] - tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty) + tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty).filterNot(_.isEmpty) metrics = (json \ "metrics").asOpt[JsObject].getOrElse(JsObject.empty) resolutionStatus = (json \ "resolutionStatus").asOpt[String] impactStatus = (json \ "impactStatus").asOpt[String] @@ -91,18 +90,13 @@ trait Conversion { tags = tags.toSeq, number = number, organisationIds = Nil, - assignee = None, + assignee = user.map(normaliseLogin), impactStatus = impactStatus, resolutionStatus = resolutionStatus, caseTemplate = None ), // organisation Ids are filled by output - user.map(normaliseLogin), Map(mainOrganisation -> Profile.orgAdmin.name), - tags, (metricsValue ++ customFieldsValue).toMap, - None, - resolutionStatus, - impactStatus, metaData ) } @@ -133,13 +127,10 @@ trait Conversion { ioc = ioc, sighted = sighted, ignoreSimilarity = None, - data = dataOrAttachment.swap.toOption, dataType = dataType, tags = tags.toSeq ), Seq(mainOrganisation), - dataType, - tags, dataOrAttachment ) } @@ -207,7 +198,7 @@ trait Conversion { read = status == "Ignored" || status == "Imported" follow <- (json \ "follow").validate[Boolean] caseId <- (json \ "case").validateOpt[String] - tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty) + tags = (json \ "tags").asOpt[Set[String]].getOrElse(Set.empty).filterNot(_.isEmpty) customFields = (json \ "metrics").asOpt[JsObject].getOrElse(JsObject.empty) customFieldsValue = customFields.value.map { case (name, value) => @@ -234,7 +225,6 @@ trait Conversion { ), caseId, mainOrganisation, - tags, customFieldsValue.toMap, caseTemplate: Option[String] ) @@ -265,15 +255,10 @@ trait Conversion { ioc = ioc.getOrElse(false), sighted = false, ignoreSimilarity = None, - data = dataOrAttachment.swap.toOption, dataType = dataType, - tags = tags.toSeq, - organisationIds = Nil, - relatedId = EntityId("") + tags = tags.toSeq ), - Nil, - dataType, - tags, + Seq(mainOrganisation), dataOrAttachment ) @@ -497,13 +482,10 @@ trait Conversion { ioc = ioc, sighted = sighted, ignoreSimilarity = None, - data = dataOrAttachment.swap.toOption, dataType = dataType, tags = tags.toSeq ), Seq(mainOrganisation), - dataType, - tags, dataOrAttachment ) } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/DBConfiguration.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/DBConfiguration.scala index 315df391b3..ebb6d9f032 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/DBConfiguration.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/DBConfiguration.scala @@ -2,16 +2,16 @@ package org.thp.thehive.migration.th3 import java.nio.file.{Files, Paths} import java.security.KeyStore - import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.{Sink, Source} -import com.sksamuel.elastic4s.http._ -import com.sksamuel.elastic4s.http.bulk.BulkResponseItem -import com.sksamuel.elastic4s.http.search.SearchHit -import com.sksamuel.elastic4s.searches._ +import com.sksamuel.elastic4s._ +import com.sksamuel.elastic4s.http.JavaClient +import com.sksamuel.elastic4s.requests.bulk.BulkResponseItem +import com.sksamuel.elastic4s.requests.searches.{SearchHit, SearchRequest} import com.sksamuel.elastic4s.streams.ReactiveElastic.ReactiveElastic import com.sksamuel.elastic4s.streams.{RequestBuilder, ResponseListener} + import javax.inject.{Inject, Named, Singleton} import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} @@ -39,29 +39,29 @@ class DBConfiguration @Inject() ( config: Configuration, lifecycle: ApplicationLifecycle, @Named("databaseVersion") val version: Int, - implicit val ec: ExecutionContext, implicit val actorSystem: ActorSystem ) { private[DBConfiguration] lazy val logger = Logger(getClass) - def requestConfigCallback: RequestConfigCallback = (requestConfigBuilder: RequestConfig.Builder) => { - requestConfigBuilder.setAuthenticationEnabled(credentialsProviderMaybe.isDefined) - config.getOptional[Boolean]("search.circularRedirectsAllowed").foreach(requestConfigBuilder.setCircularRedirectsAllowed) - config.getOptional[Int]("search.connectionRequestTimeout").foreach(requestConfigBuilder.setConnectionRequestTimeout) - config.getOptional[Int]("search.connectTimeout").foreach(requestConfigBuilder.setConnectTimeout) - config.getOptional[Boolean]("search.contentCompressionEnabled").foreach(requestConfigBuilder.setContentCompressionEnabled) - config.getOptional[String]("search.cookieSpec").foreach(requestConfigBuilder.setCookieSpec) - config.getOptional[Boolean]("search.expectContinueEnabled").foreach(requestConfigBuilder.setExpectContinueEnabled) - // config.getOptional[InetAddress]("search.localAddress").foreach(requestConfigBuilder.setLocalAddress) - config.getOptional[Int]("search.maxRedirects").foreach(requestConfigBuilder.setMaxRedirects) - // config.getOptional[Boolean]("search.proxy").foreach(requestConfigBuilder.setProxy) - config.getOptional[Seq[String]]("search.proxyPreferredAuthSchemes").foreach(v => requestConfigBuilder.setProxyPreferredAuthSchemes(v.asJava)) - config.getOptional[Boolean]("search.redirectsEnabled").foreach(requestConfigBuilder.setRedirectsEnabled) - config.getOptional[Boolean]("search.relativeRedirectsAllowed").foreach(requestConfigBuilder.setRelativeRedirectsAllowed) - config.getOptional[Int]("search.socketTimeout").foreach(requestConfigBuilder.setSocketTimeout) - config.getOptional[Seq[String]]("search.targetPreferredAuthSchemes").foreach(v => requestConfigBuilder.setTargetPreferredAuthSchemes(v.asJava)) - requestConfigBuilder - } + def requestConfigCallback: RequestConfigCallback = + (requestConfigBuilder: RequestConfig.Builder) => { + requestConfigBuilder.setAuthenticationEnabled(credentialsProviderMaybe.isDefined) + config.getOptional[Boolean]("search.circularRedirectsAllowed").foreach(requestConfigBuilder.setCircularRedirectsAllowed) + config.getOptional[Int]("search.connectionRequestTimeout").foreach(requestConfigBuilder.setConnectionRequestTimeout) + config.getOptional[Int]("search.connectTimeout").foreach(requestConfigBuilder.setConnectTimeout) + config.getOptional[Boolean]("search.contentCompressionEnabled").foreach(requestConfigBuilder.setContentCompressionEnabled) + config.getOptional[String]("search.cookieSpec").foreach(requestConfigBuilder.setCookieSpec) + config.getOptional[Boolean]("search.expectContinueEnabled").foreach(requestConfigBuilder.setExpectContinueEnabled) + // config.getOptional[InetAddress]("search.localAddress").foreach(requestConfigBuilder.setLocalAddress) + config.getOptional[Int]("search.maxRedirects").foreach(requestConfigBuilder.setMaxRedirects) + // config.getOptional[Boolean]("search.proxy").foreach(requestConfigBuilder.setProxy) + config.getOptional[Seq[String]]("search.proxyPreferredAuthSchemes").foreach(v => requestConfigBuilder.setProxyPreferredAuthSchemes(v.asJava)) + config.getOptional[Boolean]("search.redirectsEnabled").foreach(requestConfigBuilder.setRedirectsEnabled) + config.getOptional[Boolean]("search.relativeRedirectsAllowed").foreach(requestConfigBuilder.setRelativeRedirectsAllowed) + config.getOptional[Int]("search.socketTimeout").foreach(requestConfigBuilder.setSocketTimeout) + config.getOptional[Seq[String]]("search.targetPreferredAuthSchemes").foreach(v => requestConfigBuilder.setTargetPreferredAuthSchemes(v.asJava)) + requestConfigBuilder + } lazy val credentialsProviderMaybe: Option[CredentialsProvider] = for { @@ -86,9 +86,7 @@ class DBConfiguration @Inject() ( val kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) kmf.init(keyStore, keyStorePassword) kmf.getKeyManagers - } finally { - keyInputStream.close() - } + } finally keyInputStream.close() val trustManagers = config .getOptional[String]("search.trustStore.path") @@ -102,9 +100,7 @@ class DBConfiguration @Inject() ( val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) tmf.init(keyStore) tmf.getTrustManagers - } finally { - trustInputStream.close() - } + } finally trustInputStream.close() } .getOrElse(Array.empty) @@ -114,27 +110,29 @@ class DBConfiguration @Inject() ( sslContext } - def httpClientConfig: HttpClientConfigCallback = (httpClientBuilder: HttpAsyncClientBuilder) => { - sslContextMaybe.foreach(httpClientBuilder.setSSLContext) - credentialsProviderMaybe.foreach(httpClientBuilder.setDefaultCredentialsProvider) - httpClientBuilder - } + def httpClientConfig: HttpClientConfigCallback = + (httpClientBuilder: HttpAsyncClientBuilder) => { + sslContextMaybe.foreach(httpClientBuilder.setSSLContext) + credentialsProviderMaybe.foreach(httpClientBuilder.setDefaultCredentialsProvider) + httpClientBuilder + } /** * Underlying ElasticSearch client */ - val client: ElasticClient = ElasticClient(ElasticProperties(config.get[String]("search.uri")), requestConfigCallback, httpClientConfig) + private val props = ElasticProperties(config.get[String]("search.uri")) + private val client = ElasticClient(JavaClient(props, requestConfigCallback, httpClientConfig)) + // when application close, close also ElasticSearch connection lifecycle.addStopHook { () => - Future { - client.close() - } + client.close() + Future.successful(()) } - def execute[T, U](t: T)( - implicit + def execute[T, U](t: T)(implicit handler: Handler[T, U], - manifest: Manifest[U] + manifest: Manifest[U], + ec: ExecutionContext ): Future[U] = { logger.debug(s"Elasticsearch request: ${client.show(t)}") client.execute(t).flatMap { @@ -157,12 +155,13 @@ class DBConfiguration @Inject() ( /** * Creates a Source (akka stream) from the result of the search */ - def source(searchRequest: SearchRequest): Source[SearchHit, NotUsed] = Source.fromPublisher(client.publisher(searchRequest)) + def source(searchRequest: SearchRequest)(implicit ec: ExecutionContext): Source[SearchHit, NotUsed] = + Source.fromPublisher(client.publisher(searchRequest)) /** * Create a Sink (akka stream) that create entity in ElasticSearch */ - def sink[T](implicit builder: RequestBuilder[T]): Sink[T, Future[Unit]] = { + def sink[T](implicit builder: RequestBuilder[T], ec: ExecutionContext): Sink[T, Future[Unit]] = { val sinkListener = new ResponseListener[T] { override def onAck(resp: BulkResponseItem, original: T): Unit = () @@ -209,5 +208,5 @@ class DBConfiguration @Inject() ( * return a new instance of DBConfiguration that points to the previous version of the index schema */ def previousVersion: DBConfiguration = - new DBConfiguration(config, lifecycle, version - 1, ec, actorSystem) + new DBConfiguration(config, lifecycle, version - 1, actorSystem) } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala index d3de9d4ee2..1288564797 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/DBFind.scala @@ -4,9 +4,9 @@ import akka.NotUsed import akka.stream.scaladsl.Source import akka.stream.stage.{AsyncCallback, GraphStage, GraphStageLogic, OutHandler} import akka.stream.{Attributes, Materializer, Outlet, SourceShape} -import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.http.search.{SearchHit, SearchResponse} -import com.sksamuel.elastic4s.searches.SearchRequest +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.{ElasticRequest, Show} +import com.sksamuel.elastic4s.requests.searches.{SearchHit, SearchRequest, SearchResponse} import javax.inject.{Inject, Singleton} import org.thp.scalligraph.{InternalError, SearchError} import play.api.libs.json._ @@ -75,6 +75,9 @@ class DBFind(pageSize: Int, keepAlive: FiniteDuration, db: DBConfiguration, impl (src, total) } + def showQuery(request: SearchRequest): String = + Show[ElasticRequest].show(SearchHandler.build(request)) + /** * Search entities in ElasticSearch * @@ -87,10 +90,10 @@ class DBFind(pageSize: Int, keepAlive: FiniteDuration, db: DBConfiguration, impl def apply(range: Option[String], sortBy: Seq[String])(query: String => SearchRequest): (Source[JsObject, NotUsed], Future[Long]) = { val (offset, limit) = getOffsetAndLimitFromRange(range) val sortDef = DBUtils.sortDefinition(sortBy) - val searchRequest = query(db.indexName).start(offset).sortBy(sortDef).version(true) + val searchRequest = query(db.indexName).start(offset).sortBy(sortDef).seqNoPrimaryTerm(true) logger.debug( - s"search in ${searchRequest.indexesTypes.indexes.mkString(",")} / ${searchRequest.indexesTypes.types.mkString(",")} ${db.client.show(searchRequest)}" + s"search in ${searchRequest.indexes.values.mkString(",")} ${showQuery(searchRequest)}" ) val (src, total) = if (limit > 2 * pageSize) @@ -108,7 +111,7 @@ class DBFind(pageSize: Int, keepAlive: FiniteDuration, db: DBConfiguration, impl def apply(query: String => SearchRequest): Future[SearchResponse] = { val searchRequest = query(db.indexName) logger.debug( - s"search in ${searchRequest.indexesTypes.indexes.mkString(",")} / ${searchRequest.indexesTypes.types.mkString(",")} ${db.client.show(searchRequest)}" + s"search in ${searchRequest.indexes.values.mkString(",")} ${showQuery(searchRequest)}" ) db.execute(searchRequest) diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala index 7994058184..bd84c0ab9e 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/DBGet.scala @@ -1,6 +1,6 @@ package org.thp.thehive.migration.th3 -import com.sksamuel.elastic4s.http.ElasticDsl._ +import com.sksamuel.elastic4s.ElasticDsl._ import javax.inject.{Inject, Singleton} import org.thp.scalligraph.NotFoundError import play.api.libs.json.JsObject @@ -23,7 +23,7 @@ class DBGet @Inject() (db: DBConfiguration, implicit val ec: ExecutionContext) { search(db.indexName) .query(idsQuery(id) /*.types(modelName)*/ ) .size(1) - .version(true) + .seqNoPrimaryTerm(true) }.map { searchResponse => searchResponse .hits diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/DBUtils.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/DBUtils.scala index 8e9a241888..b3ea19efc7 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/DBUtils.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/DBUtils.scala @@ -1,9 +1,8 @@ package org.thp.thehive.migration.th3 -import com.sksamuel.elastic4s.http.ElasticDsl.fieldSort -import com.sksamuel.elastic4s.http.search.SearchHit -import com.sksamuel.elastic4s.searches.sort.Sort -import com.sksamuel.elastic4s.searches.sort.SortOrder.{ASC, DESC} +import com.sksamuel.elastic4s.ElasticDsl.fieldSort +import com.sksamuel.elastic4s.requests.searches.SearchHit +import com.sksamuel.elastic4s.requests.searches.sort.{Sort, SortOrder} import play.api.libs.json._ import scala.collection.IterableLike @@ -29,8 +28,8 @@ object DBUtils { def sortDefinition(sortBy: Seq[String]): Seq[Sort] = { val byFieldList: Seq[(String, Sort)] = sortBy .map { - case f if f.startsWith("+") => f.drop(1) -> fieldSort(f.drop(1)).order(ASC) - case f if f.startsWith("-") => f.drop(1) -> fieldSort(f.drop(1)).order(DESC) + case f if f.startsWith("+") => f.drop(1) -> fieldSort(f.drop(1)).order(SortOrder.ASC) + case f if f.startsWith("-") => f.drop(1) -> fieldSort(f.drop(1)).order(SortOrder.DESC) case f if f.nonEmpty => f -> fieldSort(f) } // then remove duplicates @@ -50,10 +49,10 @@ object DBUtils { case None => JsNull -> (body \ "relations").as[JsString] } body - "relations" + - ("_type" -> model) + - ("_routing" -> hit.routing.fold(id)(JsString.apply)) + - ("_parent" -> parent) + - ("_id" -> id) + - ("_version" -> JsNumber(hit.version)) + ("_type" -> model) + + ("_routing" -> hit.routing.fold(id)(JsString.apply)) + + ("_parent" -> parent) + + ("_id" -> id) + + ("_primaryTerm" -> JsNumber(hit.primaryTerm)) } } diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala index c6ce3faca7..b2fe04f1be 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala @@ -1,17 +1,14 @@ package org.thp.thehive.migration.th3 -import java.util.{Base64, Date} - import akka.NotUsed import akka.actor.ActorSystem import akka.stream.Materializer import akka.stream.scaladsl.Source import akka.util.ByteString import com.google.inject.Guice -import com.sksamuel.elastic4s.http.ElasticDsl._ -import com.sksamuel.elastic4s.searches.queries.RangeQuery -import com.sksamuel.elastic4s.searches.queries.term.TermsQuery -import javax.inject.{Inject, Singleton} +import com.sksamuel.elastic4s.ElasticDsl._ +import com.sksamuel.elastic4s.requests.searches.queries.RangeQuery +import com.sksamuel.elastic4s.requests.searches.queries.term.TermsQuery import net.codingwell.scalaguice.ScalaModule import org.thp.thehive.migration import org.thp.thehive.migration.Filter @@ -21,6 +18,8 @@ import play.api.inject.{ApplicationLifecycle, DefaultApplicationLifecycle} import play.api.libs.json._ import play.api.{Configuration, Logger} +import java.util.{Base64, Date} +import javax.inject.{Inject, Singleton} import scala.collection.immutable import scala.concurrent.{ExecutionContext, Future} import scala.reflect.{classTag, ClassTag} diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/JanusDatabaseProvider.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/JanusDatabaseProvider.scala new file mode 100644 index 0000000000..1927e0cc1c --- /dev/null +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/JanusDatabaseProvider.scala @@ -0,0 +1,25 @@ +package org.thp.thehive.migration.th4 + +import akka.actor.ActorSystem +import org.thp.scalligraph.SingleInstance +import org.thp.scalligraph.janus.JanusDatabase +import org.thp.scalligraph.models.{Database, UpdatableSchema} +import play.api.Configuration + +import javax.inject.{Inject, Provider, Singleton} +import scala.collection.immutable + +@Singleton +class JanusDatabaseProvider @Inject() (configuration: Configuration, system: ActorSystem, schemas: immutable.Set[UpdatableSchema]) + extends Provider[Database] { + override lazy val get: Database = { + val db = new JanusDatabase( + JanusDatabase.openDatabase(configuration, system), + configuration, + system, + new SingleInstance(true) + ) + schemas.foreach(schema => db.createSchemaFrom(schema)(schema.authContext)) + db + } +} diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 0a7b54d102..7706330f2d 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -1,22 +1,22 @@ package org.thp.thehive.migration.th4 import akka.actor.ActorSystem +import akka.actor.typed.Scheduler import akka.stream.Materializer import com.google.inject.Guice -import net.codingwell.scalaguice.ScalaModule +import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder} import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.scalligraph._ import org.thp.scalligraph.auth.{AuthContext, AuthContextImpl, UserSrv => UserDB} import org.thp.scalligraph.janus.JanusDatabase -import org.thp.scalligraph.models.{Database, Entity, Schema, UMapping} +import org.thp.scalligraph.models._ import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Graph, Traversal} import org.thp.thehive.connector.cortex.models.{CortexSchemaDefinition, TheHiveCortexSchemaProvider} import org.thp.thehive.connector.cortex.services.{ActionSrv, JobSrv} -import org.thp.thehive.controllers.v1.Conversion._ import org.thp.thehive.dto.v1.InputCustomFieldValue -import org.thp.thehive.migration +import org.thp.thehive.{migration, ClusterSetup} import org.thp.thehive.migration.IdMapping import org.thp.thehive.migration.dto._ import org.thp.thehive.models._ @@ -25,7 +25,7 @@ import play.api.cache.SyncCacheApi import play.api.cache.ehcache.EhCacheModule import play.api.inject.guice.GuiceInjector import play.api.inject.{ApplicationLifecycle, DefaultApplicationLifecycle, Injector} -import play.api.libs.concurrent.AkkaGuiceSupport +import play.api.libs.concurrent.{AkkaGuiceSupport, AkkaSchedulerProvider} import play.api.{Configuration, Environment, Logger} import javax.inject.{Inject, Provider, Singleton} @@ -43,6 +43,7 @@ object Output { override def configure(): Unit = { bind[Configuration].toInstance(configuration) bind[ActorSystem].toInstance(actorSystem) + bind[Scheduler].toProvider[AkkaSchedulerProvider] bind[Materializer].toInstance(Materializer(actorSystem)) bind[ExecutionContext].toInstance(actorSystem.dispatcher) bind[Injector].to[GuiceInjector] @@ -52,9 +53,13 @@ object Output { bindActor[DummyActor]("cortex-actor") bindActor[DummyActor]("integrity-check-actor") + val schemaBindings = ScalaMultibinder.newSetBinder[UpdatableSchema](binder) + schemaBindings.addBinding.to[TheHiveSchemaDefinition] + schemaBindings.addBinding.to[CortexSchemaDefinition] + bind[SingleInstance].toInstance(new SingleInstance(true)) + bind[AuditSrv].to[NoAuditSrv] - bind[Database].to[JanusDatabase] - bind[Database].toProvider[BasicDatabaseProvider] + bind[Database].toProvider[JanusDatabaseProvider] bind[Configuration].toInstance(configuration) bind[Environment].toInstance(Environment.simple()) bind[ApplicationLifecycle].to[DefaultApplicationLifecycle] @@ -65,6 +70,7 @@ object Output { case "hdfs" => bind(classOf[StorageSrv]).to(classOf[HadoopStorageSrv]) case "s3" => bind(classOf[StorageSrv]).to(classOf[S3StorageSrv]) } + bind[ClusterSetup].asEagerSingleton() () } }).asJava @@ -79,11 +85,6 @@ object Output { } } -@Singleton -class BasicDatabaseProvider @Inject() (database: Database) extends Provider[Database] { - override def get(): Database = database -} - @Singleton class Output @Inject() ( theHiveSchema: TheHiveSchemaDefinition, @@ -455,36 +456,34 @@ class Output @Inject() ( private def getCaseTemplate(name: String): Option[CaseTemplate with Entity] = caseTemplates.get(name) - override def createCaseTemplate(inputCaseTemplate: InputCaseTemplate): Try[IdMapping] = ??? -// authTransaction(inputCaseTemplate.metaData.createdBy) { implicit graph => implicit authContext => -// logger.debug(s"Create case template ${inputCaseTemplate.caseTemplate.name}") -// for { -// organisation <- getOrganisation(inputCaseTemplate.organisation) -// tags <- inputCaseTemplate.tags.toTry(getTag) -// richCaseTemplate <- caseTemplateSrv.create(inputCaseTemplate.caseTemplate, organisation, tags, Nil, Nil) -// _ = updateMetaData(richCaseTemplate.caseTemplate, inputCaseTemplate.metaData) -// _ = inputCaseTemplate.customFields.foreach { -// case InputCustomFieldValue(name, value, order) => -// (for { -// cf <- getCustomField(name) -// ccf <- CustomFieldType.map(cf.`type`).setValue(CaseTemplateCustomField(order = order), value) -// _ <- caseTemplateSrv.caseTemplateCustomFieldSrv.create(ccf, richCaseTemplate.caseTemplate, cf) -// } yield ()).logFailure(s"Unable to set custom field $name=${value.getOrElse("")}") -// } -// _ = caseTemplates += (inputCaseTemplate.caseTemplate.name -> richCaseTemplate.caseTemplate) -// } yield IdMapping(inputCaseTemplate.metaData.id, richCaseTemplate._id) -// } + override def createCaseTemplate(inputCaseTemplate: InputCaseTemplate): Try[IdMapping] = + authTransaction(inputCaseTemplate.metaData.createdBy) { implicit graph => implicit authContext => + logger.debug(s"Create case template ${inputCaseTemplate.caseTemplate.name}") + for { + organisation <- getOrganisation(inputCaseTemplate.organisation) + richCaseTemplate <- caseTemplateSrv.create(inputCaseTemplate.caseTemplate, organisation, Nil, Nil) + _ = updateMetaData(richCaseTemplate.caseTemplate, inputCaseTemplate.metaData) + _ = inputCaseTemplate.customFields.foreach { + case InputCustomFieldValue(name, value, order) => + (for { + cf <- getCustomField(name) + ccf <- CustomFieldType.map(cf.`type`).setValue(CaseTemplateCustomField(order = order), value) + _ <- caseTemplateSrv.caseTemplateCustomFieldSrv.create(ccf, richCaseTemplate.caseTemplate, cf) + } yield ()).logFailure(s"Unable to set custom field $name=${value.getOrElse("")}") + } + _ = caseTemplates += (inputCaseTemplate.caseTemplate.name -> richCaseTemplate.caseTemplate) + } yield IdMapping(inputCaseTemplate.metaData.id, richCaseTemplate._id) + } - override def createCaseTemplateTask(caseTemplateId: EntityId, inputTask: InputTask): Try[IdMapping] = ??? -// authTransaction(inputTask.metaData.createdBy) { implicit graph => implicit authContext => -// logger.debug(s"Create task ${inputTask.task.title} in case template $caseTemplateId") -// for { -// caseTemplate <- caseTemplateSrv.getOrFail(caseTemplateId) -// richTask <- taskSrv.create(inputTask.task) -// _ = updateMetaData(richTask.task, inputTask.metaData) -// _ <- caseTemplateSrv.addTask(caseTemplate, richTask.task) -// } yield IdMapping(inputTask.metaData.id, richTask._id) -// } + override def createCaseTemplateTask(caseTemplateId: EntityId, inputTask: InputTask): Try[IdMapping] = + authTransaction(inputTask.metaData.createdBy) { implicit graph => implicit authContext => + logger.debug(s"Create task ${inputTask.task.title} in case template $caseTemplateId") + for { + caseTemplate <- caseTemplateSrv.getOrFail(caseTemplateId) + richTask <- caseTemplateSrv.createTask(caseTemplate, inputTask.task) + _ = updateMetaData(richTask.task, inputTask.metaData) + } yield IdMapping(inputTask.metaData.id, richTask._id) + } override def caseExists(inputCase: InputCase): Boolean = caseNumbers.contains(inputCase.`case`.number) @@ -493,28 +492,58 @@ class Output @Inject() ( override def createCase(inputCase: InputCase): Try[IdMapping] = authTransaction(inputCase.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create case #${inputCase.`case`.number}") - val organisationIds = inputCase.organisations.flatMap { - case (orgName, _) => getOrganisation(orgName).map(_._id).toOption - } - val `case` = inputCase.`case`.copy(organisationIds = organisationIds.toSeq) + val organisationIds = inputCase + .organisations + .flatMap { + case (orgName, _) => getOrganisation(orgName).map(_._id).toOption + } + .toSeq + val assignee = inputCase + .`case` + .assignee + .flatMap(getUser(_).toOption) + val caseTemplate = inputCase + .`case` + .caseTemplate + .flatMap(getCaseTemplate) + val resolutionStatus = inputCase + .`case` + .resolutionStatus + .flatMap(getResolutionStatus(_).toOption) + val impactStatus = inputCase + .`case` + .impactStatus + .flatMap(getImpactStatus(_).toOption) + val `case` = inputCase + .`case` + .copy( + assignee = assignee.map(_.login), + organisationIds = organisationIds, + caseTemplate = caseTemplate.map(_.name), + impactStatus = impactStatus.map(_.value), + resolutionStatus = resolutionStatus.map(_.value) + ) caseSrv.createEntity(`case`).map { createdCase => updateMetaData(createdCase, inputCase.metaData) - inputCase - .user - .foreach { userLogin => - getUser(userLogin) - .flatMap(user => caseSrv.caseUserSrv.create(CaseUser(), createdCase, user)) - .logFailure(s"Unable to assign case #${createdCase.number} to $userLogin") + assignee + .foreach { user => + caseSrv + .caseUserSrv + .create(CaseUser(), createdCase, user) + .logFailure(s"Unable to assign case #${createdCase.number} to ${user.login}") } - inputCase - .caseTemplate - .flatMap(getCaseTemplate) + caseTemplate .foreach { ct => caseSrv .caseCaseTemplateSrv .create(CaseCaseTemplate(), createdCase, ct) .logFailure(s"Unable to set case template ${ct.name} to case #${createdCase.number}") } + inputCase.`case`.tags.foreach { tagName => + getTag(tagName) + .flatMap(tag => caseSrv.caseTagSrv.create(CaseTag(), createdCase, tag)) + .logFailure(s"Unable to add tag $tagName to case #${createdCase.number}") + } inputCase.customFields.foreach { case (name, value) => // TODO Add order getCustomField(name) @@ -537,23 +566,18 @@ class Output @Inject() ( shared.logFailure(s"Unable to share case #${createdCase.number} with organisation $organisationName, profile $profileName") ownerSet || owner } - inputCase.tags.filterNot(_.isEmpty).foreach { tagName => - getTag(tagName) - .flatMap(tag => caseSrv.caseTagSrv.create(CaseTag(), createdCase, tag)) - .logFailure(s"Unable to add tag $tagName to case #${createdCase.number}") - } - inputCase - .resolutionStatus + resolutionStatus .foreach { resolutionStatus => - getResolutionStatus(resolutionStatus) - .flatMap(caseSrv.caseResolutionStatusSrv.create(CaseResolutionStatus(), createdCase, _)) + caseSrv + .caseResolutionStatusSrv + .create(CaseResolutionStatus(), createdCase, resolutionStatus) .logFailure(s"Unable to set resolution status $resolutionStatus to case #${createdCase.number}") } - inputCase - .impactStatus + impactStatus .foreach { impactStatus => - getImpactStatus(impactStatus) - .flatMap(caseSrv.caseImpactStatusSrv.create(CaseImpactStatus(), createdCase, _)) + caseSrv + .caseImpactStatusSrv + .create(CaseImpactStatus(), createdCase, impactStatus) .logFailure(s"Unable to set impact status $impactStatus to case #${createdCase.number}") } @@ -561,28 +585,27 @@ class Output @Inject() ( } } - override def createCaseTask(caseId: EntityId, inputTask: InputTask): Try[IdMapping] = ??? -// authTransaction(inputTask.metaData.createdBy) { implicit graph => implicit authContext => -// logger.debug(s"Create task ${inputTask.task.title} in case $caseId") -// for { -// richTask <- taskSrv.create(inputTask.task) -// _ = updateMetaData(richTask.task, inputTask.metaData) -// case0 <- getCase(caseId) -// _ <- inputTask.organisations.toTry { organisation => -// getOrganisation(organisation).flatMap(shareSrv.shareTask(richTask, case0, _)) -// } -// } yield IdMapping(inputTask.metaData.id, richTask._id) -// } + override def createCaseTask(caseId: EntityId, inputTask: InputTask): Try[IdMapping] = + authTransaction(inputTask.metaData.createdBy) { implicit graph => implicit authContext => + logger.debug(s"Create task ${inputTask.task.title} in case $caseId") + val assignee = inputTask.owner.flatMap(getUser(_).toOption) + val organisations = inputTask.organisations.flatMap(getOrganisation(_).toOption) + for { + richTask <- taskSrv.create(inputTask.task.copy(relatedId = caseId, organisationIds = organisations.map(_._id)), assignee) + _ = updateMetaData(richTask.task, inputTask.metaData) + case0 <- getCase(caseId) + _ <- organisations.toTry(o => shareSrv.shareTask(richTask, case0, o._id)) + } yield IdMapping(inputTask.metaData.id, richTask._id) + } def createCaseTaskLog(taskId: EntityId, inputLog: InputLog): Try[IdMapping] = authTransaction(inputLog.metaData.createdBy) { implicit graph => implicit authContext => for { task <- taskSrv.getOrFail(taskId) _ = logger.debug(s"Create log in task ${task.title}") - log <- logSrv.createEntity(inputLog.log) - _ <- logSrv.taskLogSrv.create(TaskLog(), task, log) - _ <- auditSrv.log.create(log, task, RichLog(log, Nil).toJson) + log <- logSrv.createEntity(inputLog.log.copy(taskId = task._id, organisationIds = task.organisationIds)) _ = updateMetaData(log, inputLog.metaData) + _ <- logSrv.taskLogSrv.create(TaskLog(), task, log) _ <- inputLog.attachments.toTry { inputAttachment => attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { attachment => logSrv.logAttachmentSrv.create(LogAttachment(), log, attachment) @@ -591,39 +614,74 @@ class Output @Inject() ( } yield IdMapping(inputLog.metaData.id, log._id) } - override def createCaseObservable(caseId: EntityId, inputObservable: InputObservable): Try[IdMapping] = ??? -// authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => -// logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in case $caseId") -// for { -// observableType <- getObservableType(inputObservable.`type`) -// tags <- inputObservable.tags.filterNot(_.isEmpty).toTry(getTag) -// orgs <- inputObservable.organisations.toTry(getOrganisation) -// richObservable <- -// inputObservable -// .dataOrAttachment -// .fold( -// dataValue => -// dataSrv.createEntity(Data(dataValue)).flatMap { data => -// observableSrv -// .create( -// inputObservable.observable.copy(organisationIds = orgs.map(_._id), relatedId = caseId), -// data.data -// ) // FIXME don't check duplicates -// }, -// inputAttachment => -// attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { -// attachment => -// observableSrv.create( -// inputObservable.observable.copy(organisationIds = orgs.map(_._id), relatedId = caseId), -// attachment -// ) -// } -// ) -// _ = updateMetaData(richObservable.observable, inputObservable.metaData) -// case0 <- getCase(caseId) -// _ <- orgs.toTry(o => shareSrv.shareObservable(richObservable, case0, o)) -// } yield IdMapping(inputObservable.metaData.id, richObservable._id) -// } + private def createSimpleObservable(observable: Observable, observableType: ObservableType with Entity, dataValue: String)(implicit + graph: Graph, + authContext: AuthContext + ): Try[Observable with Entity] = + for { + data <- dataSrv.createEntity(Data(dataValue)) + _ <- + if (observableType.isAttachment) Failure(BadRequestError("A attachment observable doesn't accept string value")) + else Success(()) + createdObservable <- observableSrv.createEntity(observable.copy(data = Some(dataValue))) + _ <- observableSrv.observableDataSrv.create(ObservableData(), createdObservable, data) + } yield createdObservable + + private def createAttachmentObservable( + observable: Observable, + observableType: ObservableType with Entity, + inputAttachment: InputAttachment + )(implicit graph: Graph, authContext: AuthContext): Try[Observable with Entity] = + for { + attachment <- attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data) + _ <- + if (!observableType.isAttachment) Failure(BadRequestError("A text observable doesn't accept attachment")) + else Success(()) + createdObservable <- observableSrv.createEntity(observable.copy(data = None)) + _ <- observableSrv.observableAttachmentSrv.create(ObservableAttachment(), createdObservable, attachment) + } yield createdObservable + + private def createObservable(relatedId: EntityId, inputObservable: InputObservable, organisationIds: Seq[EntityId])(implicit + graph: Graph, + authContext: AuthContext + ) = + for { + observableType <- getObservableType(inputObservable.observable.dataType) + observable <- + inputObservable + .dataOrAttachment + .fold( + data => + createSimpleObservable( + inputObservable.observable.copy(organisationIds = organisationIds, relatedId = relatedId), + observableType, + data + ), + attachment => + createAttachmentObservable( + inputObservable.observable.copy(organisationIds = organisationIds, relatedId = relatedId), + observableType, + attachment + ) + ) + _ = updateMetaData(observable, inputObservable.metaData) + _ <- observableSrv.observableObservableType.create(ObservableObservableType(), observable, observableType) + _ = inputObservable.observable.tags.foreach { tagName => + getTag(tagName) + .foreach(tag => observableSrv.observableTagSrv.create(ObservableTag(), observable, tag)) + } + } yield observable + + override def createCaseObservable(caseId: EntityId, inputObservable: InputObservable): Try[IdMapping] = + authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => + logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in case $caseId") + for { + organisations <- inputObservable.organisations.toTry(getOrganisation) + richObservable <- createObservable(caseId, inputObservable, organisations.map(_._id)) + case0 <- getCase(caseId) + _ <- organisations.toTry(o => shareSrv.shareObservable(RichObservable(richObservable, None, None, Nil), case0, o._id)) + } yield IdMapping(inputObservable.metaData.id, richObservable._id) + } override def createJob(observableId: EntityId, inputJob: InputJob): Try[IdMapping] = authTransaction(inputJob.metaData.createdBy) { implicit graph => implicit authContext => @@ -635,38 +693,16 @@ class Output @Inject() ( } yield IdMapping(inputJob.metaData.id, job._id) } - override def createJobObservable(jobId: EntityId, inputObservable: InputObservable): Try[IdMapping] = ??? -// authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => -// logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in job $jobId") -// for { -// job <- jobSrv.getOrFail(jobId) -// jobObs <- jobSrv.get(job).observable.getOrFail("Observable") -// observableType <- getObservableType(inputObservable.`type`) -// tags = inputObservable.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq -// richObservable <- -// inputObservable -// .dataOrAttachment -// .fold( -// dataValue => -// dataSrv.createEntity(Data(dataValue)).flatMap { data => -// observableSrv.create( -// inputObservable.observable.copy(organisationIds = jobObs.organisationIds, relatedId = jobId) -// ) -// }, -// inputAttachment => -// attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { -// attachment => -// observableSrv -// .create( -// inputObservable.observable.copy(organisationIds = jobObs.organisationIds, relatedId = jobId), -// attachment -// ) -// } -// ) -// _ = updateMetaData(richObservable.observable, inputObservable.metaData) -// _ <- jobSrv.addObservable(job, richObservable.observable) -// } yield IdMapping(inputObservable.metaData.id, richObservable._id) -// } + override def createJobObservable(jobId: EntityId, inputObservable: InputObservable): Try[IdMapping] = + authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => + logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in job $jobId") + for { + organisations <- inputObservable.organisations.toTry(getOrganisation) + observable <- createObservable(jobId, inputObservable, organisations.map(_._id)) + job <- jobSrv.getOrFail(jobId) + _ <- jobSrv.addObservable(job, observable) + } yield IdMapping(inputObservable.metaData.id, observable._id) + } override def alertExists(inputAlert: InputAlert): Boolean = alerts.contains((inputAlert.alert.`type`, inputAlert.alert.source, inputAlert.alert.sourceRef)) @@ -674,61 +710,44 @@ class Output @Inject() ( override def createAlert(inputAlert: InputAlert): Try[IdMapping] = authTransaction(inputAlert.metaData.createdBy) { implicit graph => implicit authContext => logger.debug(s"Create alert ${inputAlert.alert.`type`}:${inputAlert.alert.source}:${inputAlert.alert.sourceRef}") + val `case` = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption) + val tags = inputAlert.alert.tags.flatMap(getTag(_).toOption) for { organisation <- getOrganisation(inputAlert.organisation) - caseTemplate = + createdAlert <- alertSrv.createEntity(inputAlert.alert.copy(organisationId = organisation._id, caseId = `case`.map(_._id))) + _ = updateMetaData(createdAlert, inputAlert.metaData) + _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), createdAlert, organisation) + _ <- inputAlert .caseTemplate - .flatMap(ct => - getCaseTemplate(ct).orElse { - logger.warn( - s"Case template $ct not found (used in alert ${inputAlert.alert.`type`}:${inputAlert.alert.source}:${inputAlert.alert.sourceRef})" - ) - None + .flatMap(getCaseTemplate) + .map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), createdAlert, ct)) + .flip + _ = tags.foreach(t => alertSrv.alertTagSrv.create(AlertTag(), createdAlert, t)) + _ = inputAlert.customFields.foreach { + case (name, value) => // TODO Add order + getCustomField(name) + .flatMap { cf => + CustomFieldType + .map(cf.`type`) + .setValue(AlertCustomField(), value) + .flatMap(acf => alertSrv.alertCustomFieldSrv.create(acf, createdAlert, cf)) } - ) - tags = inputAlert.tags.filterNot(_.isEmpty).flatMap(getTag(_).toOption).toSeq -// alert <- alertSrv.create(inputAlert.alert, organisation, tags, inputAlert.customFields, caseTemplate) // FIXME don't check duplicate - alert <- alertSrv.createEntity(inputAlert.alert.copy(organisationId = organisation._id)) - _ <- alertSrv.alertOrganisationSrv.create(AlertOrganisation(), alert, organisation) - _ <- caseTemplate.map(ct => alertSrv.alertCaseTemplateSrv.create(AlertCaseTemplate(), alert, ct)).flip - _ <- tags.toTry(t => alertSrv.alertTagSrv.create(AlertTag(), alert, t)) - _ <- inputAlert.customFields.toTry { case (name, value) => alertSrv.createCustomField(alert, InputCustomFieldValue(name, value, None)) } - _ = updateMetaData(alert, inputAlert.metaData) - _ = inputAlert.caseId.flatMap(c => getCase(EntityId.read(c)).toOption).foreach(alertSrv.alertCaseSrv.create(AlertCase(), alert, _)) - } yield IdMapping(inputAlert.metaData.id, alert._id) + .logFailure(s"Unable to set custom field $name=${value + .getOrElse("")} to alert ${inputAlert.alert.`type`}:${inputAlert.alert.source}:${inputAlert.alert.sourceRef}") + } + } yield IdMapping(inputAlert.metaData.id, createdAlert._id) } - override def createAlertObservable(alertId: EntityId, inputObservable: InputObservable): Try[IdMapping] = ??? -// authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => -// logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in alert $alertId") -// for { -// alert <- alertSrv.getOrFail(alertId) -// richObservable <- -// inputObservable -// .dataOrAttachment -// .fold( -// dataValue => -// dataSrv.createEntity(Data(dataValue)).flatMap { data => -// observableSrv.create( -// inputObservable.observable.copy(organisationIds = Seq(alert.organisationId), relatedId = alertId) -// ) -// }, -// inputAttachment => -// attachmentSrv.create(inputAttachment.name, inputAttachment.size, inputAttachment.contentType, inputAttachment.data).flatMap { -// attachment => -// observableSrv -// .create( -// inputObservable.observable.copy(organisationIds = Seq(alert.organisationId), relatedId = alertId), -// attachment -// ) -// } -// ) -// _ = updateMetaData(richObservable.observable, inputObservable.metaData) -// -// _ <- alertSrv.alertObservableSrv.create(AlertObservable(), alert, richObservable.observable) -// } yield IdMapping(inputObservable.metaData.id, richObservable._id) -// } + override def createAlertObservable(alertId: EntityId, inputObservable: InputObservable): Try[IdMapping] = + authTransaction(inputObservable.metaData.createdBy) { implicit graph => implicit authContext => + logger.debug(s"Create observable ${inputObservable.dataOrAttachment.fold(identity, _.name)} in alert $alertId") + for { + alert <- alertSrv.getOrFail(alertId) + observable <- createObservable(alert._id, inputObservable, Seq(alert.organisationId)) + _ <- alertSrv.alertObservableSrv.create(AlertObservable(), alert, observable) + } yield IdMapping(inputObservable.metaData.id, observable._id) + } private def getEntity(entityType: String, entityId: EntityId)(implicit graph: Graph): Try[Product with Entity] = entityType match { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6027c26778..3502a03152 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,13 +3,14 @@ import sbt._ object Dependencies { val janusVersion = "0.5.2" val akkaVersion: String = play.core.PlayVersion.akkaVersion - val elastic4sVersion = "6.7.4" + val elastic4sVersion = "7.10.2" lazy val specs = "com.typesafe.play" %% "play-specs2" % play.core.PlayVersion.current lazy val playLogback = "com.typesafe.play" %% "play-logback" % play.core.PlayVersion.current lazy val playGuice = "com.typesafe.play" %% "play-guice" % play.core.PlayVersion.current lazy val playFilters = "com.typesafe.play" %% "filters-helpers" % play.core.PlayVersion.current lazy val playMockws = "de.leanovate.play-mockws" %% "play-mockws" % "2.8.0" + lazy val akkaActor = "com.typesafe.akka" %% "akka-actor" % akkaVersion lazy val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % akkaVersion lazy val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion lazy val akkaClusterTyped = "com.typesafe.akka" %% "akka-cluster-typed" % akkaVersion @@ -42,7 +43,7 @@ object Dependencies { lazy val chimney = "io.scalaland" %% "chimney" % "0.4.0" lazy val elastic4sCore = "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion lazy val elastic4sHttpStreams = "com.sksamuel.elastic4s" %% "elastic4s-http-streams" % elastic4sVersion - lazy val elastic4sHttp = "com.sksamuel.elastic4s" %% "elastic4s-http" % elastic4sVersion + lazy val elastic4sClient = "com.sksamuel.elastic4s" %% "elastic4s-client-esjava" % elastic4sVersion lazy val log4jOverSlf4j = "org.slf4j" % "log4j-over-slf4j" % "1.7.25" lazy val log4jToSlf4j = "org.apache.logging.log4j" % "log4j-to-slf4j" % "2.9.1" lazy val reflections = "org.reflections" % "reflections" % "0.9.12"