From 984349b8aef724dbf9b507bc9f379ed841098742 Mon Sep 17 00:00:00 2001 From: To-om Date: Sun, 6 Feb 2022 17:26:12 +0100 Subject: [PATCH] #2335 Fix initial values creation --- ScalliGraph | 2 +- migration/src/main/resources/reference.conf | 1 + .../org/thp/thehive/migration/Migrate.scala | 3 ++ .../thehive/migration/th4/DummyActor.scala | 15 +++++- .../thp/thehive/migration/th4/Output.scala | 53 +++++++++++++------ .../models/TheHiveSchemaDefinition.scala | 7 --- .../services/IntegrityCheckActor.scala | 13 ++++- .../org/thp/thehive/services/ProfileSrv.scala | 2 +- 8 files changed, 67 insertions(+), 29 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index abfb47e16a..c8a0ca1e40 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit abfb47e16a9c9091cb33fdfaabcae24eb08a7cd5 +Subproject commit c8a0ca1e40672971304e856711e64615761bed81 diff --git a/migration/src/main/resources/reference.conf b/migration/src/main/resources/reference.conf index ced00a1ff7..95836bcadd 100644 --- a/migration/src/main/resources/reference.conf +++ b/migration/src/main/resources/reference.conf @@ -43,6 +43,7 @@ output { caseNumberShift: 0 resume: false removeData: false + integrityCheck.enabled: false db { provider: janusgraph janusgraph { diff --git a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala index db679f07a6..2d8a1c43e8 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/Migrate.scala @@ -91,6 +91,9 @@ object Migrate extends App with MigrationOps { opt[Int]('t', "thread-count") .text("number of threads") .action((t, c) => addConfig(c, "threadCount", t)), + opt[Unit]('k', "integrity-checks") + .text("run integrity checks after the migration") + .action((_, c) => addConfig(c, "output.integrityCheck.enabled", true)), /* case age */ opt[String]("max-case-age") .valueName("") diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/DummyActor.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/DummyActor.scala index 5179c3c26d..c1ba722963 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/DummyActor.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/DummyActor.scala @@ -1,7 +1,20 @@ package org.thp.thehive.migration.th4 -import akka.actor.Actor +import akka.actor.typed.scaladsl.Behaviors +import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps +import akka.actor.{Actor, ActorSystem} +import akka.actor.typed.{ActorRef => TypedActorRef} + +import java.util.UUID +import javax.inject.{Inject, Provider} class DummyActor extends Actor { override def receive: Receive = PartialFunction.empty } + +class DummyTypedActorProvider[T] @Inject() (actorSystem: ActorSystem) extends Provider[TypedActorRef[T]] { + override def get(): TypedActorRef[T] = + actorSystem + .toTyped + .systemActorOf(Behaviors.empty, UUID.randomUUID().toString) +} diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 5406c568eb..9e30053e43 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -55,7 +55,7 @@ object Output { bindActor[DummyActor]("notification-actor") bindActor[DummyActor]("config-actor") bindActor[DummyActor]("cortex-actor") - bindActor[DummyActor]("integrity-check-actor") + bind[ActorRef[IntegrityCheck.Request]].toProvider[DummyTypedActorProvider[IntegrityCheck.Request]] bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider] val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[IntegrityCheck](binder) integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheck] @@ -131,6 +131,7 @@ class Output @Inject() ( resolutionStatusSrv: ResolutionStatusSrv, jobSrv: JobSrv, actionSrv: ActionSrv, + dashboardSrv: DashboardSrv, db: Database, cache: SyncCacheApi, checks: immutable.Set[IntegrityCheck] @@ -204,23 +205,23 @@ class Output @Inject() ( db.addSchemaIndexes(theHiveSchema) .flatMap(_ => db.addSchemaIndexes(cortexSchema)) .foreach { _ => - implicit val authContext: AuthContext = LocalUserSrv.getSystemAuthContext - checks.foreach { c => - logger.info(s"Running check on ${c.name} ...") - val desupStats = c match { - case dc: DedupCheck[_] => dc.dedup(KillSwitch.alwaysOn) - case _ => Map.empty[String, Long] - } - val globalStats = c match { - case gc: GlobalCheck[_] => gc.runGlobalCheck(24.hours, KillSwitch.alwaysOn) - case _ => Map.empty[String, Long] + if (configuration.get[Boolean]("integrityCheck.enabled")) + checks.foreach { c => + logger.info(s"Running check on ${c.name} ...") + val desupStats = c match { + case dc: DedupCheck[_] => dc.dedup(KillSwitch.alwaysOn) + case _ => Map.empty[String, Long] + } + val globalStats = c match { + case gc: GlobalCheck[_] => gc.runGlobalCheck(24.hours, KillSwitch.alwaysOn) + case _ => Map.empty[String, Long] + } + val statsStr = (desupStats <+> globalStats) + .collect { case (k, v) if v != 0 => s"$k:$v" } + .mkString(" ") + if (statsStr.isEmpty) logger.info(s"Check on ${c.name}: no change needed") + else logger.info(s"Check on ${c.name}: $statsStr") } - val statsStr = (desupStats <+> globalStats) - .collect { case (k, v) if v != 0 => s"$k:$v" } - .mkString(" ") - if (statsStr.isEmpty) logger.info(s"Check on ${c.name}: no change needed") - else logger.info(s"Check on ${c.name}: $statsStr") - } } Try(db.close()) @@ -874,4 +875,22 @@ class Output @Inject() ( _ <- context.map(auditSrv.auditContextSrv.create(AuditContext(), createdAudit, _)).flip } yield () } + + def dashboardExists(graph: Graph, inputDashboard: InputDashboard): Boolean = + if (!resumeMigration) false + else + db.roTransaction { implicit graph => + dashboardSrv.startTraversal.has(_.title, inputDashboard.dashboard.title).exists + } + + override def createDashboard(graph: Graph, inputDashboard: InputDashboard): Try[IdMapping] = + withAuthContext(inputDashboard.metaData.createdBy) { implicit authContext => + implicit val g: Graph = graph + logger.debug(s"Create dashboard ${inputDashboard.dashboard.title}") + for { + dashboard <- dashboardSrv.create(inputDashboard.dashboard).map(_.dashboard) + _ <- inputDashboard.organisation.map { case (org, writable) => dashboardSrv.share(dashboard, EntityName(org), writable) }.flip + _ = updateMetaData(dashboard, inputDashboard.metaData) + } yield IdMapping(inputDashboard.metaData.id, dashboard._id) + } } diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 3aaec79806..6bd46229ad 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -11,7 +11,6 @@ import org.thp.scalligraph.EntityId import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.janus.JanusDatabase import org.thp.scalligraph.models._ -import org.thp.scalligraph.traversal.Graph import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.thehive.services.LocalUserSrv import play.api.Logger @@ -525,16 +524,10 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .toSeq } - override lazy val initialValues: Seq[InitialValue[_]] = modelList.collect { - case vertexModel: VertexModel => vertexModel.getInitialValues - }.flatten - private def tagString(namespace: String, predicate: String, value: String): String = (if (namespace.headOption.getOrElse('_') == '_') "" else namespace + ':') + (if (predicate.headOption.getOrElse('_') == '_') "" else predicate) + (if (value.isEmpty) "" else f"""="$value"""") - override def init(db: Database)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = Success(()) - override val authContext: AuthContext = LocalUserSrv.getSystemAuthContext } diff --git a/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala b/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala index 4fd066caf2..d69f303239 100644 --- a/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala +++ b/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala @@ -7,6 +7,8 @@ import akka.actor.typed.scaladsl.{Behaviors, TimerScheduler} import akka.cluster.typed.{ClusterSingleton, SingletonActor} import org.quartz import org.quartz._ +import org.thp.scalligraph.auth.AuthContext +import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.services.config.ApplicationConfig.finiteDurationFormat import org.thp.scalligraph.services.{DedupCheck, GlobalCheck, IntegrityCheck, KillSwitch} @@ -17,7 +19,7 @@ import play.api.libs.json._ import javax.inject.{Inject, Provider, Singleton} import scala.collection.immutable import scala.concurrent.duration.{DurationDouble, DurationLong, FiniteDuration} -import scala.util.Try +import scala.util.{Success, Try} case class CheckStats(global: Map[String, Long], last: Map[String, Long], lastDate: Long) { def +(stats: Map[String, Long]): CheckStats = { @@ -112,6 +114,7 @@ object IntegrityCheck { private val actorRefContextKey = "IntegrityCheck-actor" def behavior( + db: Database, quartzScheduler: quartz.Scheduler, appConfig: ApplicationConfig, integrityChecks: Seq[IntegrityCheck] @@ -128,6 +131,11 @@ object IntegrityCheck { } ) + db.tryTransaction { implicit graph => + implicit val authContext: AuthContext = LocalUserSrv.getSystemAuthContext + integrityChecks.foreach(_.initialCheck()) + Success(()) + } setupScheduling(context.self, quartzScheduler, integrityChecks, configItem) behavior(context.self, quartzScheduler, configItem, timers, integrityChecks.map(_.name)) } @@ -428,6 +436,7 @@ object IntegrityCheck { @Singleton class IntegrityCheckActorProvider @Inject() ( + db: Database, system: ActorSystem, quartzScheduler: quartz.Scheduler, appConfig: ApplicationConfig, @@ -435,5 +444,5 @@ class IntegrityCheckActorProvider @Inject() ( ) extends Provider[ActorRef[IntegrityCheck.Request]] { override lazy val get: ActorRef[IntegrityCheck.Request] = ClusterSingleton(system.toTyped) - .init(SingletonActor(IntegrityCheck.behavior(quartzScheduler, appConfig, integrityChecks.toSeq), "IntegrityCheckActor")) + .init(SingletonActor(IntegrityCheck.behavior(db, quartzScheduler, appConfig, integrityChecks.toSeq), "IntegrityCheckActor")) } diff --git a/thehive/app/org/thp/thehive/services/ProfileSrv.scala b/thehive/app/org/thp/thehive/services/ProfileSrv.scala index d2376361d4..075dbfd55f 100644 --- a/thehive/app/org/thp/thehive/services/ProfileSrv.scala +++ b/thehive/app/org/thp/thehive/services/ProfileSrv.scala @@ -14,7 +14,7 @@ import org.thp.thehive.services.ProfileOps._ import play.api.libs.json.JsObject import javax.inject.{Inject, Provider, Singleton} -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Try} @Singleton class ProfileSrv @Inject() (