#2334 Improve integrity checks

TheHive-Project · Feb 4, 2022 · 431760c · 431760c
1 parent ec179a8
commit 431760c
Show file tree

Hide file tree

Showing 41 changed files with 1,120 additions and 1,001 deletions.
diff --git a/ScalliGraph b/ScalliGraph
diff --git a/build.sbt b/build.sbt
@@ -165,7 +165,8 @@ lazy val thehiveCore = (project in file("thehive"))
       pbkdf2,
       commonCodec,
       scalaGuice,
-      reflections
+      reflections,
+      quartzScheduler
     )
   )
 

diff --git a/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala b/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala
@@ -20,6 +20,7 @@ import javax.inject.Inject
 import scala.collection.JavaConverters._
 import scala.collection.immutable
 import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.DurationInt
 import scala.util.Success
 
 trait IntegrityCheckApp {
@@ -43,22 +44,22 @@ trait IntegrityCheckApp {
               bindActor[DummyActor]("integrity-check-actor")
               bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
 
-              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
-              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
+              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[IntegrityCheck](binder)
+              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheck]
 
               bind[Environment].toInstance(Environment.simple())
               bind[ApplicationLifecycle].to[DefaultApplicationLifecycle]
@@ -77,25 +78,29 @@ trait IntegrityCheckApp {
     buildApp(configuration, db).getInstance(classOf[IntegrityChecks]).runChecks()
 }
 
-class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[GenIntegrityCheckOps], userSrv: UserDB) extends MapMerger {
+class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[IntegrityCheck], userSrv: UserDB) extends MapMerger {
   def runChecks(): Unit = {
     implicit val authContext: AuthContext = userSrv.getSystemAuthContext
     checks.foreach { c =>
-      db.tryTransaction { implicit graph =>
-        println(s"Running check on ${c.name} ...")
-        c.initialCheck()
-        val stats = c.duplicationCheck() <+> c.globalCheck()
-        val statsStr = stats
-          .collect {
-            case (k, v) if v != 0 => s"$k:$v"
-          }
-          .mkString(" ")
-        if (statsStr.isEmpty)
-          println("  no change needed")
-        else
-          println(s"  $statsStr")
-        Success(())
+      println(s"Running check on ${c.name} ...")
+      val desupStats = c match {
+        case dc: DedupCheck[_] => dc.dedup(KillSwitch.alwaysOn)
+        case _                 => Map.empty[String, Long]
       }
+      val globalStats = c match {
+        case gc: GlobalCheck[_] => gc.runGlobalCheck(24.hours, KillSwitch.alwaysOn)
+        case _                  => Map.empty[String, Long]
+      }
+      val statsStr = (desupStats <+> globalStats)
+        .collect {
+          case (k, v) if v != 0 => s"$k:$v"
+        }
+        .mkString(" ")
+      if (statsStr.isEmpty)
+        println("  no change needed")
+      else
+        println(s"  $statsStr")
+
     }
   }
 }
diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala
@@ -57,22 +57,22 @@ object Output {
               bindActor[DummyActor]("cortex-actor")
               bindActor[DummyActor]("integrity-check-actor")
               bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
-              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
-              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
+              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[IntegrityCheck](binder)
+              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheck]
 
               val schemaBindings = ScalaMultibinder.newSetBinder[UpdatableSchema](binder)
               schemaBindings.addBinding.to[TheHiveSchemaDefinition]
@@ -133,7 +133,7 @@ class Output @Inject() (
     actionSrv: ActionSrv,
     db: Database,
     cache: SyncCacheApi,
-    checks: immutable.Set[GenIntegrityCheckOps]
+    checks: immutable.Set[IntegrityCheck]
 ) extends migration.Output[Graph] {
   lazy val logger: Logger      = Logger(getClass)
   val resumeMigration: Boolean = configuration.get[Boolean]("resume")
@@ -206,17 +206,20 @@ class Output @Inject() (
       .foreach { _ =>
         implicit val authContext: AuthContext = LocalUserSrv.getSystemAuthContext
         checks.foreach { c =>
-          db.tryTransaction { implicit graph =>
-            logger.info(s"Running check on ${c.name} ...")
-            c.initialCheck()
-            val stats = c.duplicationCheck() <+> c.globalCheck()
-            val statsStr = stats
-              .collect { case (k, v) if v != 0 => s"$k:$v" }
-              .mkString(" ")
-            if (statsStr.isEmpty) logger.info(s"Check on ${c.name}: no change needed")
-            else logger.info(s"Check on ${c.name}: $statsStr")
-            Success(())
+          logger.info(s"Running check on ${c.name} ...")
+          val desupStats = c match {
+            case dc: DedupCheck[_] => dc.dedup(KillSwitch.alwaysOn)
+            case _                 => Map.empty[String, Long]
           }
+          val globalStats = c match {
+            case gc: GlobalCheck[_] => gc.runGlobalCheck(24.hours, KillSwitch.alwaysOn)
+            case _                  => Map.empty[String, Long]
+          }
+          val statsStr = (desupStats <+> globalStats)
+            .collect { case (k, v) if v != 0 => s"$k:$v" }
+            .mkString(" ")
+          if (statsStr.isEmpty) logger.info(s"Check on ${c.name}: no change needed")
+          else logger.info(s"Check on ${c.name}: $statsStr")
         }
       }
 

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -44,6 +44,7 @@ object Dependencies {
   lazy val scopt                   = "com.github.scopt"           %% "scopt"                              % "4.0.0"
   lazy val aix                     = "ai.x"                       %% "play-json-extensions"               % "0.42.0"
   lazy val bloomFilter             = "com.github.alexandrnikitin" %% "bloom-filter"                       % "0.13.1"
+  lazy val quartzScheduler         = "org.quartz-scheduler"        % "quartz"                             % "2.3.2"
 
   def scalaReflect(scalaVersion: String)  = "org.scala-lang" % "scala-reflect"  % scalaVersion
   def scalaCompiler(scalaVersion: String) = "org.scala-lang" % "scala-compiler" % scalaVersion

diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala
@@ -1,33 +1,31 @@
 package org.thp.thehive
 
-import akka.actor.ActorRef
 import akka.actor.typed.{ActorRef => TypedActorRef}
+import akka.actor.{ActorRef, ActorSystem}
 import com.google.inject.AbstractModule
 import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder}
+import org.quartz.Scheduler
+import org.quartz.impl.StdSchedulerFactory
 import org.thp.scalligraph.SingleInstance
 import org.thp.scalligraph.auth._
 import org.thp.scalligraph.janus.{ImmenseTermProcessor, JanusDatabaseProvider}
 import org.thp.scalligraph.models.{Database, UpdatableSchema}
-import org.thp.scalligraph.services.{GenIntegrityCheckOps, HadoopStorageSrv, S3StorageSrv}
-import org.thp.thehive.controllers.v0.QueryExecutorVersion0Provider
+import org.thp.scalligraph.query.QueryExecutor
+import org.thp.scalligraph.services.config.ConfigActor
+import org.thp.scalligraph.services.{IntegrityCheck, _}
+import org.thp.thehive.controllers.v0.{QueryExecutorVersion0Provider, TheHiveQueryExecutor => TheHiveQueryExecutorV0}
+import org.thp.thehive.controllers.v1.{TheHiveQueryExecutor => TheHiveQueryExecutorV1}
 import org.thp.thehive.models.{TheHiveSchemaDefinition, UseHashToIndex}
+import org.thp.thehive.services.notification.NotificationActor
 import org.thp.thehive.services.notification.notifiers._
 import org.thp.thehive.services.notification.triggers._
-import org.thp.thehive.services.{UserSrv => _, _}
+import org.thp.thehive.services.{Connector, LocalKeyAuthProvider, LocalPasswordAuthProvider, LocalUserSrv, UserSrv => _, _}
 import play.api.libs.concurrent.AkkaGuiceSupport
-//import org.thp.scalligraph.orientdb.{OrientDatabase, OrientDatabaseStorageSrv}
-import org.thp.scalligraph.services.config.ConfigActor
-import org.thp.scalligraph.services.{DatabaseStorageSrv, LocalFileSystemStorageSrv, StorageSrv}
-import org.thp.thehive.services.notification.NotificationActor
-import org.thp.thehive.services.{Connector, LocalKeyAuthProvider, LocalPasswordAuthProvider, LocalUserSrv}
-//import org.thp.scalligraph.neo4j.Neo4jDatabase
-//import org.thp.scalligraph.orientdb.OrientDatabase
-import org.thp.scalligraph.query.QueryExecutor
-import org.thp.thehive.controllers.v0.{TheHiveQueryExecutor => TheHiveQueryExecutorV0}
-import org.thp.thehive.controllers.v1.{TheHiveQueryExecutor => TheHiveQueryExecutorV1}
 import play.api.routing.{Router => PlayRouter}
 import play.api.{Configuration, Environment, Logger}
 
+import javax.inject.{Inject, Provider, Singleton}
+
 class TheHiveModule(environment: Environment, configuration: Configuration) extends AbstractModule with ScalaModule with AkkaGuiceSupport {
   lazy val logger: Logger = Logger(getClass)
 
@@ -90,25 +88,27 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte
     bindActor[ConfigActor]("config-actor")
     bindActor[NotificationActor]("notification-actor")
 
-    val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
-    integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
-    integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
-    bind[ActorRef].annotatedWithName("integrity-check-actor").toProvider[IntegrityCheckActorProvider]
+    val integrityChecksBindings = ScalaMultibinder.newSetBinder[IntegrityCheck](binder)
+    integrityChecksBindings.addBinding.to[ProfileIntegrityCheck]
+    integrityChecksBindings.addBinding.to[OrganisationIntegrityCheck]
+    integrityChecksBindings.addBinding.to[TagIntegrityCheck]
+    integrityChecksBindings.addBinding.to[UserIntegrityCheck]
+    integrityChecksBindings.addBinding.to[ImpactStatusIntegrityCheck]
+    integrityChecksBindings.addBinding.to[ResolutionStatusIntegrityCheck]
+    integrityChecksBindings.addBinding.to[ObservableTypeIntegrityCheck]
+    integrityChecksBindings.addBinding.to[CustomFieldIntegrityCheck]
+    integrityChecksBindings.addBinding.to[CaseTemplateIntegrityCheck]
+    integrityChecksBindings.addBinding.to[DataIntegrityCheck]
+    integrityChecksBindings.addBinding.to[CaseIntegrityCheck]
+    integrityChecksBindings.addBinding.to[AlertIntegrityCheck]
+    integrityChecksBindings.addBinding.to[TaskIntegrityCheck]
+    integrityChecksBindings.addBinding.to[ObservableIntegrityCheck]
+    integrityChecksBindings.addBinding.to[LogIntegrityCheck]
+    bind[TypedActorRef[IntegrityCheck.Request]].toProvider[IntegrityCheckActorProvider].asEagerSingleton()
     bind[TypedActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
 
+    bind[Scheduler].toProvider[QuartzSchedulerProvider].asEagerSingleton()
+
     bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider]
 
     bind[SingleInstance].to[ClusterSetup].asEagerSingleton()
@@ -117,3 +117,15 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte
     ()
   }
 }
+
+@Singleton
+class QuartzSchedulerProvider @Inject() (actorSystem: ActorSystem) extends Provider[Scheduler] {
+  override def get(): Scheduler = {
+    val factory = new StdSchedulerFactory
+    factory.initialize()
+    val scheduler = factory.getScheduler()
+    actorSystem.registerOnTermination(scheduler.shutdown())
+    scheduler.start()
+    scheduler
+  }
+}
diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseRenderer.scala
@@ -5,7 +5,6 @@ import java.util.{Collection => JCollection, List => JList, Map => JMap}
 
 import org.apache.tinkerpop.gremlin.process.traversal.P
 import org.thp.scalligraph.auth.AuthContext
-import org.thp.scalligraph.traversal.Converter.CList
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{Converter, IdentityConverter, Traversal}
 import org.thp.thehive.models._
@@ -50,7 +49,7 @@ trait CaseRenderer {
 
   def mergeIntoStats: Traversal.V[Case] => Traversal[JsNull.type, JsNull.type, IdentityConverter[JsNull.type]] = _.constant(JsNull)
 
-  def sharedWithStats: Traversal.V[Case] => Traversal[Seq[String], JList[String], CList[String, String, Converter[String, String]]] =
+  def sharedWithStats: Traversal.V[Case] => Traversal[Seq[String], JList[String], Converter.CList[String, String, Converter[String, String]]] =
     _.organisations.value(_.name).fold
 
   def originStats: Traversal.V[Case] => Traversal[String, String, Converter[String, String]] = _.origin.value(_.name)

diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala
@@ -2,7 +2,6 @@ package org.thp.thehive.controllers.v0
 
 import net.lingala.zip4j.ZipFile
 import net.lingala.zip4j.model.FileHeader
-import org.apache.tinkerpop.gremlin.process.traversal.Compare
 import org.thp.scalligraph._
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers._
@@ -11,7 +10,6 @@ import org.thp.scalligraph.query.PredicateOps.PredicateOpsDefs
 import org.thp.scalligraph.query._
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{IteratorOutput, Traversal}
-import org.thp.scalligraph.utils.Hasher
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.{InputAttachment, InputObservable}
 import org.thp.thehive.models._

diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableRenderer.scala
@@ -1,7 +1,6 @@
 package org.thp.thehive.controllers.v0
 
 import org.thp.scalligraph.auth.AuthContext
-import org.thp.scalligraph.traversal.Traversal.V
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{Converter, Traversal}
 import org.thp.thehive.controllers.v0.Conversion._
@@ -32,7 +31,7 @@ trait ObservableRenderer {
         )
       }
 
-  def observableLinkRenderer: V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] =
+  def observableLinkRenderer: Traversal.V[Observable] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] =
     _.coalesceMulti(
       _.alert.richAlert.domainMap(a => Json.obj("alert" -> a.toJson)),
       _.`case`.richCaseWithoutPerms.domainMap(c => Json.obj("case" -> c.toJson)),
+1 −1		core-test/src/test/scala/org/thp/scalligraph/models/Mesh.scala
+74 −83		core-test/src/test/scala/org/thp/scalligraph/services/IntegrityCheckTest.scala
+19 −14		core/src/main/scala/org/thp/scalligraph/models/Database.scala
+1 −1		core/src/main/scala/org/thp/scalligraph/models/Operation.scala
+24 −4		core/src/main/scala/org/thp/scalligraph/package.scala
+196 −101		core/src/main/scala/org/thp/scalligraph/services/IntegrityCheckOps.scala
+5 −5		core/src/main/scala/org/thp/scalligraph/services/VertexSrv.scala
+13 −10		core/src/main/scala/org/thp/scalligraph/traversal/TraversalOps.scala