From 5667b7e8380d69dcdb76a6cb87db698599dcf313 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 17 May 2021 18:50:11 +0200 Subject: [PATCH] #2033 Rewrite integrity chekcs --- ScalliGraph | 2 +- .../app/org/thp/thehive/IndexCleanup.scala | 43 ++++++ .../org/thp/thehive/services/AlertSrv.scala | 142 +++++------------- .../org/thp/thehive/services/CaseSrv.scala | 118 ++++----------- .../thehive/services/CaseTemplateSrv.scala | 8 +- .../thp/thehive/services/CustomFieldSrv.scala | 2 +- .../org/thp/thehive/services/DataSrv.scala | 8 +- .../thehive/services/ImpactStatusSrv.scala | 2 +- .../app/org/thp/thehive/services/LogSrv.scala | 53 ++----- .../thp/thehive/services/ObservableSrv.scala | 6 +- .../thehive/services/ObservableTypeSrv.scala | 2 +- .../thehive/services/OrganisationSrv.scala | 2 +- .../org/thp/thehive/services/ProfileSrv.scala | 2 +- .../services/ResolutionStatusSrv.scala | 2 +- .../app/org/thp/thehive/services/TagSrv.scala | 12 +- .../org/thp/thehive/services/UserSrv.scala | 14 +- 16 files changed, 152 insertions(+), 266 deletions(-) create mode 100644 thehive/app/org/thp/thehive/IndexCleanup.scala diff --git a/ScalliGraph b/ScalliGraph index 29b0b1a6cf..8fefbc7ddb 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 29b0b1a6cff505bf8b95c26cba59d14267b2b74e +Subproject commit 8fefbc7ddb9bf84835a6e4aa452026684ca66d8c diff --git a/thehive/app/org/thp/thehive/IndexCleanup.scala b/thehive/app/org/thp/thehive/IndexCleanup.scala new file mode 100644 index 0000000000..b28a700b0b --- /dev/null +++ b/thehive/app/org/thp/thehive/IndexCleanup.scala @@ -0,0 +1,43 @@ +package org.thp.thehive + +import org.apache.tinkerpop.gremlin.structure.{Direction, Property} +import org.janusgraph.core.JanusGraph +import org.janusgraph.core.schema.{JanusGraphManagement, Parameter} +import org.janusgraph.graphdb.database.StandardJanusGraph +import org.janusgraph.graphdb.database.management.ManagementSystem +import org.janusgraph.graphdb.internal.JanusGraphSchemaCategory +import org.janusgraph.graphdb.types.TypeDefinitionDescription +import org.janusgraph.graphdb.types.system.BaseLabel +import org.thp.scalligraph.janus.JanusDatabase + +import scala.jdk.CollectionConverters._ + +class IndexCleanup(db: JanusDatabase) { + def propertyStr[A](property: Property[A]): String = { + val p = property.asInstanceOf[Property[TypeDefinitionDescription]] + def modStr(modifier: Any): String = + modifier match { + case a: Array[_] => a.map(modStr).mkString("[", ",", "]") + case p: Parameter[_] => s"${p.key()}=${p.value()}" + case _ => modifier.toString + } + s"${p.key}=${p.value.getCategory}:${modStr(p.value.getModifier)}" + } + + db.managementTransaction { mgmt => + val tx = mgmt.asInstanceOf[ManagementSystem].getWrappedTx + val indexVertex = tx.getSchemaVertex(JanusGraphSchemaCategory.GRAPHINDEX.getSchemaName("global")) + indexVertex.remove() + val edges = tx + .query(indexVertex) + .`type`(BaseLabel.SchemaDefinitionEdge) + .direction(Direction.BOTH) + .edges() + .asScala + edges + .map(e => e.edgeLabel() + ": " + e.properties().asScala.map(propertyStr).mkString("<", " - ", ">")) + .mkString("\n") + indexVertex.remove() + ??? + } +} diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 23a3b9253a..6b9cbb4c04 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -1,7 +1,7 @@ package org.thp.thehive.services import akka.actor.ActorRef -import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} +import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.controllers.FFile import org.thp.scalligraph.models._ @@ -597,120 +597,48 @@ object AlertOps { implicit class AlertCustomFieldsOpsDefs(traversal: Traversal.E[AlertCustomField]) extends CustomFieldValueOpsDefs(traversal) } -class AlertIntegrityCheckOps @Inject() (val db: Database, val service: AlertSrv, organisationSrv: OrganisationSrv) extends IntegrityCheckOps[Alert] { +class AlertIntegrityCheckOps @Inject() (val db: Database, val service: AlertSrv, caseSrv: CaseSrv, organisationSrv: OrganisationSrv) + extends IntegrityCheckOps[Alert] { override def resolve(entities: Seq[Alert with Entity])(implicit graph: Graph): Try[Unit] = { val (imported, notImported) = entities.partition(_.caseId.isDefined) - if (imported.nonEmpty && notImported.nonEmpty) + val remainingAlerts = if (imported.nonEmpty && notImported.nonEmpty) { // Remove all non imported alerts service.getByIds(notImported.map(_._id): _*).remove() + imported + } else entities // Keep the last created alert - lastCreatedEntity(entities).foreach(e => service.getByIds(e._2.map(_._id): _*).remove()) + EntitySelector.lastCreatedEntity(remainingAlerts).foreach(e => service.getByIds(e._2.map(_._id): _*).remove()) Success(()) } - override def globalCheck(): Map[String, Long] = { - implicit val authContext: AuthContext = LocalUserSrv.getSystemAuthContext - - val multiImport = db.tryTransaction { implicit graph => - // Remove extra link with case - val linkIds = service - .startTraversal - .flatMap(_.outE[AlertCase].range(1, 100)._id) - .toSeq - if (linkIds.nonEmpty) - graph.E[AlertCase](linkIds: _*).remove() - Success(linkIds.length.toLong) - } - - val orgMetrics: Map[String, Long] = db - .tryTransaction { implicit graph => - // Check links with organisation - Try { - service - .startTraversal - .project( - _.by - .by(_.organisation._id.fold) - ) - .toIterator - .flatMap { - case (alert, Seq(organisationId)) if alert.organisationId == organisationId => None // It's OK - - case (alert, Seq(organisationId)) => - logger.warn( - s"Invalid organisationId in alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}), " + - s"got ${alert.organisationId}, should be $organisationId. Fixing it." - ) - service.get(alert).update(_.organisationId, organisationId).iterate() - Some("invalidOrganisationId") - - case (alert, organisationIds) if organisationIds.isEmpty => - organisationSrv.getOrFail(alert.organisationId) match { - case Success(organisation) => - logger.warn( - s"Link between alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}) and " + - s"organisation ${alert.organisationId} has disappeared. Fixing it." - ) - service - .alertOrganisationSrv - .create(AlertOrganisation(), alert, organisation) - .fold( - error => { - logger.error( - s"Fail to create link between alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}) " + - s"and organisation ${alert.organisationId}", - error - ) - Some("missingOrganisationAndFail") - }, - _ => Some("missingOrganisation") - ) - case _ => - logger.warn( - s"Alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}) is not linked to " + - s"existing organisation. Fixing it." - ) - service.get(alert).remove() - Some("nonExistentOrganisation") - } - - case (alert, organisationIds) if organisationIds.contains(alert.organisationId) => - val (extraLinks, extraOrganisationIds) = organisationIds.partition(_ == alert.organisationId) - if (extraOrganisationIds.nonEmpty) { - logger.warn( - s"Alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}) is not linked to " + - s"extra organisation(s): ${extraOrganisationIds.mkString(",")}. Fixing it." - ) - service.get(alert).outE[AlertOrganisation].filter(_.inV.hasId(extraOrganisationIds: _*)).remove() - } - if (extraLinks.length > 1) { - logger.warn( - s"Alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}) is linked more than once to " + - s"organisation: ${alert.organisationId}. Fixing it." - ) - service.get(alert).flatMap(_.outE[AlertOrganisation].range(1, 100)).remove() - } - Some("extraOrganisation") - - case (alert, organisationIds) => - logger.warn( - s"Alert ${alert._id}(${alert.`type`}:${alert.source}:${alert.sourceRef}) has inconsistent organisation links: " + - s"organisation is ${alert.organisationId} but links are ${organisationIds.mkString(",")}. Fixing it." - ) - service.get(alert).flatMap(_.outE[AlertOrganisation].sort(_.by("_createdAt", Order.asc)).range(1, 100)).remove() - service.get(alert).organisation._id.getOrFail("Organisation").foreach { organisationId => - service.get(alert).update(_.organisationId, organisationId).iterate() - } - Some("incoherent") - } - .toSeq - } + override def globalCheck(): Map[String, Int] = + db.tryTransaction { implicit graph => + Try { + service + .startTraversal + .project( + _.by + .by(_.`case`._id.fold) + .by(_.organisation._id.fold) + .by(_.removeDuplicateOutEdges[AlertCase]()) + .by(_.removeDuplicateOutEdges[AlertOrganisation]()) + ) + .toIterator + .map { + case (alert, caseIds, orgIds, extraCaseEdges, extraOrgEdges) => + val caseStats = singleIdLink[Case]("caseId", caseSrv)(_.outEdge[AlertCase], _.set(EntityId.empty)) +// alert => cases => { +// service.get(alert).outE[AlertCase].filter(_.inV.hasId(cases.map(_._id): _*)).project(_.by.by(_.inV.v[Case])).toSeq +// } + .check(alert, alert.caseId, caseIds) + val orgStats = singleIdLink[Organisation]("organisationId", organisationSrv)(_.outEdge[AlertOrganisation], _.remove) + .check(alert, alert.organisationId, orgIds) + + caseStats <+> orgStats <+> extraCaseEdges <+> extraOrgEdges + } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - .getOrElse(Seq("globalFailure")) - .groupBy(identity) - .mapValues(_.size.toLong) - - orgMetrics + ("multiImport" -> multiImport.getOrElse(0L)) - } + }.getOrElse(Map("Alert-globalFailure" -> 1)) } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 9abec18158..4480526f91 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -688,87 +688,31 @@ object CaseOps { } } -class CaseIntegrityCheckOps @Inject() (val db: Database, val service: CaseSrv, userSrv: UserSrv, caseTemplateSrv: CaseTemplateSrv) - extends IntegrityCheckOps[Case] { - def removeDuplicates(): Unit = - findDuplicates() - .foreach { entities => - db.tryTransaction { implicit graph => - resolve(entities) - } - } +class CaseIntegrityCheckOps @Inject() ( + val db: Database, + val service: CaseSrv, + userSrv: UserSrv, + caseTemplateSrv: CaseTemplateSrv, + organisationSrv: OrganisationSrv +) extends IntegrityCheckOps[Case] { override def resolve(entities: Seq[Case with Entity])(implicit graph: Graph): Try[Unit] = { val nextNumber = service.nextCaseNumber - firstCreatedEntity(entities).foreach( - _._2 - .flatMap(service.get(_).setConverter[Vertex, Converter.Identity[Vertex]](Converter.identity).headOption) - .zipWithIndex - .foreach { - case (vertex, index) => - UMapping.int.setProperty(vertex, "number", nextNumber + index) - } - ) + EntitySelector + .firstCreatedEntity(entities) + .foreach( + _._2 + .flatMap(service.get(_).setConverter[Vertex, Converter.Identity[Vertex]](Converter.identity).headOption) + .zipWithIndex + .foreach { + case (vertex, index) => + UMapping.int.setProperty(vertex, "number", nextNumber + index) + } + ) Success(()) } - private def organisationCheck(`case`: Case with Entity, organisationIds: Set[EntityId])(implicit graph: Graph): Seq[String] = - if (`case`.organisationIds == organisationIds) Nil - else { - service.get(`case`).update(_.organisationIds, organisationIds).iterate() - Seq("invalidOrganisationIds") - } - - private def assigneeCheck(`case`: Case with Entity, assignees: Seq[String])(implicit graph: Graph, authContext: AuthContext): Seq[String] = - `case`.assignee match { - case None if assignees.isEmpty => Nil - case Some(a) if assignees == Seq(a) => Nil - case None if assignees.size == 1 => - service.get(`case`).update(_.assignee, assignees.headOption).iterate() - Seq("invalidAssigneeLink") - case Some(a) if assignees.isEmpty => - userSrv.getByName(a).getOrFail("User") match { - case Success(user) => - service.caseUserSrv.create(CaseUser(), `case`, user) - Seq("missingAssigneeLink") - case _ => - service.get(`case`).update(_.assignee, None).iterate() - Seq("invalidAssignee") - } - case None if assignees.toSet.size == 1 => - service.get(`case`).update(_.assignee, assignees.headOption).flatMap(_.outE[CaseUser].range(1, 100)).remove() - Seq("multiAssignment") - case _ => - service.get(`case`).flatMap(_.outE[CaseUser].sort(_.by("_createdAt", Order.desc)).range(1, 100)).remove() - service.get(`case`).update(_.assignee, service.get(`case`).assignee.value(_.login).headOption).iterate() - Seq("incoherentAssignee") - } - - def caseTemplateCheck(`case`: Case with Entity, caseTemplates: Seq[String])(implicit graph: Graph, authContext: AuthContext): Seq[String] = - `case`.caseTemplate match { - case None if caseTemplates.isEmpty => Nil - case Some(ct) if caseTemplates == Seq(ct) => Nil - case None if caseTemplates.size == 1 => - service.get(`case`).update(_.caseTemplate, caseTemplates.headOption).iterate() - Seq("invalidCaseTemplateLink") - case Some(ct) if caseTemplates.isEmpty => - caseTemplateSrv.getByName(ct).getOrFail("User") match { - case Success(caseTemplate) => - service.caseCaseTemplateSrv.create(CaseCaseTemplate(), `case`, caseTemplate) - Seq("missingCaseTemplateLink") - case _ => - service.get(`case`).update(_.caseTemplate, None).iterate() - Seq("invalidCaseTemplate") - } - case None if caseTemplates.toSet.size == 1 => - service.get(`case`).update(_.caseTemplate, caseTemplates.headOption).flatMap(_.outE[CaseCaseTemplate].range(1, 100)).remove() - Seq("multiCaseTemplate") - case _ => - service.get(`case`).flatMap(_.outE[CaseCaseTemplate].sort(_.by("_createdAt", Order.asc)).range(1, 100)).remove() - service.get(`case`).update(_.caseTemplate, service.get(`case`).caseTemplate.value(_.name).headOption).iterate() - Seq("incoherentCaseTemplate") - } - override def globalCheck(): Map[String, Long] = { + override def globalCheck(): Map[String, Int] = { implicit val authContext: AuthContext = LocalUserSrv.getSystemAuthContext db.tryTransaction { implicit graph => @@ -782,17 +726,21 @@ class CaseIntegrityCheckOps @Inject() (val db: Database, val service: CaseSrv, u .by(_.caseTemplate.value(_.name).fold) ) .toIterator - .flatMap { - case (case0, organisationIds, assigneeIds, caseTemplateNames) if organisationIds.nonEmpty => - organisationCheck(case0, organisationIds.toSet) ++ assigneeCheck(case0, assigneeIds) ++ caseTemplateCheck(case0, caseTemplateNames) - case (case0, _, _, _) => - service.get(case0).remove() - Seq("orphan") + .map { + case (case0, organisationIds, assigneeIds, caseTemplateNames) => + val assigneeStats = singleOptionLink[User, String]("assignee", userSrv.getByName(_).head, _.login)(_.outEdge[CaseUser]) + .check(case0, case0.assignee, assigneeIds) + val orgStats = multiIdLink[Organisation]("organisationIds", organisationSrv)(_.remove) + .check(case0, case0.organisationIds.toSeq, organisationIds) + val templateStats = + singleOptionLink[CaseTemplate, String]("caseTemplate", caseTemplateSrv.getByName(_).head, _.name)(_.outEdge[CaseCaseTemplate]) + .check(case0, case0.caseTemplate, caseTemplateNames) + + assigneeStats <+> orgStats <+> templateStats } - .toSeq + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - }.getOrElse(Seq("globalFailure")) - .groupBy(identity) - .mapValues(_.size.toLong) + }.getOrElse(Map("globalFailure" -> 1)) } } diff --git a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala index 413d96b57f..e712d310fb 100644 --- a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala @@ -280,7 +280,7 @@ class CaseTemplateIntegrityCheckOps @Inject() ( val service: CaseTemplateSrv, organisationSrv: OrganisationSrv ) extends IntegrityCheckOps[CaseTemplate] { - override def findDuplicates: Seq[Seq[CaseTemplate with Entity]] = + override def findDuplicates(): Seq[Seq[CaseTemplate with Entity]] = db.roTransaction { implicit graph => organisationSrv .startTraversal @@ -306,7 +306,7 @@ class CaseTemplateIntegrityCheckOps @Inject() ( case _ => Success(()) } - override def globalCheck(): Map[String, Long] = + override def globalCheck(): Map[String, Int] = db.tryTransaction { implicit graph => Try { val orphanIds = service.startTraversal.filterNot(_.organisation)._id.toSeq @@ -314,7 +314,7 @@ class CaseTemplateIntegrityCheckOps @Inject() ( logger.warn(s"Found ${orphanIds.length} caseTemplate orphan(s) (${orphanIds.mkString(",")})") service.getByIds(orphanIds: _*).remove() } - Map("orphans" -> orphanIds.size.toLong) + Map("orphans" -> orphanIds.size) } - }.getOrElse(Map("globalFailure" -> 1L)) + }.getOrElse(Map("globalFailure" -> 1)) } diff --git a/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala b/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala index 9f379b5400..3cbdce18ff 100644 --- a/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala +++ b/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala @@ -188,5 +188,5 @@ class CustomFieldIntegrityCheckOps @Inject() (val db: Database, val service: Cus case _ => Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty } diff --git a/thehive/app/org/thp/thehive/services/DataSrv.scala b/thehive/app/org/thp/thehive/services/DataSrv.scala index 957ae3209e..14c07c4d5d 100644 --- a/thehive/app/org/thp/thehive/services/DataSrv.scala +++ b/thehive/app/org/thp/thehive/services/DataSrv.scala @@ -86,14 +86,14 @@ class DataIntegrityCheckOps @Inject() (val db: Database, val service: DataSrv) e case _ => Success(()) } - override def globalCheck(): Map[String, Long] = + override def globalCheck(): Map[String, Int] = db.tryTransaction { implicit graph => Try { val orphans = service.startTraversal.filterNot(_.inE[ObservableData])._id.toSeq if (orphans.nonEmpty) { service.getByIds(orphans: _*).remove() - Map("orphan" -> orphans.size.toLong) - } else Map.empty[String, Long] + Map("orphan" -> orphans.size) + } else Map.empty[String, Int] } - }.getOrElse(Map("globalFailure" -> 1L)) + }.getOrElse(Map("globalFailure" -> 1)) } diff --git a/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala b/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala index 3444fd8315..7379426722 100644 --- a/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala +++ b/thehive/app/org/thp/thehive/services/ImpactStatusSrv.scala @@ -52,5 +52,5 @@ class ImpactStatusIntegrityCheckOps @Inject() (val db: Database, val service: Im case _ => Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty } diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 3ffe236605..e3d452dc38 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -112,7 +112,7 @@ object LogOps { class LogIntegrityCheckOps @Inject() (val db: Database, val service: LogSrv, taskSrv: TaskSrv) extends IntegrityCheckOps[Log] { override def resolve(entities: Seq[Log with Entity])(implicit graph: Graph): Try[Unit] = Success(()) - override def globalCheck(): Map[String, Long] = { + override def globalCheck(): Map[String, Int] = { implicit val authContext: AuthContext = LocalUserSrv.getSystemAuthContext db.tryTransaction { implicit graph => @@ -121,50 +121,17 @@ class LogIntegrityCheckOps @Inject() (val db: Database, val service: LogSrv, tas .startTraversal .project(_.by.by(_.task.fold)) .toIterator - .flatMap { + .map { case (log, tasks) => - val (extraLinks, extraTasks) = tasks.partition(_._id == log.taskId) - if (extraLinks.nonEmpty) - (if (extraLinks.length == 1) Nil - else { - service.get(log).inE[TaskLog].flatMap(_.range(1, 100)).remove() - Seq("extraTaskLink") - }) ++ - (if (extraTasks.isEmpty) Nil - else { - service.get(log).inE[TaskLog].filterNot(_.outV.hasId(log.taskId)).remove() - Seq("extraTask") - }) ++ - (if (log.organisationIds != extraLinks.head.organisationIds) { - service.get(log).update(_.organisationIds, extraLinks.head.organisationIds).iterate() - Seq("invalidOrganisationIds") - } else Nil) - else if (extraTasks.nonEmpty) - if (extraTasks.size == 1) { - service.get(log).update(_.taskId, extraTasks.head._id).update(_.organisationIds, extraTasks.head.organisationIds).iterate() - Seq("invalidTaskId") - } else { - service.get(log).remove() - Seq("incoherent") - } - else { - taskSrv.getOrFail(log.taskId) match { - case Success(task) => - service - .taskLogSrv - .create(TaskLog(), task, log) - service.get(log).update(_.organisationIds, task.organisationIds).iterate() - Seq("taskMissing") - case _ => Seq("nonExistentTask") - } - service.get(log).remove() - Seq("incoherent") - } + val taskStats = singleIdLink[Task]("taskId", taskSrv)(_.inEdge[TaskLog], _.remove).check(log, log.taskId, tasks.map(_._id)) + if (tasks.size == 1 && tasks.head.organisationIds != log.organisationIds) { + service.get(log).update(_.organisationIds, tasks.head.organisationIds).iterate() + taskStats + ("Log-invalidOrgs" -> 1) + } else taskStats } - .toSeq + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - }.getOrElse(Seq("globalFailure")) - .groupBy(identity) - .mapValues(_.size.toLong) + }.getOrElse(Map("globalFailure" -> 1)) } } diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index 915eec1471..64d9480897 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -358,7 +358,7 @@ object ObservableOps { class ObservableIntegrityCheckOps @Inject() (val db: Database, val service: ObservableSrv) extends IntegrityCheckOps[Observable] { override def resolve(entities: Seq[Observable with Entity])(implicit graph: Graph): Try[Unit] = Success(()) - override def globalCheck(): Map[String, Long] = + override def globalCheck(): Map[String, Int] = db.tryTransaction { implicit graph => Try { val orphanIds = service.startTraversal.filterNot(_.or(_.shares, _.alert, _.in("ReportObservable")))._id.toSeq @@ -366,7 +366,7 @@ class ObservableIntegrityCheckOps @Inject() (val db: Database, val service: Obse logger.warn(s"Found ${orphanIds.length} observables orphan(s) (${orphanIds.mkString(",")})") service.getByIds(orphanIds: _*).remove() } - Map("orphans" -> orphanIds.size.toLong) + Map("orphans" -> orphanIds.size) } - }.getOrElse(Map("globalFailure" -> 1L)) + }.getOrElse(Map("globalFailure" -> 1)) } diff --git a/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala b/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala index 6c9b255329..99d4e4c42c 100644 --- a/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableTypeSrv.scala @@ -63,5 +63,5 @@ class ObservableTypeIntegrityCheckOps @Inject() (val db: Database, val service: case _ => Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty } diff --git a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala index f94815b8ec..ec3c46fe0a 100644 --- a/thehive/app/org/thp/thehive/services/OrganisationSrv.scala +++ b/thehive/app/org/thp/thehive/services/OrganisationSrv.scala @@ -239,5 +239,5 @@ class OrganisationIntegrityCheckOps @Inject() (val db: Database, val service: Or case _ => Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty } diff --git a/thehive/app/org/thp/thehive/services/ProfileSrv.scala b/thehive/app/org/thp/thehive/services/ProfileSrv.scala index bf7f10117c..739548c9bf 100644 --- a/thehive/app/org/thp/thehive/services/ProfileSrv.scala +++ b/thehive/app/org/thp/thehive/services/ProfileSrv.scala @@ -92,5 +92,5 @@ class ProfileIntegrityCheckOps @Inject() (val db: Database, val service: Profile case _ => Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty } diff --git a/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala b/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala index a35f333143..a61e11ba28 100644 --- a/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala +++ b/thehive/app/org/thp/thehive/services/ResolutionStatusSrv.scala @@ -52,5 +52,5 @@ class ResolutionStatusIntegrityCheckOps @Inject() (val db: Database, val service case _ => Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty } diff --git a/thehive/app/org/thp/thehive/services/TagSrv.scala b/thehive/app/org/thp/thehive/services/TagSrv.scala index 8ba71be3ae..2af4390578 100644 --- a/thehive/app/org/thp/thehive/services/TagSrv.scala +++ b/thehive/app/org/thp/thehive/services/TagSrv.scala @@ -7,7 +7,7 @@ import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} -import org.thp.scalligraph.services.{EdgeSrv, IntegrityCheckOps, VertexSrv} +import org.thp.scalligraph.services.{EdgeSrv, EntitySelector, IntegrityCheckOps, VertexSrv} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Graph, Traversal} import org.thp.scalligraph.utils.FunctionalCondition.When @@ -176,7 +176,7 @@ object TagOps { class TagIntegrityCheckOps @Inject() (val db: Database, val service: TagSrv) extends IntegrityCheckOps[Tag] { override def resolve(entities: Seq[Tag with Entity])(implicit graph: Graph): Try[Unit] = { - firstCreatedEntity(entities).foreach { + EntitySelector.firstCreatedEntity(entities).foreach { case (head, tail) => tail.foreach(copyEdge(_, head)) val tailIds = tail.map(_._id) @@ -186,7 +186,7 @@ class TagIntegrityCheckOps @Inject() (val db: Database, val service: TagSrv) ext Success(()) } - override def globalCheck(): Map[String, Long] = + override def globalCheck(): Map[String, Int] = db.tryTransaction { implicit graph => Try { val orphans = service @@ -197,8 +197,8 @@ class TagIntegrityCheckOps @Inject() (val db: Database, val service: TagSrv) ext .toSeq if (orphans.nonEmpty) { service.getByIds(orphans: _*).remove() - Map("orphan" -> orphans.size.toLong) - } else Map.empty[String, Long] + Map("orphan" -> orphans.size) + } else Map.empty[String, Int] } - }.getOrElse(Map("globalFailure" -> 1L)) + }.getOrElse(Map("globalFailure" -> 1)) } diff --git a/thehive/app/org/thp/thehive/services/UserSrv.scala b/thehive/app/org/thp/thehive/services/UserSrv.scala index dcf8255262..da95d9091d 100644 --- a/thehive/app/org/thp/thehive/services/UserSrv.scala +++ b/thehive/app/org/thp/thehive/services/UserSrv.scala @@ -336,18 +336,18 @@ class UserIntegrityCheckOps @Inject() ( () } - override def duplicationCheck(): Map[String, Long] = { + override def duplicationCheck(): Map[String, Int] = { super.duplicationCheck() db.tryTransaction { implicit graph => val duplicateTaskAssignments = - duplicateInEdges[TaskUser](service.startTraversal).flatMap(firstCreatedElement(_)).map(e => removeEdges(e._2)).size.toLong + duplicateInEdges[TaskUser](service.startTraversal).flatMap(ElementSelector.firstCreatedElement(_)).map(e => removeEdges(e._2)).size val duplicateCaseAssignments = - duplicateInEdges[CaseUser](service.startTraversal).flatMap(firstCreatedElement(_)).map(e => removeEdges(e._2)).size.toLong + duplicateInEdges[CaseUser](service.startTraversal).flatMap(ElementSelector.firstCreatedElement(_)).map(e => removeEdges(e._2)).size val duplicateUsers = duplicateLinks[Vertex, Vertex]( service.startTraversal, (_.out("UserRole"), _.in("UserRole")), (_.out("RoleOrganisation"), _.in("RoleOrganisation")) - ).flatMap(firstCreatedElement(_)).map(e => removeVertices(e._2)).size.toLong + ).flatMap(ElementSelector.firstCreatedElement(_)).map(e => removeVertices(e._2)).size Success( Map( "duplicateTaskAssignments" -> duplicateTaskAssignments, @@ -355,11 +355,11 @@ class UserIntegrityCheckOps @Inject() ( "duplicateUsers" -> duplicateUsers ) ) - }.getOrElse(Map("globalFailure" -> 1L)) + }.getOrElse(Map("globalFailure" -> 1)) } override def resolve(entities: Seq[User with Entity])(implicit graph: Graph): Try[Unit] = { - firstCreatedEntity(entities).foreach { + EntitySelector.firstCreatedEntity(entities).foreach { case (firstUser, otherUsers) => otherUsers.foreach(copyEdge(_, firstUser)) otherUsers.foreach(service.get(_).remove()) @@ -367,5 +367,5 @@ class UserIntegrityCheckOps @Inject() ( Success(()) } - override def globalCheck(): Map[String, Long] = Map.empty + override def globalCheck(): Map[String, Int] = Map.empty }