From 5e42814868dd939dc0bba5d51db23ada93074b34 Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 22 Jan 2022 09:42:06 +0100 Subject: [PATCH] #2305 Use paged traversals --- .../org/thp/thehive/services/AlertSrv.scala | 74 +++++++++------ .../org/thp/thehive/services/CaseSrv.scala | 88 +++++++++++------- .../thehive/services/CaseTemplateSrv.scala | 49 ++++++++-- .../app/org/thp/thehive/services/LogSrv.scala | 39 ++++---- .../thp/thehive/services/ObservableSrv.scala | 90 +++++++++---------- .../app/org/thp/thehive/services/TagSrv.scala | 29 +++--- .../org/thp/thehive/services/TaskSrv.scala | 67 +++++++------- 7 files changed, 261 insertions(+), 175 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index fb9f208389..7537765634 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -598,7 +598,7 @@ object AlertOps { implicit class AlertCustomFieldsOpsDefs(traversal: Traversal.E[AlertCustomField]) extends CustomFieldValueOpsDefs(traversal) } -class AlertIntegrityCheckOps @Inject() (val db: Database, val service: AlertSrv, caseSrv: CaseSrv, organisationSrv: OrganisationSrv) +class AlertIntegrityCheckOps @Inject() (val db: Database, val service: AlertSrv, caseSrv: CaseSrv, organisationSrv: OrganisationSrv, tagSrv: TagSrv) extends IntegrityCheckOps[Alert] { override def resolve(entities: Seq[Alert with Entity])(implicit graph: Graph): Try[Unit] = { @@ -614,32 +614,52 @@ class AlertIntegrityCheckOps @Inject() (val db: Database, val service: AlertSrv, } override def globalCheck(): Map[String, Int] = - db.tryTransaction { implicit graph => - Try { - service - .startTraversal - .project( - _.by - .by(_.`case`._id.fold) - .by(_.organisation._id.fold) - .by(_.removeDuplicateOutEdges[AlertCase]()) - .by(_.removeDuplicateOutEdges[AlertOrganisation]()) - ) - .toIterator - .map { - case (alert, caseIds, orgIds, extraCaseEdges, extraOrgEdges) => - val caseStats = singleIdLink[Case]("caseId", caseSrv)(_.outEdge[AlertCase], _.set(EntityId.empty)) -// alert => cases => { -// service.get(alert).outE[AlertCase].filter(_.inV.hasId(cases.map(_._id): _*)).project(_.by.by(_.inV.v[Case])).toSeq -// } - .check(alert, alert.caseId, caseIds) - val orgStats = singleIdLink[Organisation]("organisationId", organisationSrv)(_.outEdge[AlertOrganisation], _.remove) - .check(alert, alert.organisationId, orgIds) - - caseStats <+> orgStats <+> extraCaseEdges <+> extraOrgEdges + service + .pagedTraversalIds(db, 100) { ids => + db.tryTransaction { implicit graph => + val caseCheck = singleIdLink[Case]("caseId", caseSrv)(_.outEdge[AlertCase], _.set(EntityId.empty)) + val orgCheck = singleIdLink[Organisation]("organisationId", organisationSrv)(_.outEdge[AlertOrganisation], _.remove) + Try { + service + .getByIds(ids: _*) + .project( + _.by + .by(_.`case`._id.fold) + .by(_.organisation._id.fold) + .by(_.removeDuplicateOutEdges[AlertCase]()) + .by(_.removeDuplicateOutEdges[AlertOrganisation]()) + .by(_.tags.fold) + ) + .toIterator + .map { + case (alert, caseIds, orgIds, extraCaseEdges, extraOrgEdges, tags) => + val caseStats = caseCheck.check(alert, alert.caseId, caseIds) + val orgStats = orgCheck.check(alert, alert.organisationId, orgIds) + val tagStats = { + val alertTagSet = alert.tags.toSet + val tagSet = tags.map(_.toString).toSet + if (alertTagSet == tagSet) Map.empty[String, Int] + else { + implicit val authContext: AuthContext = + LocalUserSrv.getSystemAuthContext.changeOrganisation(alert.organisationId, Permissions.all) + + val extraTagField = alertTagSet -- tagSet + val extraTagLink = tagSet -- alertTagSet + extraTagField.flatMap(tagSrv.getOrCreate(_).toOption).foreach(service.alertTagSrv.create(AlertTag(), alert, _)) + service.get(alert).update(_.tags, alert.tags ++ extraTagLink).iterate() + Map( + "case-tags-extraField" -> extraTagField.size, + "case-tags-extraLink" -> extraTagLink.size + ) + } + } + caseStats <+> orgStats <+> extraCaseEdges <+> extraOrgEdges <+> tagStats + } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - .reduceOption(_ <+> _) - .getOrElse(Map.empty) + }.getOrElse(Map("Alert-globalFailure" -> 1)) } - }.getOrElse(Map("Alert-globalFailure" -> 1)) + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 665870b313..2b058ded09 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -752,7 +752,8 @@ class CaseIntegrityCheckOps @Inject() ( val service: CaseSrv, userSrv: UserSrv, caseTemplateSrv: CaseTemplateSrv, - organisationSrv: OrganisationSrv + organisationSrv: OrganisationSrv, + tagSrv: TagSrv ) extends IntegrityCheckOps[Case] { override def resolve(entities: Seq[Case with Entity])(implicit graph: Graph): Try[Unit] = { @@ -770,37 +771,60 @@ class CaseIntegrityCheckOps @Inject() ( } override def globalCheck(): Map[String, Int] = - db.tryTransaction { implicit graph => - Try { - service - .startTraversal - .project( - _.by - .by(_.organisations._id.fold) - .by(_.assignee.value(_.login).fold) - .by(_.caseTemplate.value(_.name).fold) - .by(_.origin._id.fold) - ) - .toIterator - .map { - case (case0, organisationIds, assigneeIds, caseTemplateNames, owningOrganisationIds) => - val fixOwningOrg: LinkRemover = - (caseId, orgId) => service.get(caseId).shares.filter(_.organisation.get(orgId._id)).update(_.owner, false).iterate() - - val assigneeStats = singleOptionLink[User, String]("assignee", userSrv.getByName(_).head, _.login)(_.outEdge[CaseUser]) - .check(case0, case0.assignee, assigneeIds) - val orgStats = multiIdLink[Organisation]("organisationIds", organisationSrv)(_.remove) // FIXME => Seq => Set - .check(case0, case0.organisationIds, organisationIds) - val templateStats = - singleOptionLink[CaseTemplate, String]("caseTemplate", caseTemplateSrv.getByName(_).head, _.name)(_.outEdge[CaseCaseTemplate]) - .check(case0, case0.caseTemplate, caseTemplateNames) - val owningOrgStats = singleIdLink[Organisation]("owningOrganisation", organisationSrv)(_ => fixOwningOrg, _.remove) - .check(case0, case0.owningOrganisation, owningOrganisationIds) - - assigneeStats <+> orgStats <+> templateStats <+> owningOrgStats + service + .pagedTraversalIds(db, 100) { ids => + db.tryTransaction { implicit graph => + val assigneeCheck = singleOptionLink[User, String]("assignee", userSrv.getByName(_).head, _.login)(_.outEdge[CaseUser]) + val orgCheck = multiIdLink[Organisation]("organisationIds", organisationSrv)(_.remove) // FIXME => Seq => Set + val templateCheck = + singleOptionLink[CaseTemplate, String]("caseTemplate", caseTemplateSrv.getByName(_).head, _.name)(_.outEdge[CaseCaseTemplate]) + val fixOwningOrg: LinkRemover = + (caseId, orgId) => service.get(caseId).shares.filter(_.organisation.get(orgId._id)).update(_.owner, false).iterate() + val owningOrgCheck = singleIdLink[Organisation]("owningOrganisation", organisationSrv)(_ => fixOwningOrg, _.remove) + + Try { + service + .getByIds(ids: _*) + .project( + _.by + .by(_.organisations._id.fold) + .by(_.assignee.value(_.login).fold) + .by(_.caseTemplate.value(_.name).fold) + .by(_.origin._id.fold) + .by(_.tags.fold) + ) + .toIterator + .map { + case (case0, organisationIds, assigneeIds, caseTemplateNames, owningOrganisationIds, tags) => + val assigneeStats = assigneeCheck.check(case0, case0.assignee, assigneeIds) + val orgStats = orgCheck.check(case0, case0.organisationIds, organisationIds) + val templateStats = templateCheck.check(case0, case0.caseTemplate, caseTemplateNames) + val owningOrgStats = owningOrgCheck.check(case0, case0.owningOrganisation, owningOrganisationIds) + val tagStats = { + val caseTagSet = case0.tags.toSet + val tagSet = tags.map(_.toString).toSet + if (caseTagSet == tagSet) Map.empty[String, Int] + else { + implicit val authContext: AuthContext = + LocalUserSrv.getSystemAuthContext.changeOrganisation(case0.owningOrganisation, Permissions.all) + + val extraTagField = caseTagSet -- tagSet + val extraTagLink = tagSet -- caseTagSet + extraTagField.flatMap(tagSrv.getOrCreate(_).toOption).foreach(service.caseTagSrv.create(CaseTag(), case0, _)) + service.get(case0).update(_.tags, case0.tags ++ extraTagLink).iterate() + Map( + "case-tags-extraField" -> extraTagField.size, + "case-tags-extraLink" -> extraTagLink.size + ) + } + } + assigneeStats <+> orgStats <+> templateStats <+> owningOrgStats <+> tagStats + } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - .reduceOption(_ <+> _) - .getOrElse(Map.empty) + }.getOrElse(Map("globalFailure" -> 1)) } - }.getOrElse(Map("globalFailure" -> 1)) + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } diff --git a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala index 613557f3d5..583ebb9d8c 100644 --- a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala @@ -276,7 +276,8 @@ object CaseTemplateOps { class CaseTemplateIntegrityCheckOps @Inject() ( val db: Database, val service: CaseTemplateSrv, - organisationSrv: OrganisationSrv + organisationSrv: OrganisationSrv, + tagSrv: TagSrv ) extends IntegrityCheckOps[CaseTemplate] { override def findDuplicates(): Seq[Seq[CaseTemplate with Entity]] = db.roTransaction { implicit graph => @@ -307,12 +308,46 @@ class CaseTemplateIntegrityCheckOps @Inject() ( override def globalCheck(): Map[String, Int] = db.tryTransaction { implicit graph => Try { - val orphanIds = service.startTraversal.filterNot(_.organisation)._id.toSeq - if (orphanIds.nonEmpty) { - logger.warn(s"Found ${orphanIds.length} caseTemplate orphan(s) (${orphanIds.mkString(",")})") - service.getByIds(orphanIds: _*).remove() - } - Map("orphans" -> orphanIds.size) + service + .startTraversal + .project(_.by.by(_.organisation._id.fold).by(_.tags.fold)) + .toIterator + .map { + case (caseTemplate, organisationIds, tags) => + if (organisationIds.isEmpty) { + service.get(caseTemplate).remove() + Map("caseTemplate-orphans" -> 1) + } else { + val orgStats = if (organisationIds.size > 1) { + service.get(caseTemplate).out[CaseTemplateOrganisation].range(1, Int.MaxValue).remove() + Map("caseTemplate-organisation-extraLink" -> organisationIds.size) + } else Map.empty[String, Int] + val tagStats = { + val caseTemplateTagSet = caseTemplate.tags.toSet + val tagSet = tags.map(_.toString).toSet + if (caseTemplateTagSet == tagSet) Map.empty[String, Int] + else { + implicit val authContext: AuthContext = + LocalUserSrv.getSystemAuthContext.changeOrganisation(organisationIds.head, Permissions.all) + + val extraTagField = caseTemplateTagSet -- tagSet + val extraTagLink = tagSet -- caseTemplateTagSet + extraTagField + .flatMap(tagSrv.getOrCreate(_).toOption) + .foreach(service.caseTemplateTagSrv.create(CaseTemplateTag(), caseTemplate, _)) + service.get(caseTemplate).update(_.tags, caseTemplate.tags ++ extraTagLink).iterate() + Map( + "caseTemplate-tags-extraField" -> extraTagField.size, + "caseTemplate-tags-extraLink" -> extraTagLink.size + ) + } + } + + orgStats <+> tagStats + } + } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } }.getOrElse(Map("globalFailure" -> 1)) } diff --git a/thehive/app/org/thp/thehive/services/LogSrv.scala b/thehive/app/org/thp/thehive/services/LogSrv.scala index 20d69f69f4..1f9303c3e9 100644 --- a/thehive/app/org/thp/thehive/services/LogSrv.scala +++ b/thehive/app/org/thp/thehive/services/LogSrv.scala @@ -113,22 +113,29 @@ class LogIntegrityCheckOps @Inject() (val db: Database, val service: LogSrv, tas override def resolve(entities: Seq[Log with Entity])(implicit graph: Graph): Try[Unit] = Success(()) override def globalCheck(): Map[String, Int] = - db.tryTransaction { implicit graph => - Try { - service - .startTraversal - .project(_.by.by(_.task.fold)) - .toIterator - .map { - case (log, tasks) => - val taskStats = singleIdLink[Task]("taskId", taskSrv)(_.inEdge[TaskLog], _.remove).check(log, log.taskId, tasks.map(_._id)) - if (tasks.size == 1 && tasks.head.organisationIds != log.organisationIds) { - service.get(log).update(_.organisationIds, tasks.head.organisationIds).iterate() - taskStats + ("Log-invalidOrgs" -> 1) - } else taskStats + service + .pagedTraversalIds(db, 100) { ids => + println(s"get ids: ${ids.mkString(",")}") + db.tryTransaction { implicit graph => + val taskCheck = singleIdLink[Task]("taskId", taskSrv)(_.inEdge[TaskLog], _.remove) + Try { + service + .getByIds(ids: _*) + .project(_.by.by(_.task.fold)) + .toIterator + .map { + case (log, tasks) => + val taskStats = taskCheck.check(log, log.taskId, tasks.map(_._id)) + if (tasks.size == 1 && tasks.head.organisationIds != log.organisationIds) { + service.get(log).update(_.organisationIds, tasks.head.organisationIds).iterate() + taskStats + ("Log-invalidOrgs" -> 1) + } else taskStats + } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - .reduceOption(_ <+> _) - .getOrElse(Map.empty) + }.getOrElse(Map("globalFailure" -> 1)) } - }.getOrElse(Map("globalFailure" -> 1)) + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index bcbcf3d8c0..aed399153a 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -14,7 +14,6 @@ import org.thp.scalligraph.utils.Hash import org.thp.scalligraph.{BadRequestError, CreateError, EntityId, EntityIdOrName, EntityName, RichSeq} import org.thp.thehive.models._ import org.thp.thehive.services.AlertOps._ -import org.thp.thehive.services.DataOps._ import org.thp.thehive.services.ObservableOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.ShareOps._ @@ -22,6 +21,7 @@ import play.api.libs.json.{JsObject, JsString, Json} import java.util.{Date, Map => JMap} import javax.inject.{Inject, Provider, Singleton} +import scala.concurrent.ExecutionContext import scala.util.{Failure, Success, Try} @Singleton @@ -428,56 +428,48 @@ class ObservableIntegrityCheckOps @Inject() ( singleOptionLink[Data, String]("data", d => dataSrv.create(Data(d, None)).get, _.data)(_.outEdge[ObservableData]) } - val processStats = new ProcessStats - - processStats - .showStats(10.seconds, msg => logger.debug(s"observable integrity checks: $msg")) { - Try { - service - .getByIds(ids: _*) - .project( - _.by - .by(_.organisations._id.fold) - .by(_.unionFlat(_.`case`._id, _.alert._id, _.in("ReportObservable")._id).fold) - .by(_.data.value(_.data).fold) - .by(_.tags.fold) - ) - .toIterator - .map { - case (observable, organisationIds, relatedIds, data, tags) => - processStats("all-checks") { - // logger.debug(s"processing integrity check of ${observable.dataType}:${observable.data} (${observable.tags.mkString(",")}") - val orgStats = processStats("org-checks")(orgCheck.check(observable, observable.organisationIds, organisationIds)) - val relatedStats = processStats("related-checks")(relatedCheck.check(observable, observable.relatedId, relatedIds)) - val observableDataStats = processStats("data-checks")(observableDataCheck.check(observable, observable.data, data)) - val tagStats = processStats("tag-checks") { - val observableTagSet = observable.tags.toSet - val tagSet = tags.map(_.toString).toSet - if (observableTagSet == tagSet) Map.empty[String, Int] - else { - implicit val authContext: AuthContext = - LocalUserSrv.getSystemAuthContext.changeOrganisation(observable.organisationIds.head, Permissions.all) - - val extraTagField = observableTagSet -- tagSet - val extraTagLink = tagSet -- observableTagSet - extraTagField - .flatMap(tagSrv.getOrCreate(_).toOption) - .foreach(service.observableTagSrv.create(ObservableTag(), observable, _)) - service.get(observable).update(_.tags, observable.tags ++ extraTagLink).iterate() - Map( - "observable-tags-extraField" -> extraTagField.size, - "observable-tags-extraLink" -> extraTagLink.size - ) - } - } - - orgStats <+> relatedStats <+> observableDataStats <+> tagStats - } + Try { + service + .getByIds(ids: _*) + .project( + _.by + .by(_.organisations._id.fold) + .by(_.unionFlat(_.`case`._id, _.alert._id, _.in("ReportObservable")._id).fold) + .by(_.data.value(_.data).fold) + .by(_.tags.fold) + ) + .toIterator + .map { + case (observable, organisationIds, relatedIds, data, tags) => + val orgStats = orgCheck.check(observable, observable.organisationIds, organisationIds) + val relatedStats = relatedCheck.check(observable, observable.relatedId, relatedIds) + val observableDataStats = observableDataCheck.check(observable, observable.data, data) + val tagStats = { + val observableTagSet = observable.tags.toSet + val tagSet = tags.map(_.toString).toSet + if (observableTagSet == tagSet) Map.empty[String, Int] + else { + implicit val authContext: AuthContext = + LocalUserSrv.getSystemAuthContext.changeOrganisation(observable.organisationIds.head, Permissions.all) + + val extraTagField = observableTagSet -- tagSet + val extraTagLink = tagSet -- observableTagSet + extraTagField + .flatMap(tagSrv.getOrCreate(_).toOption) + .foreach(service.observableTagSrv.create(ObservableTag(), observable, _)) + service.get(observable).update(_.tags, observable.tags ++ extraTagLink).iterate() + Map( + "observable-tags-extraField" -> extraTagField.size, + "observable-tags-extraLink" -> extraTagLink.size + ) + } } - .reduceOption(_ <+> _) - .getOrElse(Map.empty) + + orgStats <+> relatedStats <+> observableDataStats <+> tagStats } - } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) + } }.getOrElse(Map("globalFailure" -> 1)) } .reduceOption(_ <+> _) diff --git a/thehive/app/org/thp/thehive/services/TagSrv.scala b/thehive/app/org/thp/thehive/services/TagSrv.scala index 32cd727355..d5a429b129 100644 --- a/thehive/app/org/thp/thehive/services/TagSrv.scala +++ b/thehive/app/org/thp/thehive/services/TagSrv.scala @@ -188,18 +188,21 @@ class TagIntegrityCheckOps @Inject() (val db: Database, val service: TagSrv) ext } override def globalCheck(): Map[String, Int] = - db.tryTransaction { implicit graph => - Try { - val orphans = service - .startTraversal - .filter(_.taxonomy.has(_.namespace, TextP.startingWith("_freetags_"))) - .filterNot(_.or(_.inE[AlertTag], _.inE[ObservableTag], _.inE[CaseTag], _.inE[CaseTemplateTag])) - ._id - .toSeq - if (orphans.nonEmpty) { - service.getByIds(orphans: _*).remove() - Map("orphan" -> orphans.size) - } else Map.empty[String, Int] + service + .pagedTraversalIds(db, 100, _.filterNot(_.or(_.alert, _.observable, _.`case`, _.caseTemplate, _.taxonomy))) { ids => + db.tryTransaction { implicit graph => + Try { + val orphans = service + .getByIds(ids: _*) + ._id + .toSeq + if (orphans.nonEmpty) { + service.getByIds(orphans: _*).remove() + Map("orphan" -> orphans.size) + } else Map.empty[String, Int] + } + }.getOrElse(Map("globalFailure" -> 1)) } - }.getOrElse(Map("globalFailure" -> 1)) + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } diff --git a/thehive/app/org/thp/thehive/services/TaskSrv.scala b/thehive/app/org/thp/thehive/services/TaskSrv.scala index 9214a0d548..37bb50ed2d 100644 --- a/thehive/app/org/thp/thehive/services/TaskSrv.scala +++ b/thehive/app/org/thp/thehive/services/TaskSrv.scala @@ -258,39 +258,44 @@ class TaskIntegrityCheckOps @Inject() (val db: Database, val service: TaskSrv, o override def resolve(entities: Seq[Task with Entity])(implicit graph: Graph): Try[Unit] = Success(()) override def globalCheck(): Map[String, Int] = - db.tryTransaction { implicit graph => - Try { - service - .startTraversal - .project( - _.by - .by(_.unionFlat(_.`case`._id, _.caseTemplate._id).fold) - .by(_.unionFlat(_.organisations._id, _.caseTemplate.organisation._id).fold) + service + .pagedTraversalIds(db, 100) { ids => + db.tryTransaction { implicit graph => + val orgCheck = multiIdLink[Organisation]("organisationIds", organisationSrv)(_.remove) + val removeOrphan: OrphanStrategy[Task, EntityId] = { (_, entity) => + service.get(entity).remove() + Map("Task-relatedId-removeOrphan" -> 1) + } + val relatedCheck = new SingleLinkChecker[Product, EntityId, EntityId]( + orphanStrategy = removeOrphan, + setField = (entity, link) => UMapping.entityId.setProperty(service.get(entity), "relatedId", link._id).iterate(), + entitySelector = _ => EntitySelector.firstCreatedEntity, + removeLink = (_, _) => (), + getLink = id => graph.VV(id).entity.head, + Some(_) ) - .toIterator - .map { - case (task, relatedIds, organisationIds) => - val orgStats = multiIdLink[Organisation]("organisationIds", organisationSrv)(_.remove) - .check(task, task.organisationIds, organisationIds) - - val removeOrphan: OrphanStrategy[Task, EntityId] = { (_, entity) => - service.get(entity).remove() - Map("Task-relatedId-removeOrphan" -> 1) - } - val relatedStats = new SingleLinkChecker[Product, EntityId, EntityId]( - orphanStrategy = removeOrphan, - setField = (entity, link) => UMapping.entityId.setProperty(service.get(entity), "relatedId", link._id).iterate(), - entitySelector = _ => EntitySelector.firstCreatedEntity, - removeLink = (_, _) => (), - getLink = id => graph.VV(id).entity.head, - Some(_) - ).check(task, task.relatedId, relatedIds) - - orgStats <+> relatedStats + Try { + service + .getByIds(ids: _*) + .project( + _.by + .by(_.unionFlat(_.`case`._id, _.caseTemplate._id).fold) + .by(_.unionFlat(_.organisations._id, _.caseTemplate.organisation._id).fold) + ) + .toIterator + .map { + case (task, relatedIds, organisationIds) => + val orgStats = orgCheck.check(task, task.organisationIds, organisationIds) + val relatedStats = relatedCheck.check(task, task.relatedId, relatedIds) + + orgStats <+> relatedStats + } + .reduceOption(_ <+> _) + .getOrElse(Map.empty) } - .reduceOption(_ <+> _) - .getOrElse(Map.empty) + }.getOrElse(Map("globalFailure" -> 1)) } - }.getOrElse(Map("globalFailure" -> 1)) + .reduceOption(_ <+> _) + .getOrElse(Map.empty) }