#2261 Use limited count in page query

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala

@@ -85,10 +85,11 @@ class PublicAction @Inject() (actionSrv: ActionSrv, organisationSrv: Organisatio
     "getAction",
     (idOrName, graph, authContext) => actionSrv.get(idOrName)(graph).visible(organisationSrv)(authContext)
   )
-  override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Action], IteratorOutput](
-    "page",
-    (range, actionSteps, _) => actionSteps.richPage(range.from, range.to, withTotal = true)(_.richAction)
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Action], IteratorOutput](
+      "page",
+      (range, actionSteps, _) => actionSteps.richPage(range.from, range.to, withTotal = true, limitedCountThreshold)(_.richAction)
+    )
   override val outputQuery: Query = Query.output[RichAction, Traversal.V[Action]](_.richAction)
   val actionsQuery: Query = new Query {
     override val name: String = "actions"

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/AnalyzerTemplateCtrl.scala

@@ -104,10 +104,10 @@ class PublicAnalyzerTemplate @Inject() (analyzerTemplateSrv: AnalyzerTemplateSrv
     "getReportTemplate",
     (idOrName, graph, _) => analyzerTemplateSrv.get(idOrName)(graph)
   )
-  override val pageQuery: ParamQuery[OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[AnalyzerTemplate], IteratorOutput](
       "page",
-      (range, analyzerTemplateTraversal, _) => analyzerTemplateTraversal.page(range.from, range.to, withTotal = true)
+      (range, analyzerTemplateTraversal, _) => analyzerTemplateTraversal.page(range.from, range.to, withTotal = true, limitedCountThreshold)
     )
   override val outputQuery: Query = Query.output[AnalyzerTemplate with Entity]
   override val publicProperties: PublicProperties = PublicPropertyListBuilder[AnalyzerTemplate]

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala

@@ -37,7 +37,7 @@ class CortexQueryExecutor @Inject() (
   override lazy val queries: Seq[ParamQuery[_]] =
     controllers.map(_.initialQuery) :::
       controllers.map(_.getQuery) :::
-      controllers.map(_.pageQuery) :::
+      controllers.map(_.pageQuery(limitedCountThreshold)) ::: // FIXME the value of limitedCountThreshold is read only once. The value is not updated.
       controllers.map(_.outputQuery) :::
       controllers.flatMap(_.extraQueries)
 

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala

@@ -83,14 +83,16 @@ class PublicJob @Inject() (jobSrv: JobSrv) extends PublicData with JobRenderer {
     "getJob",
     (idOrName, graph, authContext) => jobSrv.get(idOrName)(graph).visible(authContext)
   )
-  override val pageQuery: ParamQuery[OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[Job], IteratorOutput](
       "page",
       {
         case (OutputParam(from, to, _, withParents), jobSteps, authContext) if withParents > 0 =>
-          jobSteps.richPage(from, to, withTotal = true)(_.richJobWithCustomRenderer(jobParents(_)(authContext))(authContext))
+          jobSteps.richPage(from, to, withTotal = true, limitedCountThreshold)(_.richJobWithCustomRenderer(jobParents(_)(authContext))(authContext))
         case (range, jobSteps, authContext) =>
-          jobSteps.richPage(range.from, range.to, withTotal = true)(_.richJob(authContext).domainMap((_, None: Option[(RichObservable, RichCase)])))
+          jobSteps.richPage(range.from, range.to, withTotal = true, limitedCountThreshold)(
+            _.richJob(authContext).domainMap((_, None: Option[(RichObservable, RichCase)]))
+          )
       }
     )
   override val outputQuery: Query = Query.outputWithContext[RichJob, Traversal.V[Job]]((jobSteps, authContext) => jobSteps.richJob(authContext))

frontend/app/views/partials/search/list.html

@@ -1,6 +1,6 @@
 <div class="box search-list">
     <!-- <div class="box-header">
-        <h3 class="box-title">Search result ({{searchResults.total}} records(s) found)</h3>
+        <h3 class="box-title">Search result ({{searchResults.total | limitedCount}} records(s) found)</h3>
     </div> -->
     <div class="m-xs text-primary">
         <h3>Search scope</h3>
@@ -80,7 +80,7 @@ <h3>Search filters <small ng-show="config.entity !== 'all'">{{config[config.enti
         </form>
     </div>
     <div class="m-xs text-primary" ng-if="searchResults">
-        <h3>Search Result <small>{{searchResults.total}} records(s) found</small></h3>
+        <h3>Search Result <small>{{searchResults.total | limitedCount}} records(s) found</small></h3>
     </div>
     <div class="box-body">
         <div class="row">

thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala

@@ -381,12 +381,12 @@ class PublicAlert @Inject() (
     "getAlert",
     (idOrName, graph, authContext) => alertSrv.get(idOrName)(graph).visible(organisationSrv)(authContext)
   )
-  override val pageQuery: ParamQuery[OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[Alert], IteratorOutput](
       "page",
       (range, alertSteps, _) =>
         alertSteps
-          .richPage(range.from, range.to, withTotal = true) { alerts =>
+          .richPage(range.from, range.to, withTotal = true, limitedCountThreshold) { alerts =>
             alerts.project(_.by(_.richAlert).by(_.observables.richObservable.fold))
           }
     )

thehive/app/org/thp/thehive/controllers/v0/AuditCtrl.scala

@@ -75,10 +75,10 @@ class PublicAudit @Inject() (auditSrv: AuditSrv, organisationSrv: OrganisationSr
   override val initialQuery: Query =
     Query.init[Traversal.V[Audit]]("listAudit", (graph, authContext) => auditSrv.startTraversal(graph).visible(organisationSrv)(authContext))
 
-  override val pageQuery: ParamQuery[org.thp.thehive.controllers.v0.OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[org.thp.thehive.controllers.v0.OutputParam] =
     Query.withParam[OutputParam, Traversal.V[Audit], IteratorOutput](
       "page",
-      (range, auditSteps, _) => auditSteps.richPage(range.from, range.to, withTotal = true)(_.richAudit)
+      (range, auditSteps, _) => auditSteps.richPage(range.from, range.to, withTotal = true, limitedCountThreshold)(_.richAudit)
     )
   override val outputQuery: Query = Query.output[RichAudit, Traversal.V[Audit]](_.richAudit)
 

thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala

@@ -195,13 +195,13 @@ class PublicCase @Inject() (
       "getCase",
       (idOrName, graph, authContext) => caseSrv.get(idOrName)(graph).visible(organisationSrv)(authContext)
     )
-  override val pageQuery: ParamQuery[OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[Case], IteratorOutput](
       "page",
       {
         case (OutputParam(from, to, withStats, _), caseSteps, authContext) =>
           caseSteps
-            .richPage(from, to, withTotal = true) {
+            .richPage(from, to, withTotal = true, limitedCountThreshold) {
               case c if withStats =>
                 c.richCaseWithCustomRenderer(caseStatsRenderer(authContext))(authContext)
               case c =>

thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala

@@ -104,10 +104,11 @@ class PublicCaseTemplate @Inject() (
     "getCaseTemplate",
     (idOrName, graph, authContext) => caseTemplateSrv.get(idOrName)(graph).visible(authContext)
   )
-  override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[CaseTemplate], IteratorOutput](
-    "page",
-    (range, caseTemplateSteps, _) => caseTemplateSteps.richPage(range.from, range.to, withTotal = true)(_.richCaseTemplate)
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[CaseTemplate], IteratorOutput](
+      "page",
+      (range, caseTemplateSteps, _) => caseTemplateSteps.richPage(range.from, range.to, withTotal = true, limitedCountThreshold)(_.richCaseTemplate)
+    )
   override val outputQuery: Query = Query.output[RichCaseTemplate, Traversal.V[CaseTemplate]](_.richCaseTemplate)
   override val extraQueries: Seq[ParamQuery[_]] = Seq(
     Query[Traversal.V[CaseTemplate], Traversal.V[Task]]("tasks", (caseTemplateSteps, _) => caseTemplateSteps.tasks)

thehive/app/org/thp/thehive/controllers/v0/CustomFieldCtrl.scala

@@ -90,13 +90,14 @@ class CustomFieldCtrl @Inject() (
 class PublicCustomField @Inject() (customFieldSrv: CustomFieldSrv) extends PublicData {
   override val entityName: String  = "CustomField"
   override val initialQuery: Query = Query.init[Traversal.V[CustomField]]("listCustomField", (graph, _) => customFieldSrv.startTraversal(graph))
-  override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[CustomField], IteratorOutput](
-    "page",
-    {
-      case (OutputParam(from, to, _, _), customFieldSteps, _) =>
-        customFieldSteps.page(from, to, withTotal = true)
-    }
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[CustomField], IteratorOutput](
+      "page",
+      {
+        case (OutputParam(from, to, _, _), customFieldSteps, _) =>
+          customFieldSteps.page(from, to, withTotal = true, limitedCountThreshold)
+      }
+    )
   override val outputQuery: Query = Query.output[CustomField with Entity]
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[CustomField]](
     "getCustomField",

thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala

@@ -105,10 +105,12 @@ class PublicDashboard @Inject() (
     (idOrName, graph, authContext) => dashboardSrv.get(idOrName)(graph).visible(authContext)
   )
 
-  val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Dashboard], IteratorOutput](
-    "page",
-    (range, dashboardSteps, authContext) => dashboardSteps.richPage(range.from, range.to, withTotal = true)(_.richDashboard(authContext))
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Dashboard], IteratorOutput](
+      "page",
+      (range, dashboardSteps, authContext) =>
+        dashboardSteps.richPage(range.from, range.to, withTotal = true, limitedCountThreshold)(_.richDashboard(authContext))
+    )
   override val outputQuery: Query = Query.outputWithContext[RichDashboard, Traversal.V[Dashboard]](_.richDashboard(_))
   val publicProperties: PublicProperties = PublicPropertyListBuilder[Dashboard]
     .property("title", UMapping.string)(_.field.updatable)

thehive/app/org/thp/thehive/controllers/v0/LogCtrl.scala

@@ -81,21 +81,22 @@ class PublicLog @Inject() (logSrv: LogSrv, organisationSrv: OrganisationSrv) ext
     "getLog",
     (idOrName, graph, authContext) => logSrv.get(idOrName)(graph).visible(organisationSrv)(authContext)
   )
-  override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Log], IteratorOutput](
-    "page",
-    {
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Log], IteratorOutput](
+      "page",
+      {
 
-      case (OutputParam(from, to, _, 0), logSteps, _) => logSteps.richPage(from, to, withTotal = true)(_.richLog)
-      case (OutputParam(from, to, _, _), logSteps, authContext) =>
-        logSteps.richPage(from, to, withTotal = true)(
-          _.richLogWithCustomRenderer(
-            _.task.richTaskWithCustomRenderer(
-              _.`case`.richCase(authContext).option
+        case (OutputParam(from, to, _, 0), logSteps, _) => logSteps.richPage(from, to, withTotal = true, limitedCountThreshold)(_.richLog)
+        case (OutputParam(from, to, _, _), logSteps, authContext) =>
+          logSteps.richPage(from, to, withTotal = true, limitedCountThreshold)(
+            _.richLogWithCustomRenderer(
+              _.task.richTaskWithCustomRenderer(
+                _.`case`.richCase(authContext).option
+              )
             )
           )
-        )
-    }
-  )
+      }
+    )
 
   override val outputQuery: Query = Query.output[RichLog, Traversal.V[Log]](_.richLog)
   override val publicProperties: PublicProperties =

thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala

@@ -372,21 +372,21 @@ class PublicObservable @Inject() (
     "getObservable",
     (idOrName, graph, authContext) => observableSrv.get(idOrName)(graph).visible(organisationSrv)(authContext)
   )
-  override val pageQuery: ParamQuery[OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[Observable], IteratorOutput](
       "page",
       {
         case (OutputParam(from, to, withStats, 0), observableSteps, authContext) =>
           observableSteps
-            .richPage(from, to, withTotal = true) {
+            .richPage(from, to, withTotal = true, limitedCountThreshold) {
               case o if withStats =>
                 o.richObservableWithCustomRenderer(organisationSrv, observableStatsRenderer(organisationSrv)(authContext))(authContext)
                   .domainMap(ros => (ros._1, ros._2, None: Option[Either[RichCase, RichAlert]]))
               case o =>
                 o.richObservable.domainMap(ro => (ro, JsObject.empty, None))
             }
         case (OutputParam(from, to, _, _), observableSteps, authContext) =>
-          observableSteps.richPage(from, to, withTotal = true)(
+          observableSteps.richPage(from, to, withTotal = true, limitedCountThreshold)(
             _.richObservableWithCustomRenderer(
               organisationSrv,
               o => o.project(_.by(_.`case`.richCase(authContext).option).by(_.alert.richAlert.option))

thehive/app/org/thp/thehive/controllers/v0/ObservableTypeCtrl.scala

@@ -52,10 +52,10 @@ class PublicObservableType @Inject() (observableTypeSrv: ObservableTypeSrv) exte
   override val entityName: String = "ObservableType"
   override val initialQuery: Query =
     Query.init[Traversal.V[ObservableType]]("listObservableType", (graph, _) => observableTypeSrv.startTraversal(graph))
-  override val pageQuery: ParamQuery[OutputParam] =
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
     Query.withParam[OutputParam, Traversal.V[ObservableType], IteratorOutput](
       "page",
-      (range, observableTypeSteps, _) => observableTypeSteps.richPage(range.from, range.to, withTotal = true)(identity)
+      (range, observableTypeSteps, _) => observableTypeSteps.richPage(range.from, range.to, withTotal = true, limitedCountThreshold)(identity)
     )
   override val outputQuery: Query = Query.output[ObservableType with Entity]
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[ObservableType]](

thehive/app/org/thp/thehive/controllers/v0/OrganisationCtrl.scala

@@ -131,10 +131,11 @@ class PublicOrganisation @Inject() (organisationSrv: OrganisationSrv) extends Pu
 
   override val initialQuery: Query =
     Query.init[Traversal.V[Organisation]]("listOrganisation", (graph, authContext) => organisationSrv.startTraversal(graph).visible(authContext))
-  override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Organisation], IteratorOutput](
-    "page",
-    (range, organisationSteps, _) => organisationSteps.page(range.from, range.to, withTotal = true)
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Organisation], IteratorOutput](
+      "page",
+      (range, organisationSteps, _) => organisationSteps.page(range.from, range.to, withTotal = true, limitedCountThreshold)
+    )
   override val outputQuery: Query = Query.output[Organisation with Entity]
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Organisation]](
     "getOrganisation",

thehive/app/org/thp/thehive/controllers/v0/PageCtrl.scala

@@ -76,10 +76,11 @@ class PublicPage @Inject() (pageSrv: PageSrv, organisationSrv: OrganisationSrv)
     "getPage",
     (idOrName, graph, authContext) => pageSrv.get(idOrName)(graph).visible(authContext)
   )
-  val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Page], IteratorOutput](
-    "page",
-    (range, pageSteps, _) => pageSteps.page(range.from, range.to, withTotal = true)
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Page], IteratorOutput](
+      "page",
+      (range, pageSteps, _) => pageSteps.page(range.from, range.to, withTotal = true, limitedCountThreshold)
+    )
   override val outputQuery: Query = Query.output[Page with Entity]
   override val publicProperties: PublicProperties = PublicPropertyListBuilder[Page]
     .property("title", UMapping.string)(_.field.updatable)

thehive/app/org/thp/thehive/controllers/v0/ProfileCtrl.scala

@@ -80,10 +80,11 @@ class PublicProfile @Inject() (profileSrv: ProfileSrv) extends PublicData {
   val initialQuery: Query =
     Query.init[Traversal.V[Profile]]("listProfile", (graph, _) => profileSrv.startTraversal(graph))
 
-  val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Profile], IteratorOutput](
-    "page",
-    (range, profileSteps, _) => profileSteps.page(range.from, range.to, withTotal = true)
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Profile], IteratorOutput](
+      "page",
+      (range, profileSteps, _) => profileSteps.page(range.from, range.to, withTotal = true, limitedCountThreshold)
+    )
   override val outputQuery: Query = Query.output[Profile with Entity]
   val publicProperties: PublicProperties = PublicPropertyListBuilder[Profile]
     .property("name", UMapping.string)(_.field.updatable)

thehive/app/org/thp/thehive/controllers/v0/QueryCtrl.scala

@@ -21,7 +21,7 @@ trait PublicData {
   val entityName: String
   val publicProperties: PublicProperties
   val initialQuery: Query
-  val pageQuery: ParamQuery[OutputParam]
+  def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam]
   val outputQuery: Query
   val getQuery: ParamQuery[EntityIdOrName]
   val extraQueries: Seq[ParamQuery[_]] = Nil
@@ -105,7 +105,7 @@ trait QueryCtrl {
           inputSort <- sortParser(field.get("sort"))
           sortedQuery = filteredQuery andThen new SortQuery(queryExecutor.publicProperties).toQuery(inputSort)
           outputParam <- outputParamParser.optional(field).map(_.getOrElse(OutputParam(0, 10, withStats = false, withParents = 0)))
-          outputQuery = publicData.pageQuery.toQuery(outputParam)
+          outputQuery = publicData.pageQuery(queryExecutor.limitedCountThreshold).toQuery(outputParam)
         } yield sortedQuery andThen outputQuery
     }
 

thehive/app/org/thp/thehive/controllers/v0/TagCtrl.scala

@@ -43,10 +43,11 @@ class PublicTag @Inject() (tagSrv: TagSrv, organisationSrv: OrganisationSrv) ext
   override val entityName: String = "tag"
   override val initialQuery: Query =
     Query.init[Traversal.V[Tag]]("listTag", (graph, authContext) => tagSrv.startTraversal(graph).visible(authContext))
-  override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Tag], IteratorOutput](
-    "page",
-    (range, tagSteps, _) => tagSteps.page(range.from, range.to, withTotal = true)
-  )
+  override def pageQuery(limitedCountThreshold: Long): ParamQuery[OutputParam] =
+    Query.withParam[OutputParam, Traversal.V[Tag], IteratorOutput](
+      "page",
+      (range, tagSteps, _) => tagSteps.page(range.from, range.to, withTotal = true, limitedCountThreshold)
+    )
   override val outputQuery: Query = Query.output[Tag with Entity]
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Tag]](
     "getTag",