From 14eb88109fe6abd0409e21c7b9c40bfbec9b244e Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 1 Apr 2021 10:19:58 +0200 Subject: [PATCH 01/63] #1921 Fix customField selection query --- ScalliGraph | 2 +- .../thp/thehive/controllers/v0/CaseCtrl.scala | 4 +- .../controllers/v0/CaseTemplateCtrl.scala | 42 ++++---- .../thehive/controllers/v1/Properties.scala | 44 ++++++--- .../org/thp/thehive/models/CustomField.scala | 17 ++++ .../org/thp/thehive/services/AlertSrv.scala | 8 ++ .../org/thp/thehive/services/CaseSrv.scala | 8 ++ .../thehive/services/CaseTemplateSrv.scala | 95 +++++++++++++++---- .../services/CaseTemplateSrvTest.scala | 5 +- 9 files changed, 174 insertions(+), 51 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 4aa2d6084e..e7cf9fc714 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 4aa2d6084eaca86a8dbc7e078a53a103f0d63c8c +Subproject commit e7cf9fc7141ab4d12b8c06b5db427fb009a4f19c diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala index 35baed7a5a..5dd56482af 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseCtrl.scala @@ -275,9 +275,7 @@ class PublicCase @Inject() ( }) .property("customFields", UMapping.jsonNative)(_.subSelect { case (FPathElem(_, FPathElem(idOrName, _)), caseSteps) => - caseSteps - .customFields(EntityIdOrName(idOrName)) - .jsonValue + caseSteps.customFieldJsonValue(customFieldSrv, EntityIdOrName(idOrName)) case (_, caseSteps) => caseSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter[JsValue] { diff --git a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala index a0c435e2d9..832b808f56 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/CaseTemplateCtrl.scala @@ -16,7 +16,7 @@ import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.UserOps._ import org.thp.thehive.services._ import play.api.Logger -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsObject, JsValue, Json} import play.api.mvc.{Action, AnyContent, Results} import javax.inject.{Inject, Named, Singleton} @@ -134,22 +134,32 @@ class PublicCaseTemplate @Inject() ( .property("summary", UMapping.string.optional)(_.field.updatable) .property("user", UMapping.string)(_.field.updatable) .property("customFields", UMapping.jsonNative)(_.subSelect { - case (FPathElem(_, FPathElem(name, _)), caseTemplateSteps) => caseTemplateSteps.customFields(name).jsonValue + case (FPathElem(_, FPathElem(name, _)), caseTemplateSteps) => caseTemplateSteps.customFieldJsonValue(customFieldSrv, EntityIdOrName(name)) case (_, caseTemplateSteps) => caseTemplateSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) - }.custom { - case (FPathElem(_, FPathElem(name, _)), value, vertex, graph, authContext) => - for { - c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") - _ <- caseTemplateSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) - } yield Json.obj(s"customFields.$name" -> value) - case (FPathElem(_, FPathEmpty), values: JsObject, vertex, graph, authContext) => - for { - c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") - cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(_ -> v) } - _ <- caseTemplateSrv.updateCustomField(c, cfv)(graph, authContext) - } yield Json.obj("customFields" -> values) - case _ => Failure(BadRequestError("Invalid custom fields format")) - }) + } + .filter[JsValue] { + case (FPathElem(_, FPathElem(name, _)), caseTemplateTraversal, _, predicate) => + predicate match { + case Right(predicate) => caseTemplateTraversal.customFieldFilter(customFieldSrv, EntityIdOrName(name), predicate) + case Left(true) => caseTemplateTraversal.hasCustomField(customFieldSrv, EntityIdOrName(name)) + case Left(false) => caseTemplateTraversal.hasNotCustomField(customFieldSrv, EntityIdOrName(name)) + } + case (_, caseTraversal, _, _) => caseTraversal.empty + } + .custom { + case (FPathElem(_, FPathElem(name, _)), value, vertex, graph, authContext) => + for { + c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") + _ <- caseTemplateSrv.setOrCreateCustomField(c, EntityIdOrName(name), Some(value), None)(graph, authContext) + } yield Json.obj(s"customField.$name" -> value) + case (FPathElem(_, FPathEmpty), values: JsObject, vertex, graph, authContext) => + for { + c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") + cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(cf => (cf, v, None)) } + _ <- caseTemplateSrv.updateCustomField(c, cfv)(graph, authContext) + } yield Json.obj("customFields" -> values) + case _ => Failure(BadRequestError("Invalid custom fields format")) + }) .property("tasks", UMapping.jsonNative.sequence)( _.select(_.tasks.richTaskWithoutActionRequired.domainMap(_.toJson)).custom { // FIXME select the correct mapping (_, value, vertex, graph, authContext) => diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 819844c133..5a82f5ad7a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -106,8 +106,7 @@ class Properties @Inject() ( .property("customFields", UMapping.jsonNative)(_.subSelect { case (FPathElem(_, FPathElem(idOrName, _)), alerts) => alerts - .customFields(EntityIdOrName(idOrName)) - .jsonValue + .customFieldJsonValue(customFieldSrv, EntityIdOrName(idOrName)) case (_, alerts) => alerts.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter[JsValue] { @@ -209,8 +208,7 @@ class Properties @Inject() ( .property("customFields", UMapping.jsonNative)(_.subSelect { case (FPathElem(_, FPathElem(idOrName, _)), caseSteps) => caseSteps - .customFields(EntityIdOrName(idOrName)) - .jsonValue + .customFieldJsonValue(customFieldSrv, EntityIdOrName(idOrName)) case (_, caseSteps) => caseSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter[JsValue] { @@ -273,16 +271,34 @@ class Properties @Inject() ( .property("summary", UMapping.string.optional)(_.field.updatable) .property("user", UMapping.string)(_.field.updatable) .property("customFields", UMapping.jsonNative)(_.subSelect { - case (FPathElem(_, FPathElem(name, _)), alertSteps) => alertSteps.customFields(name).jsonValue - case (_, alertSteps) => alertSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) - }.custom { - case (FPathElem(_, FPathElem(name, _)), value, vertex, graph, authContext) => - for { - c <- caseTemplateSrv.getOrFail(vertex)(graph) - _ <- caseTemplateSrv.setOrCreateCustomField(c, name, Some(value), None)(graph, authContext) - } yield Json.obj(s"customField.$name" -> value) - case _ => Failure(BadRequestError("Invalid custom fields format")) - }) + case (FPathElem(_, FPathElem(idOrName, _)), caseTemplateSteps) => + caseTemplateSteps + .customFieldJsonValue(customFieldSrv, EntityIdOrName(idOrName)) + case (_, caseTemplateSteps) => caseTemplateSteps.customFields.nameJsonValue.fold.domainMap(JsObject(_)) + } + .filter[JsValue] { + case (FPathElem(_, FPathElem(name, _)), caseTemplateTraversal, _, predicate) => + predicate match { + case Right(predicate) => caseTemplateTraversal.customFieldFilter(customFieldSrv, EntityIdOrName(name), predicate) + case Left(true) => caseTemplateTraversal.hasCustomField(customFieldSrv, EntityIdOrName(name)) + case Left(false) => caseTemplateTraversal.hasNotCustomField(customFieldSrv, EntityIdOrName(name)) + } + case (_, caseTraversal, _, _) => caseTraversal.empty + } + .custom { + case (FPathElem(_, FPathElem(idOrName, _)), value, vertex, graph, authContext) => + for { + c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") + _ <- caseTemplateSrv.setOrCreateCustomField(c, EntityIdOrName(idOrName), Some(value), None)(graph, authContext) + } yield Json.obj(s"customField.$idOrName" -> value) + case (FPathElem(_, FPathEmpty), values: JsObject, vertex, graph, authContext) => + for { + c <- caseTemplateSrv.get(vertex)(graph).getOrFail("CaseTemplate") + cfv <- values.fields.toTry { case (n, v) => customFieldSrv.getOrFail(EntityIdOrName(n))(graph).map(cf => (cf, v, None)) } + _ <- caseTemplateSrv.updateCustomField(c, cfv)(graph, authContext) + } yield Json.obj("customFields" -> values) + case _ => Failure(BadRequestError("Invalid custom fields format")) + }) .build lazy val organisation: PublicProperties = diff --git a/thehive/app/org/thp/thehive/models/CustomField.scala b/thehive/app/org/thp/thehive/models/CustomField.scala index fb4edb5241..e30b8710c6 100644 --- a/thehive/app/org/thp/thehive/models/CustomField.scala +++ b/thehive/app/org/thp/thehive/models/CustomField.scala @@ -3,6 +3,9 @@ package org.thp.thehive.models import org.apache.tinkerpop.gremlin.structure.Edge import org.thp.scalligraph._ import org.thp.scalligraph.models._ +import org.thp.scalligraph.traversal.Traversal +import org.thp.scalligraph.traversal.Traversal.{Domain, E} +import org.thp.scalligraph.traversal.TraversalOps._ import play.api.libs.json._ import java.util.{Date, NoSuchElementException} @@ -91,6 +94,10 @@ sealed abstract class CustomFieldType[T] { def getJsonValue(ccf: CustomFieldValue[_]): JsValue = getValue(ccf).fold[JsValue](JsNull)(writes.writes) + def getValue[C <: CustomFieldValue[_]](traversal: Traversal.E[C]): Traversal.Domain[T] + + def getJsonValue[C <: CustomFieldValue[_]](traversal: Traversal.E[C]): Traversal.Domain[JsValue] = getValue(traversal).domainMap(writes.writes) + override def toString: String = name protected def setValueFailure(value: Any): Failure[Nothing] = @@ -117,6 +124,8 @@ object CustomFieldString extends CustomFieldType[String] { } override def getValue(ccf: CustomFieldValue[_]): Option[String] = ccf.stringValue + + override def getValue[C <: CustomFieldValue[_]](traversal: E[C]): Traversal.Domain[String] = traversal.value(_.stringValue).castDomain } object CustomFieldBoolean extends CustomFieldType[Boolean] { @@ -137,6 +146,8 @@ object CustomFieldBoolean extends CustomFieldType[Boolean] { } override def getValue(ccf: CustomFieldValue[_]): Option[Boolean] = ccf.booleanValue + + override def getValue[C <: CustomFieldValue[_]](traversal: E[C]): Domain[Boolean] = traversal.value(_.booleanValue).castDomain } object CustomFieldInteger extends CustomFieldType[Int] { @@ -158,6 +169,8 @@ object CustomFieldInteger extends CustomFieldType[Int] { } override def getValue(ccf: CustomFieldValue[_]): Option[Int] = ccf.integerValue + + override def getValue[C <: CustomFieldValue[_]](traversal: E[C]): Domain[Int] = traversal.value(_.integerValue).castDomain } object CustomFieldFloat extends CustomFieldType[Double] { @@ -178,6 +191,8 @@ object CustomFieldFloat extends CustomFieldType[Double] { } override def getValue(ccf: CustomFieldValue[_]): Option[Double] = ccf.floatValue + + override def getValue[C <: CustomFieldValue[_]](traversal: E[C]): Domain[Double] = traversal.value(_.floatValue).castDomain } object CustomFieldDate extends CustomFieldType[Date] { @@ -199,6 +214,8 @@ object CustomFieldDate extends CustomFieldType[Date] { } override def getValue(ccf: CustomFieldValue[_]): Option[Date] = ccf.dateValue + + override def getValue[C <: CustomFieldValue[_]](traversal: E[C]): Domain[Date] = traversal.value(_.dateValue).castDomain } @DefineIndex(IndexType.unique, "name") diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index c39e6387fd..eebc4a9d91 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -469,6 +469,14 @@ object AlertOps { def customFields: Traversal.E[AlertCustomField] = traversal.outE[AlertCustomField] + def customFieldJsonValue(customFieldSrv: CustomFieldSrv, customField: EntityIdOrName): Traversal.Domain[JsValue] = + customFieldSrv + .get(customField)(traversal.graph) + .value(_.`type`) + .headOption + .map(t => CustomFieldType.map(t).getJsonValue(traversal.customFields(customField))) + .getOrElse(traversal.empty.castDomain) + def richCustomFields: Traversal[RichCustomField, JMap[String, Any], Converter[RichCustomField, JMap[String, Any]]] = traversal .outE[AlertCustomField] diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 094978ef53..0d7b4d5cd1 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -494,6 +494,14 @@ object CaseOps { name => customFields.filter(_.inV.v[CustomField].has(_.name, name)) ) + def customFieldJsonValue(customFieldSrv: CustomFieldSrv, customField: EntityIdOrName): Traversal.Domain[JsValue] = + customFieldSrv + .get(customField)(traversal.graph) + .value(_.`type`) + .headOption + .map(t => CustomFieldType.map(t).getJsonValue(traversal.customFields(customField))) + .getOrElse(traversal.empty.castDomain) + def richCustomFields: Traversal[RichCustomField, JMap[String, Any], Converter[RichCustomField, JMap[String, Any]]] = customFields .project(_.by.by(_.inV.v[CustomField])) diff --git a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala index cda9c8d8e1..d5f208f698 100644 --- a/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala @@ -4,6 +4,7 @@ import akka.actor.ActorRef import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.scalligraph.auth.{AuthContext, Permission} import org.thp.scalligraph.models.{Database, Entity} +import org.thp.scalligraph.query.PredicateOps.PredicateOpsDefs import org.thp.scalligraph.query.PropertyUpdater import org.thp.scalligraph.services._ import org.thp.scalligraph.traversal.TraversalOps._ @@ -16,9 +17,9 @@ import org.thp.thehive.services.CustomFieldOps._ import org.thp.thehive.services.OrganisationOps._ import org.thp.thehive.services.TaskOps._ import org.thp.thehive.services.UserOps._ -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.{JsObject, JsValue, Json} -import java.util.{Map => JMap} +import java.util.{Date, Map => JMap} import javax.inject.{Inject, Named} import scala.util.{Failure, Success, Try} @@ -61,7 +62,9 @@ class CaseTemplateSrv @Inject() ( _ <- caseTemplateOrganisationSrv.create(CaseTemplateOrganisation(), createdCaseTemplate, organisation) createdTasks <- tasks.toTry(createTask(createdCaseTemplate, _)) _ <- caseTemplate.tags.toTry(tagSrv.getOrCreate(_).flatMap(t => caseTemplateTagSrv.create(CaseTemplateTag(), createdCaseTemplate, t))) - cfs <- customFields.zipWithIndex.toTry { case ((name, value), order) => createCustomField(createdCaseTemplate, name, value, Some(order + 1)) } + cfs <- customFields.zipWithIndex.toTry { + case ((name, value), order) => createCustomField(createdCaseTemplate, EntityIdOrName(name), value, Some(order + 1)) + } richCaseTemplate = RichCaseTemplate(createdCaseTemplate, organisation.name, createdTasks, cfs) _ <- auditSrv.caseTemplate.create(createdCaseTemplate, richCaseTemplate.toJson) } yield richCaseTemplate @@ -101,12 +104,12 @@ class CaseTemplateSrv @Inject() ( def addTags(caseTemplate: CaseTemplate with Entity, tags: Set[String])(implicit graph: Graph, authContext: AuthContext): Try[Unit] = updateTags(caseTemplate, tags ++ caseTemplate.tags).map(_ => ()) - def getCustomField(caseTemplate: CaseTemplate with Entity, customFieldName: String)(implicit graph: Graph): Option[RichCustomField] = - get(caseTemplate).customFields(customFieldName).richCustomField.headOption + def getCustomField(caseTemplate: CaseTemplate with Entity, customFieldIdOrName: EntityIdOrName)(implicit graph: Graph): Option[RichCustomField] = + get(caseTemplate).customFields(customFieldIdOrName).richCustomField.headOption def updateCustomField( caseTemplate: CaseTemplate with Entity, - customFieldValues: Seq[(CustomField, Any)] + customFieldValues: Seq[(CustomField, Any, Option[Int])] )(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { val customFieldNames = customFieldValues.map(_._1.name) get(caseTemplate) @@ -114,32 +117,32 @@ class CaseTemplateSrv @Inject() ( .richCustomField .toIterator .filterNot(rcf => customFieldNames.contains(rcf.name)) - .foreach(rcf => get(caseTemplate).customFields(rcf.name).remove()) + .foreach(rcf => get(caseTemplate).customFields(EntityName(rcf.name)).remove()) customFieldValues - .zipWithIndex - .toTry { case ((cf, v), o) => setOrCreateCustomField(caseTemplate, cf.name, Some(v), Some(o + 1)) } + .toTry { case (cf, v, o) => setOrCreateCustomField(caseTemplate, EntityName(cf.name), Some(v), o) } .map(_ => ()) } - def setOrCreateCustomField(caseTemplate: CaseTemplate with Entity, customFieldName: String, value: Option[Any], order: Option[Int])(implicit + def setOrCreateCustomField(caseTemplate: CaseTemplate with Entity, customFieldIdOrName: EntityIdOrName, value: Option[Any], order: Option[Int])( + implicit graph: Graph, authContext: AuthContext ): Try[Unit] = { - val cfv = get(caseTemplate).customFields(customFieldName) + val cfv = get(caseTemplate).customFields(customFieldIdOrName) if (cfv.clone().exists) cfv.setValue(value) else - createCustomField(caseTemplate, customFieldName, value, order).map(_ => ()) + createCustomField(caseTemplate, customFieldIdOrName, value, order).map(_ => ()) } def createCustomField( caseTemplate: CaseTemplate with Entity, - customFieldName: String, + customFieldIdOrName: EntityIdOrName, customFieldValue: Option[Any], order: Option[Int] )(implicit graph: Graph, authContext: AuthContext): Try[RichCustomField] = for { - cf <- customFieldSrv.getOrFail(EntityIdOrName(customFieldName)) + cf <- customFieldSrv.getOrFail(customFieldIdOrName) ccf <- CustomFieldType.map(cf.`type`).setValue(CaseTemplateCustomField(order = order), customFieldValue) ccfe <- caseTemplateCustomFieldSrv.create(ccf, caseTemplate, cf) } yield RichCustomField(cf, ccfe) @@ -197,11 +200,71 @@ object CaseTemplateOps { def tags: Traversal.V[Tag] = traversal.out[CaseTemplateTag].v[Tag] - def customFields(name: String): Traversal.E[CaseTemplateCustomField] = - traversal.outE[CaseTemplateCustomField].filter(_.inV.v[CustomField].has(_.name, name)) + def customFields(idOrName: EntityIdOrName): Traversal.E[CaseTemplateCustomField] = + idOrName + .fold( + id => customFields.filter(_.inV.getByIds(id)), + name => customFields.filter(_.inV.v[CustomField].has(_.name, name)) + ) def customFields: Traversal.E[CaseTemplateCustomField] = traversal.outE[CaseTemplateCustomField] + + def customFieldJsonValue(customFieldSrv: CustomFieldSrv, customField: EntityIdOrName): Traversal.Domain[JsValue] = + customFieldSrv + .get(customField)(traversal.graph) + .value(_.`type`) + .headOption + .map(t => CustomFieldType.map(t).getJsonValue(traversal.customFields(customField))) + .getOrElse(traversal.empty.castDomain) + + def customFieldFilter(customFieldSrv: CustomFieldSrv, customField: EntityIdOrName, predicate: P[JsValue]): Traversal.V[CaseTemplate] = + customFieldSrv + .get(customField)(traversal.graph) + .value(_.`type`) + .headOption + .map { + case CustomFieldType.boolean => traversal.filter(_.customFields(customField).has(_.booleanValue, predicate.mapValue(_.as[Boolean]))) + case CustomFieldType.date => traversal.filter(_.customFields(customField).has(_.dateValue, predicate.mapValue(_.as[Date]))) + case CustomFieldType.float => traversal.filter(_.customFields(customField).has(_.floatValue, predicate.mapValue(_.as[Double]))) + case CustomFieldType.integer => traversal.filter(_.customFields(customField).has(_.integerValue, predicate.mapValue(_.as[Int]))) + case CustomFieldType.string => traversal.filter(_.customFields(customField).has(_.stringValue, predicate.mapValue(_.as[String]))) + } + .getOrElse(traversal.empty) + + def hasCustomField(customFieldSrv: CustomFieldSrv, customField: EntityIdOrName): Traversal.V[CaseTemplate] = { + val cfFilter = (t: Traversal.V[CustomField]) => customField.fold(id => t.hasId(id), name => t.has(_.name, name)) + + customFieldSrv + .get(customField)(traversal.graph) + .value(_.`type`) + .headOption + .map { + case CustomFieldType.boolean => traversal.filter(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.booleanValue).inV.v[CustomField])) + case CustomFieldType.date => traversal.filter(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.dateValue).inV.v[CustomField])) + case CustomFieldType.float => traversal.filter(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.floatValue).inV.v[CustomField])) + case CustomFieldType.integer => traversal.filter(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.integerValue).inV.v[CustomField])) + case CustomFieldType.string => traversal.filter(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.stringValue).inV.v[CustomField])) + } + .getOrElse(traversal.empty) + } + + def hasNotCustomField(customFieldSrv: CustomFieldSrv, customField: EntityIdOrName): Traversal.V[CaseTemplate] = { + val cfFilter = (t: Traversal.V[CustomField]) => customField.fold(id => t.hasId(id), name => t.has(_.name, name)) + + customFieldSrv + .get(customField)(traversal.graph) + .value(_.`type`) + .headOption + .map { + case CustomFieldType.boolean => traversal.filterNot(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.booleanValue).inV.v[CustomField])) + case CustomFieldType.date => traversal.filterNot(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.dateValue).inV.v[CustomField])) + case CustomFieldType.float => traversal.filterNot(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.floatValue).inV.v[CustomField])) + case CustomFieldType.integer => traversal.filterNot(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.integerValue).inV.v[CustomField])) + case CustomFieldType.string => traversal.filterNot(t => cfFilter(t.outE[CaseTemplateCustomField].has(_.stringValue).inV.v[CustomField])) + } + .getOrElse(traversal.empty) + } } implicit class CaseTemplateCustomFieldsOpsDefs(traversal: Traversal.E[CaseTemplateCustomField]) extends CustomFieldValueOpsDefs(traversal) diff --git a/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala b/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala index 63ef590903..a744266871 100644 --- a/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala +++ b/thehive/test/org/thp/thehive/services/CaseTemplateSrvTest.scala @@ -128,7 +128,10 @@ class CaseTemplateSrvTest extends PlaySpecification with TestAppBuilder { bool1 <- app[CustomFieldSrv].getOrFail(EntityName("boolean1")) integer1 <- app[CustomFieldSrv].getOrFail(EntityName("integer1")) caseTemplate <- app[CaseTemplateSrv].getOrFail(EntityName("spam")) - _ <- app[CaseTemplateSrv].updateCustomField(caseTemplate, Seq((string1, JsString("hate")), (bool1, JsTrue), (integer1, JsNumber(1)))) + _ <- app[CaseTemplateSrv].updateCustomField( + caseTemplate, + Seq((string1, JsString("hate"), None), (bool1, JsTrue, None), (integer1, JsNumber(1), None)) + ) } yield () } must beSuccessfulTry From 59a7b1b29444b4807603be1079bedc85e580fd3f Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 1 Apr 2021 17:19:10 +0200 Subject: [PATCH 02/63] #1923 Make the property "Imported" in alerts optimised for index --- .../org/thp/thehive/controllers/v0/AlertCtrl.scala | 13 ++++++++++++- .../org/thp/thehive/controllers/v1/Properties.scala | 1 - 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index da88cd00dd..777fe6bd8f 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -497,7 +497,18 @@ class PublicAlert @Inject() ( case _ => Failure(BadRequestError("Invalid custom fields format")) }) .property("case", db.idMapping)(_.select(_.`case`._id).readonly) - .property("imported", UMapping.boolean)(_.select(_.imported).readonly) + .property("imported", UMapping.boolean)( + _.select(_.imported) + .filter[Boolean]((_, alertTraversal, _, predicate) => + predicate.fold( + b => if (b) alertTraversal else alertTraversal.empty, + p => + if (p.getValue) alertTraversal.has(_.caseId) + else alertTraversal.hasNot(_.caseId) + ) + ) + .readonly + ) .property("importDate", UMapping.date.optional)(_.select(_.importDate).readonly) .property("computed.handlingDuration", UMapping.long)(_.select(_.handlingDuration).readonly) .property("computed.handlingDurationInSeconds", UMapping.long)(_.select(_.handlingDuration.math("_ / 1000").domainMap(_.toLong)).readonly) diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index 5a82f5ad7a..05c37ca713 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -133,7 +133,6 @@ class Properties @Inject() ( case _ => Failure(BadRequestError("Invalid custom fields format")) }) .property("case", db.idMapping)(_.select(_.`case`._id).readonly) - .property("imported", UMapping.boolean)(_.select(_.imported).readonly) .property("importDate", UMapping.date.optional)(_.select(_.importDate).readonly) .property("computed.handlingDuration", UMapping.long)(_.select(_.handlingDuration).readonly) .property("computed.handlingDurationInSeconds", UMapping.long)(_.select(_.handlingDuration.math("_ / 1000").domainMap(_.toLong)).readonly) From f6acf12391507c76272f05d673960f8e4cdd56fa Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 1 Apr 2021 17:19:29 +0200 Subject: [PATCH 03/63] Remove useless warning messages --- conf/logback.xml | 1 + thehive/app/org/thp/thehive/controllers/v0/DescribeCtrl.scala | 1 + thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala | 1 + 3 files changed, 3 insertions(+) diff --git a/conf/logback.xml b/conf/logback.xml index 94caf258db..8d7af55449 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -38,6 +38,7 @@ + - + diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index f8edb65334..0bb548f6b2 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -17,6 +17,8 @@ import scala.collection.immutable import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} import scala.util.Success +import ch.qos.logback.classic.{Level, LoggerContext} +import org.slf4j.LoggerFactory @Singleton class AdminCtrl @Inject() ( @@ -40,6 +42,25 @@ class AdminCtrl @Inject() ( } lazy val logger: Logger = Logger(getClass) + def setLogLevel(packageName: String, levelName: String): Action[AnyContent] = + entrypoint("Update log level") + .authPermitted(Permissions.managePlatform) { _ => + val level = levelName match { + case "ALL" => Level.ALL + case "DEBUG" => Level.DEBUG + case "INFO" => Level.INFO + case "WARN" => Level.WARN + case "ERROR" => Level.ERROR + case "OFF" => Level.OFF + case "TRACE" => Level.TRACE + case _ => Level.INFO + } + val loggerContext = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext] + val logger = loggerContext.getLogger(packageName) + logger.setLevel(level) + Success(Results.NoContent) + } + def triggerCheck(name: String): Action[AnyContent] = entrypoint("Trigger check") .authPermitted(Permissions.managePlatform) { _ => diff --git a/thehive/app/org/thp/thehive/controllers/v1/Router.scala b/thehive/app/org/thp/thehive/controllers/v1/Router.scala index 652fbdd85d..e5b3ca9fa9 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Router.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Router.scala @@ -43,10 +43,11 @@ class Router @Inject() ( case GET(p"/status") => statusCtrl.get // GET /health controllers.StatusCtrl.health - case GET(p"/admin/check/stats") => adminCtrl.checkStats - case GET(p"/admin/check/$name/trigger") => adminCtrl.triggerCheck(name) - case GET(p"/admin/index/status") => adminCtrl.indexStatus - case GET(p"/admin/index/$name/reindex") => adminCtrl.reindex(name) + case GET(p"/admin/check/stats") => adminCtrl.checkStats + case GET(p"/admin/check/$name/trigger") => adminCtrl.triggerCheck(name) + case GET(p"/admin/index/status") => adminCtrl.indexStatus + case GET(p"/admin/index/$name/reindex") => adminCtrl.reindex(name) + case GET(p"/admin/log/set/$packageName/$level") => adminCtrl.setLogLevel(packageName, level) // GET /logout controllers.AuthenticationCtrl.logout() case GET(p"/logout") => authenticationCtrl.logout diff --git a/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala b/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala index 4e26cf9682..1fc88d07c3 100644 --- a/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala +++ b/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala @@ -125,9 +125,9 @@ class IntegrityCheckActor() extends Actor { Success(integrityCheck.initialCheck()) } } - integrityCheckOps.foreach { integrityCheck => - self ! DuplicationCheck(integrityCheck.name) - } +// integrityCheckOps.foreach { integrityCheck => +// self ! DuplicationCheck(integrityCheck.name) +// } globalTimers = integrityCheckOps.map { integrityCheck => val interval = globalInterval(integrityCheck.name) val initialDelay = FiniteDuration((interval.toNanos * Random.nextDouble()).round, NANOSECONDS) From 767026931f16959d60365c7f47ff559971b4924f Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 9 Apr 2021 06:27:18 +0200 Subject: [PATCH 16/63] #1948 Remove extra case and alert count in list pages --- .../controllers/alert/AlertListCtrl.js | 341 +++++++++--------- .../scripts/controllers/case/CaseListCtrl.js | 202 ++++------- frontend/app/views/partials/alert/list.html | 194 ++++++---- .../app/views/partials/case/case.list.html | 229 ++++++++---- .../app/views/partials/case/list/toolbar.html | 26 +- 5 files changed, 535 insertions(+), 457 deletions(-) diff --git a/frontend/app/scripts/controllers/alert/AlertListCtrl.js b/frontend/app/scripts/controllers/alert/AlertListCtrl.js index 13f80ce024..da5db5a052 100755 --- a/frontend/app/scripts/controllers/alert/AlertListCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertListCtrl.js @@ -1,8 +1,8 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveControllers') - .controller('AlertListCtrl', function($rootScope, $scope, $q, $state, $uibModal, TagSrv, StreamQuerySrv, CaseTemplateSrv, ModalUtilsSrv, AlertingSrv, NotificationSrv, FilteringSrv, CortexSrv, Severity, VersionSrv) { + .controller('AlertListCtrl', function ($rootScope, $scope, $q, $state, $uibModal, TagSrv, StreamQuerySrv, CaseTemplateSrv, ModalUtilsSrv, AlertingSrv, NotificationSrv, FilteringSrv, CortexSrv, Severity, VersionSrv) { var self = this; self.urls = VersionSrv.mispUrls(); @@ -21,7 +21,7 @@ self.lastSearch = null; self.responders = null; - this.$onInit = function() { + this.$onInit = function () { self.filtering = new FilteringSrv('alert', 'alert.list', { version: 'v1', defaults: { @@ -37,33 +37,16 @@ }] }); self.filtering.initContext('list') - .then(function() { + .then(function () { self.load(); $scope.$watch('$vm.list.pageSize', function (newValue) { self.filtering.setPageSize(newValue); }); }); - - StreamQuerySrv('v1', [ - {_name: 'listAlert'}, - {_name: 'count'} - ], { - scope: $scope, - rootId: 'any', - objectType: 'alert', - query: { - params: { - name: 'alert-count' - } - }, - onUpdate: function(data) { - self.alertListCount = data; - } - }); }; - self.load = function() { + self.load = function () { var config = { scope: $scope, filter: this.filtering.buildQuery(), @@ -90,22 +73,22 @@ this.canMarkAsRead = AlertingSrv.canMarkAsRead; this.canMarkAsUnread = AlertingSrv.canMarkAsUnread; - this.markAsRead = function(event) { + this.markAsRead = function (event) { var fn = angular.noop; - if(this.canMarkAsRead(event)) { + if (this.canMarkAsRead(event)) { fn = AlertingSrv.markAsRead; } else { fn = AlertingSrv.markAsUnread; } - fn(event._id).then(function( /*data*/ ) { - }, function(response) { + fn(event._id).then(function ( /*data*/) { + }, function (response) { NotificationSrv.error('AlertListCtrl', response.data, response.status); }); }; - self.follow = function(event) { + self.follow = function (event) { var fn = angular.noop; if (event.follow === true) { @@ -114,13 +97,13 @@ fn = AlertingSrv.follow; } - fn(event._id).then(function( /*data*/ ) { - }, function(response) { + fn(event._id).then(function ( /*data*/) { + }, function (response) { NotificationSrv.error('AlertListCtrl', response.data, response.status); }); }; - self.bulkFollow = function(follow) { + self.bulkFollow = function (follow) { var ids = _.pluck(self.selection, 'id'); var fn = angular.noop; @@ -130,59 +113,59 @@ fn = AlertingSrv.unfollow; } - var promises = _.map(ids, function(id) { + var promises = _.map(ids, function (id) { return fn(id); }); - $q.all(promises).then(function( /*response*/ ) { + $q.all(promises).then(function ( /*response*/) { NotificationSrv.log('The selected events have been ' + (follow ? 'followed' : 'unfollowed'), 'success'); - }, function(response) { + }, function (response) { NotificationSrv.error('AlertListCtrl', response.data, response.status); }); }; - self.bulkMarkAsRead = function(markAsReadFlag) { + self.bulkMarkAsRead = function (markAsReadFlag) { var ids = _.pluck(self.selection, '_id'); var fn = angular.noop; var markAsRead = markAsReadFlag && this.canMarkAsRead(self.selection[0]); - if(markAsRead) { + if (markAsRead) { fn = AlertingSrv.markAsRead; } else { fn = AlertingSrv.markAsUnread; } - var promises = _.map(ids, function(id) { + var promises = _.map(ids, function (id) { return fn(id); }); - $q.all(promises).then(function( /*response*/ ) { + $q.all(promises).then(function ( /*response*/) { self.list.update(); NotificationSrv.log('The selected events have been ' + (markAsRead ? 'marked as read' : 'marked as unread'), 'success'); - }, function(response) { + }, function (response) { NotificationSrv.error('AlertListCtrl', response.data, response.status); }); }; - self.bulkDelete = function() { + self.bulkDelete = function () { - ModalUtilsSrv.confirm('Remove Alerts', 'Are you sure you want to delete the selected Alerts?', { - okText: 'Yes, remove them', - flavor: 'danger' - }).then(function() { - var ids = _.pluck(self.selection, '_id'); + ModalUtilsSrv.confirm('Remove Alerts', 'Are you sure you want to delete the selected Alerts?', { + okText: 'Yes, remove them', + flavor: 'danger' + }).then(function () { + var ids = _.pluck(self.selection, '_id'); - AlertingSrv.bulkRemove(ids) - .then(function(/*response*/) { - NotificationSrv.log('The selected events have been deleted', 'success'); - }) - .catch(function(response) { - NotificationSrv.error('AlertListCtrl', response.data, response.status); - }); - }); + AlertingSrv.bulkRemove(ids) + .then(function (/*response*/) { + NotificationSrv.log('The selected events have been deleted', 'success'); + }) + .catch(function (response) { + NotificationSrv.error('AlertListCtrl', response.data, response.status); + }); + }); }; - self.import = function(event) { + self.import = function (event) { var modalInstance = $uibModal.open({ templateUrl: 'views/partials/alert/event.dialog.html', controller: 'AlertEventCtrl', @@ -190,21 +173,21 @@ size: 'max', resolve: { event: event, - templates: function() { + templates: function () { return CaseTemplateSrv.list(); }, readonly: false } }); - modalInstance.result.catch(function(err) { - if(err && !_.isString(err)) { + modalInstance.result.catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('AlertListCtrl', err.data, err.status); } }); }; - self.resetSelection = function() { + self.resetSelection = function () { if (self.menu.selectAll) { self.selectAll(); } else { @@ -214,39 +197,39 @@ } }; - this.getResponders = function(event, force) { - if(!force && this.responders !== null) { - return; + this.getResponders = function (event, force) { + if (!force && this.responders !== null) { + return; } this.responders = null; CortexSrv.getResponders('alert', event._id) - .then(function(responders) { - self.responders = responders; - return CortexSrv.promntForResponder(responders); - }) - .then(function(response) { - if(response && _.isString(response)) { - NotificationSrv.log(response, 'warning'); - } else { - return CortexSrv.runResponder(response.id, response.name, 'alert', _.pick(event, '_id', 'tlp')); - } - }) - .then(function(response){ - NotificationSrv.log(['Responder', response.data.responderName, 'started successfully on alert', event.title].join(' '), 'success'); - }) - .catch(function(err) { - if(err && !_.isString(err)) { - NotificationSrv.error('AlertList', err.data, err.status); - } - }); - }; - - self.cancel = function() { + .then(function (responders) { + self.responders = responders; + return CortexSrv.promntForResponder(responders); + }) + .then(function (response) { + if (response && _.isString(response)) { + NotificationSrv.log(response, 'warning'); + } else { + return CortexSrv.runResponder(response.id, response.name, 'alert', _.pick(event, '_id', 'tlp')); + } + }) + .then(function (response) { + NotificationSrv.log(['Responder', response.data.responderName, 'started successfully on alert', event.title].join(' '), 'success'); + }) + .catch(function (err) { + if (err && !_.isString(err)) { + NotificationSrv.error('AlertList', err.data, err.status); + } + }); + }; + + self.cancel = function () { self.modalInstance.close(); }; - self.updateMenu = function() { + self.updateMenu = function () { var temp = _.uniq(_.pluck(self.selection, 'follow')); self.menu.unfollow = temp.length === 1 && temp[0] === true; @@ -267,11 +250,11 @@ self.menu.delete = temp.length === 0; }; - self.select = function(event) { + self.select = function (event) { if (event.selected) { self.selection.push(event); } else { - self.selection = _.reject(self.selection, function(item) { + self.selection = _.reject(self.selection, function (item) { return item._id === event._id; }); } @@ -280,9 +263,9 @@ }; - self.selectAll = function() { + self.selectAll = function () { var selected = self.menu.selectAll; - _.each(self.list.values, function(item) { + _.each(self.list.values, function (item) { item.selected = selected; }); @@ -296,109 +279,109 @@ }; - self.createNewCase = function() { + self.createNewCase = function () { var alertIds = _.pluck(self.selection, '_id'); CaseTemplateSrv.list() - .then(function(templates) { - - if(!templates || templates.length === 0) { - return $q.resolve(undefined); - } - - // Open template selection dialog - var modal = $uibModal.open({ - templateUrl: 'views/partials/case/case.templates.selector.html', - controller: 'CaseTemplatesDialogCtrl', - controllerAs: 'dialog', - size: 'lg', - resolve: { - templates: function(){ - return templates; - }, - uiSettings: ['UiSettingsSrv', function(UiSettingsSrv) { - return UiSettingsSrv.all(); - }] - } - }); - - return modal.result; - }) - .then(function(template) { - - // Open case creation dialog - var modal = $uibModal.open({ - templateUrl: 'views/partials/case/case.creation.html', - controller: 'CaseCreationCtrl', - size: 'lg', - resolve: { - template: template - } - }); - - return modal.result; - }) - .then(function(createdCase) { - // Bulk merge the selected alerts into the created case - NotificationSrv.log('New case has been created', 'success'); - - return AlertingSrv.bulkMergeInto(alertIds, createdCase.id); - }) - .then(function(response) { - if(alertIds.length === 1) { - NotificationSrv.log(alertIds.length + ' Alert has been merged into the newly created case.', 'success'); - } else { - NotificationSrv.log(alertIds.length + ' Alert(s) have been merged into the newly created case.', 'success'); - } - - $rootScope.$broadcast('alert:event-imported'); - - $state.go('app.case.details', { - caseId: response.data.id - }); - }) - .catch(function(err) { - if(err && !_.isString(err)) { - NotificationSrv.error('AlertEventCtrl', err.data, err.status); - } - }); - - }; - - self.mergeInCase = function() { + .then(function (templates) { + + if (!templates || templates.length === 0) { + return $q.resolve(undefined); + } + + // Open template selection dialog + var modal = $uibModal.open({ + templateUrl: 'views/partials/case/case.templates.selector.html', + controller: 'CaseTemplatesDialogCtrl', + controllerAs: 'dialog', + size: 'lg', + resolve: { + templates: function () { + return templates; + }, + uiSettings: ['UiSettingsSrv', function (UiSettingsSrv) { + return UiSettingsSrv.all(); + }] + } + }); + + return modal.result; + }) + .then(function (template) { + + // Open case creation dialog + var modal = $uibModal.open({ + templateUrl: 'views/partials/case/case.creation.html', + controller: 'CaseCreationCtrl', + size: 'lg', + resolve: { + template: template + } + }); + + return modal.result; + }) + .then(function (createdCase) { + // Bulk merge the selected alerts into the created case + NotificationSrv.log('New case has been created', 'success'); + + return AlertingSrv.bulkMergeInto(alertIds, createdCase.id); + }) + .then(function (response) { + if (alertIds.length === 1) { + NotificationSrv.log(alertIds.length + ' Alert has been merged into the newly created case.', 'success'); + } else { + NotificationSrv.log(alertIds.length + ' Alert(s) have been merged into the newly created case.', 'success'); + } + + $rootScope.$broadcast('alert:event-imported'); + + $state.go('app.case.details', { + caseId: response.data.id + }); + }) + .catch(function (err) { + if (err && !_.isString(err)) { + NotificationSrv.error('AlertEventCtrl', err.data, err.status); + } + }); + + }; + + self.mergeInCase = function () { var caseModal = $uibModal.open({ templateUrl: 'views/partials/case/case.merge.html', controller: 'CaseMergeModalCtrl', controllerAs: 'dialog', size: 'lg', resolve: { - source: function() { + source: function () { return self.event; }, - title: function() { + title: function () { return 'Merge selected Alert(s)'; }, - prompt: function() { + prompt: function () { return 'the ' + self.selection.length + ' selected Alert(s)'; } } }); - caseModal.result.then(function(selectedCase) { + caseModal.result.then(function (selectedCase) { return AlertingSrv.bulkMergeInto(_.pluck(self.selection, '_id'), selectedCase._id); }) - .then(function(response) { - $rootScope.$broadcast('alert:event-imported'); + .then(function (response) { + $rootScope.$broadcast('alert:event-imported'); - $state.go('app.case.details', { - caseId: response.data.id + $state.go('app.case.details', { + caseId: response.data.id + }); + }) + .catch(function (err) { + if (err && !_.isString(err)) { + NotificationSrv.error('AlertEventCtrl', err.data, err.status); + } }); - }) - .catch(function(err) { - if(err && !_.isString(err)) { - NotificationSrv.error('AlertEventCtrl', err.data, err.status); - } - }); }; this.filter = function () { @@ -428,47 +411,47 @@ this.search(); }; - this.filterByStatus = function(flag) { + this.filterByStatus = function (flag) { self.filtering.clearFilters() - .then(function(){ + .then(function () { self.addFilterValue('imported', flag); }); }; - this.filterByNewAndUpdated = function() { + this.filterByNewAndUpdated = function () { self.filtering.clearFilters() - .then(function(){ + .then(function () { // TODO nadouani: how to support updated alerts self.addFilterValue('imported', true); }); }; - this.filterBySeverity = function(numericSev) { + this.filterBySeverity = function (numericSev) { self.addFilterValue('severity', Severity.values[numericSev]); }; - this.filterBy = function(field, value) { + this.filterBy = function (field, value) { self.filtering.clearFilters() - .then(function(){ + .then(function () { self.addFilterValue(field, value); }); }; - this.sortBy = function(sort) { + this.sortBy = function (sort) { self.list.sort = sort; self.list.update(); self.filtering.setSort(sort); }; - this.sortByField = function(field) { + this.sortByField = function (field) { var context = this.filtering.context; var currentSort = Array.isArray(context.sort) ? context.sort[0] : context.sort; var sort = null; - if(currentSort.substr(1) !== field) { + if (currentSort.substr(1) !== field) { sort = ['+' + field]; } else { - sort = [(currentSort === '+' + field) ? '-'+field : '+'+field]; + sort = [(currentSort === '+' + field) ? '-' + field : '+' + field]; } self.list.sort = sort; diff --git a/frontend/app/scripts/controllers/case/CaseListCtrl.js b/frontend/app/scripts/controllers/case/CaseListCtrl.js index 1f1cea1a8f..16f3bdbcb3 100644 --- a/frontend/app/scripts/controllers/case/CaseListCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseListCtrl.js @@ -1,4 +1,4 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveControllers') .controller('CaseListCtrl', CaseListCtrl) @@ -19,7 +19,7 @@ selectAll: false }; - this.$onInit = function() { + this.$onInit = function () { self.filtering = new FilteringSrv('case', 'case.list', { version: 'v1', defaults: { @@ -41,7 +41,7 @@ }); self.filtering.initContext('list') - .then(function() { + .then(function () { self.load(); $scope.$watch('$vm.list.pageSize', function (newValue) { @@ -49,55 +49,9 @@ }); }); - - // Case stats to build quick filter menu - StreamQuerySrv('v1', [ - { - _name: 'listCase' - }, - { - _name: 'aggregation', - _agg: 'field', - _field: 'status', - _select: [ - {_agg: 'count'} - ] - } - ], { - scope: $scope, - rootId: 'any', - objectType: 'case', - query: { - params: { - name: 'case-status-stats' - } - }, - onUpdate: function(updates) { - self.caseStats = updates; - } - }); - - // Case total - StreamQuerySrv('v1', [ - {_name: 'listCase'}, - {_name: 'count'} - ], { - scope: $scope, - rootId: 'any', - objectType: 'case', - query: { - params: { - name: 'case-count-stats' - } - }, - onUpdate: function(updates) { - self.caseCount = updates; - } - }); - }; - this.load = function() { + this.load = function () { this.list = new PaginatedQuerySrv({ name: 'cases', @@ -110,16 +64,16 @@ pageSize: self.filtering.context.pageSize, filter: this.filtering.buildQuery(), operations: [ - {'_name': 'listCase'} + { '_name': 'listCase' } ], extraData: ['observableStats', 'taskStats', 'procedureCount', 'isOwner', 'shareCount', 'permissions', 'actionRequired'], - onUpdate: function() { + onUpdate: function () { self.resetSelection(); } }); }; - self.resetSelection = function() { + self.resetSelection = function () { if (self.menu.selectAll) { self.selectAll(); } else { @@ -129,7 +83,7 @@ } }; - self.updateMenu = function() { + self.updateMenu = function () { // Handle flag/unflag menu items var temp = _.uniq(_.pluck(self.selection, 'flag')); self.menu.unflag = temp.length === 1 && temp[0] === true; @@ -143,28 +97,28 @@ self.menu.delete = self.selection.length > 0; }; - self.select = function(caze) { + self.select = function (caze) { if (caze.selected) { self.selection.push(caze); } else { - self.selection = _.reject(self.selection, function(item) { + self.selection = _.reject(self.selection, function (item) { return item._id === caze._id; }); } self.updateMenu(); }; - self.selectAll = function() { + self.selectAll = function () { var selected = self.menu.selectAll; - _.each(self.list.values, function(item) { - if(SecuritySrv.checkPermissions(['manageCase'], item.extraData.permissions)) { + _.each(self.list.values, function (item) { + if (SecuritySrv.checkPermissions(['manageCase'], item.extraData.permissions)) { item.selected = selected; } }); if (selected) { - self.selection = _.filter(self.list.values, function(item) { + self.selection = _.filter(self.list.values, function (item) { return !!item.selected; }); } else { @@ -209,9 +163,9 @@ this.search(); }; - this.filterMyCases = function() { + this.filterMyCases = function () { this.filtering.clearFilters() - .then(function() { + .then(function () { var currentUser = AuthenticationSrv.currentUser; self.filtering.addFilter({ field: 'assignee', @@ -227,9 +181,9 @@ }); }; - this.filterMyOrgCases = function() { + this.filterMyOrgCases = function () { this.filtering.clearFilters() - .then(function() { + .then(function () { var currentUser = AuthenticationSrv.currentUser; self.filtering.addFilter({ field: 'owningOrganisation', @@ -246,9 +200,9 @@ }); }; - this.filterSharedWithMyOrg = function() { + this.filterSharedWithMyOrg = function () { this.filtering.clearFilters() - .then(function() { + .then(function () { var currentUser = AuthenticationSrv.currentUser; self.filtering.addFilter({ field: 'owningOrganisation', @@ -265,9 +219,9 @@ }); }; - this.filterMyOpenCases = function() { + this.filterMyOpenCases = function () { this.filtering.clearFilters() - .then(function() { + .then(function () { var currentUser = AuthenticationSrv.currentUser; self.filtering.addFilter({ field: 'assignee', @@ -283,36 +237,36 @@ }); }; - this.filterByStatus = function(status) { + this.filterByStatus = function (status) { this.filtering.clearFilters() - .then(function() { + .then(function () { self.addFilterValue('status', status); }); }; - this.filterByResolutionStatus = function(status) { + this.filterByResolutionStatus = function (status) { this.filtering.clearFilters() - .then(function() { + .then(function () { self.filtering.addFilterValue('resolutionStatus', status); self.addFilterValue('status', 'Resolved'); }); }; - this.sortBy = function(sort) { + this.sortBy = function (sort) { this.list.sort = sort; this.list.update(); this.filtering.setSort(sort); }; - this.sortByField = function(field) { + this.sortByField = function (field) { var context = this.filtering.context; var currentSort = Array.isArray(context.sort) ? _.without(context.sort, '-flag', '+flag')[0] : context.sort; var sort = null; - if(currentSort.substr(1) !== field) { + if (currentSort.substr(1) !== field) { sort = ['-flag', '+' + field]; } else { - sort = ['-flag', (currentSort === '+' + field) ? '-'+field : '+'+field]; + sort = ['-flag', (currentSort === '+' + field) ? '-' + field : '+' + field]; } self.list.sort = sort; @@ -320,20 +274,20 @@ self.filtering.setSort(sort); }; - this.bulkFlag = function(flag) { + this.bulkFlag = function (flag) { var ids = _.pluck(self.selection, '_id'); - return CaseSrv.bulkUpdate(ids, {flag: flag}) - .then(function(/*responses*/) { + return CaseSrv.bulkUpdate(ids, { flag: flag }) + .then(function (/*responses*/) { NotificationSrv.log('Selected cases have been updated successfully', 'success'); }) - .catch(function(err) { + .catch(function (err) { NotificationSrv.error('Bulk flag cases', err.data, err.status); }); } - this.bulkEdit = function() { + this.bulkEdit = function () { var modal = $uibModal.open({ animation: 'true', templateUrl: 'views/partials/case/case.update.html', @@ -341,22 +295,22 @@ controllerAs: '$dialog', size: 'lg', resolve: { - selection: function() { + selection: function () { return self.selection; } } }); - modal.result.then(function(operations) { - $q.all(_.map(operations, function(operation) { + modal.result.then(function (operations) { + $q.all(_.map(operations, function (operation) { return CaseSrv.bulkUpdate(operation.ids, operation.patch); - })).then(function(/*responses*/) { + })).then(function (/*responses*/) { NotificationSrv.log('Selected cases have been updated successfully', 'success'); }); }); }; - this.bulkRemove = function() { + this.bulkRemove = function () { var modal = $uibModal.open({ animation: 'true', templateUrl: 'views/partials/case/case.bulk.delete.confirm.html', @@ -364,49 +318,49 @@ controllerAs: '$dialog', size: 'lg', resolve: { - selection: function() { + selection: function () { return self.selection; } } }); - modal.result.catch(function(err) { - if(err && !_.isString(err)) { + modal.result.catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Case Remove', err.data, err.status); } }) } - this.bulkReopen = function() { + this.bulkReopen = function () { return ModalUtilsSrv.confirm('Reopen cases', 'Are you sure you want to reopen the selected cases?', { okText: 'Yes, proceed' - }).then(function() { + }).then(function () { var ids = _.pluck(self.selection, '_id'); - return CaseSrv.bulkUpdate(ids, {status: 'Open'}) - .then(function(/*responses*/) { + return CaseSrv.bulkUpdate(ids, { status: 'Open' }) + .then(function (/*responses*/) { NotificationSrv.log('Selected cases have been reopened successfully', 'success'); }) - .catch(function(err) { + .catch(function (err) { NotificationSrv.error('Bulk reopen cases', err.data, err.status); }); }); } - this.closeCase = function(caze) { + this.closeCase = function (caze) { var scope = $rootScope.$new(); scope.CaseResolutionStatus = CaseResolutionStatus; scope.CaseImpactStatus = CaseImpactStatus; scope.caseId = caze._id; - scope.updateField = function(data) { + scope.updateField = function (data) { return CaseSrv.update({ caseId: caze._id }, data) - .$promise - .then(function(/*response*/) { - return caze; - }); + .$promise + .then(function (/*response*/) { + return caze; + }); }; var modal = $uibModal.open({ @@ -415,63 +369,63 @@ controller: 'CaseCloseModalCtrl', size: 'lg', resolve: { - caze: function() { + caze: function () { return angular.copy(caze); } } }) - return modal.result.catch(function(err){ - if(err && !_.isString(err)) { + return modal.result.catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Case bulk close', err.data, err.status); } }); } - $scope.updateField = function(data) { + $scope.updateField = function (data) { return CaseSrv.update({ caseId: caseId }, data).$promise; }; - this.bulkClose = function() { - return ModalUtilsSrv.confirm('Close cases', 'Are you sure you want to close the selected ' + self.selection.length+' case(s)?', { + this.bulkClose = function () { + return ModalUtilsSrv.confirm('Close cases', 'Are you sure you want to close the selected ' + self.selection.length + ' case(s)?', { okText: 'Yes, proceed' - }).then(function() { - return self.selection.reduce(function(initialPromise, nextCase) { + }).then(function () { + return self.selection.reduce(function (initialPromise, nextCase) { return initialPromise .then(self.closeCase(nextCase)); }, $q.resolve()); - }).catch(function(err){ - if(err && !_.isString(err)) { + }).catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Case bulk close', err.data, err.status); } }); } - this.getCaseResponders = function(caze, force) { + this.getCaseResponders = function (caze, force) { if (!force && this.caseResponders !== null) { return; } self.caseResponders = null; CortexSrv.getResponders('case', caze._id) - .then(function(responders){ + .then(function (responders) { self.caseResponders = responders; return CortexSrv.promntForResponder(responders); }) - .then(function(response) { - if(response && _.isString(response)) { + .then(function (response) { + if (response && _.isString(response)) { NotificationSrv.log(response, 'warning'); } else { return CortexSrv.runResponder(response.id, response.name, 'case', _.pick(caze, '_id', 'tlp', 'pap')); } }) - .then(function(response){ + .then(function (response) { NotificationSrv.log(['Responder', response.data.responderName, 'started successfully on case', caze.title].join(' '), 'success'); }) - .catch(function(err) { - if(err && !_.isString(err)) { + .catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('CaseList', err.data, err.status); } }); @@ -486,28 +440,28 @@ this.typedCount = undefined; this.loading = false; - this.ok = function() { + this.ok = function () { $uibModalInstance.close(); } - this.cancel = function() { + this.cancel = function () { $uibModalInstance.dismiss(); } - this.confirm = function() { + this.confirm = function () { self.loading = true; - var promises = _.map(self.selection, function(caze) { + var promises = _.map(self.selection, function (caze) { return CaseSrv.forceRemove({ caseId: caze._id }).$promise; }); $q.all(promises) - .then(function(responses) { + .then(function (responses) { self.loading = false; NotificationSrv.log('Cases have been deleted successfully: ' + responses.length, 'success'); $uibModalInstance.close(); }) - .catch(function(errors) { + .catch(function (errors) { self.loading = false; - _.each(errors, function(err) { + _.each(errors, function (err) { NotificationSrv.error('Bulk delete cases', err.data, err.status); }); }) diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html index 7282826230..e66f05426d 100644 --- a/frontend/app/views/partials/alert/list.html +++ b/frontend/app/views/partials/alert/list.html @@ -2,20 +2,21 @@
-

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertListCount}})

+

List of alerts ({{$vm.list.values.length || 0}} of {{$vm.list.total}})

-
+
-
+
-
@@ -33,23 +34,30 @@

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList - + - - Reference - - - - + + Reference + + + + - - Type - - - - + + Type + + + + Imported @@ -58,47 +66,65 @@

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList Read - Title + Title - - Source - - - - + + Source + + + + - - Severity - - - - + + Severity + + + + - Observables + Observables Dates - + O. - - - + + + - + C. - - - + + + - + U. - - - + + + @@ -115,7 +141,8 @@

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList {{::event.sourceRef}} - + @@ -123,68 +150,96 @@

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList - {{::event.type}} + {{::event.type}} - {{event.caseId ? 'Imported' : 'New'}} + {{event.caseId ? + 'Imported' : 'New'}} - {{event.read ? 'Read' : 'Unread'}} + {{event.read ? 'Read' : + 'Unread'}}
{{::event.title}} - + {{::event.title}}
- {{event.source}} + {{event.source}} + -
+
{{::event.observableCount || 0}} -
- O. {{event.date | shortDate}} + -
- C. {{event._createdAt | shortDate}} + -
- U. {{event._updatedAt | shortDate}} +
- +
- - + + - + - + - +
- + - +
@@ -195,7 +250,8 @@

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList
- None + None
@@ -205,7 +261,9 @@

List of alerts ({{$vm.list.total || 0}} of {{$vm.alertList - + + diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 04dc3a3e6e..2ca39cecd8 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -2,7 +2,7 @@
-

List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount}})

+

List of cases ({{$vm.list.values.length || 0}} of {{$vm.list.total}})

@@ -14,15 +14,16 @@

List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount}
-
+
-
+
-
@@ -40,68 +41,101 @@

List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount} - + - + Status - - - + + + - + # Number - - - + + + - + Title - - - + + + - + Severity - - - + + + Details - + Assignee - - - + + + Dates - + S. - - - + + + - + C. - - - + + + - + U. - - - + + + @@ -109,62 +143,81 @@

List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount} - - + + - +
{{currentCase.status === 'Resolved' ? 'Closed' : currentCase.status}} + }[currentCase.status]" + ng-click="$vm.addFilterValue('status', currentCase.status)">{{currentCase.status + === 'Resolved' ? 'Closed' : currentCase.status}}
- - + +
- - - #{{currentCase.number}} - {{currentCase.title}} + + + #{{currentCase.number}} + - {{currentCase.title}}
- None - + None + +
- +
- (Closed at {{currentCase.endDate | shortDate}} as {{$vm.CaseResolutionStatus[currentCase.resolutionStatus]}}) + (Closed at {{currentCase.endDate | shortDate}} as {{$vm.CaseResolutionStatus[currentCase.resolutionStatus]}})
- - Merged from Case #{{currentCase.stats.mergeFrom[0].number}} and - Case #{{currentCase.stats.mergeFrom[1].number}} + + Merged from Case + #{{currentCase.stats.mergeFrom[0].number}} and + + Case #{{currentCase.stats.mergeFrom[1].number}} -
+

-
+
-
+
- + - + Tasks - {{currentCase.extraData.taskStats.total || 0}} + {{currentCase.extraData.taskStats.total || + 0}}
- +
Observables - {{currentCase.extraData.observableStats.total || 0}} + {{currentCase.extraData.observableStats.total + || 0}}
TTPs - {{currentCase.extraData.procedureCount || 0}} + {{currentCase.extraData.procedureCount || + 0}}
@@ -211,20 +273,31 @@

List of cases ({{$vm.list.total || 0}} of {{$vm.caseCount} -
- S. {{currentCase.startDate | shortDate}} + -
- C. {{currentCase._createdAt | shortDate}} + -
- U. {{currentCase._updatedAt | shortDate}} + - - + + diff --git a/frontend/app/views/partials/case/list/toolbar.html b/frontend/app/views/partials/case/list/toolbar.html index d4a04f62e1..0955d90856 100644 --- a/frontend/app/views/partials/case/list/toolbar.html +++ b/frontend/app/views/partials/case/list/toolbar.html @@ -3,8 +3,11 @@
-
-
-
From 58d5c47bf459c52b7851100e968c763e96267b10 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 9 Apr 2021 09:57:41 +0200 Subject: [PATCH 17/63] #1508 Add blank text to updatable-user component and use it in tasks section --- .../app/scripts/directives/updatableUser.js | 27 +-- .../app/views/directives/updatable-user.html | 3 +- .../app/views/partials/case/case.tasks.html | 195 +++++++++++------- .../views/partials/case/case.tasks.item.html | 86 ++++---- 4 files changed, 178 insertions(+), 133 deletions(-) diff --git a/frontend/app/scripts/directives/updatableUser.js b/frontend/app/scripts/directives/updatableUser.js index 1722831dd6..40a5e4c258 100644 --- a/frontend/app/scripts/directives/updatableUser.js +++ b/frontend/app/scripts/directives/updatableUser.js @@ -1,25 +1,25 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveDirectives') - .directive('updatableUser', function(UserSrv, QuerySrv, UtilsSrv, AuthenticationSrv, NotificationSrv) { + .directive('updatableUser', function (UserSrv, QuerySrv, UtilsSrv, AuthenticationSrv, NotificationSrv) { return { restrict: 'E', - link: function(scope, element, attrs, ctrl, transclude) { + link: function (scope, element, attrs, ctrl, transclude) { var cached = false; UtilsSrv.updatableLink(scope, element, attrs, ctrl, transclude); - scope.setValue = function(value) { + scope.setValue = function (value) { scope.value = value; }; scope.getUserInfo = UserSrv.getCache; - scope.$watch('updatable.updating', function(value) { + scope.$watch('updatable.updating', function (value) { - if(value === true && !cached) { + if (value === true && !cached) { var assignableUsers = []; - if(_.isFunction(scope.query)) { + if (_.isFunction(scope.query)) { assignableUsers = scope.query.apply(this, scope.queryParams); } else { assignableUsers = scope.query; @@ -32,12 +32,12 @@ }, sort: ['+name'] }) - .then(function(users) { - scope.userList = users; - }) - .catch(function(err) { - NotificationSrv.error('Fetching users', err.data, err.status); - }); + .then(function (users) { + scope.userList = users; + }) + .catch(function (err) { + NotificationSrv.error('Fetching users', err.data, err.status); + }); cached = true; } @@ -47,6 +47,7 @@ scope: { value: '=?', query: '=', + blankText: '@', queryParams: '=', onUpdate: '&', active: '=?', diff --git a/frontend/app/views/directives/updatable-user.html b/frontend/app/views/directives/updatable-user.html index a64a013626..fd9d33a7c6 100644 --- a/frontend/app/views/directives/updatable-user.html +++ b/frontend/app/views/directives/updatable-user.html @@ -1,6 +1,7 @@
- + {{blankText || 'Not Specified'}} + diff --git a/frontend/app/views/partials/case/case.tasks.html b/frontend/app/views/partials/case/case.tasks.html index 5eb8fd57e3..220c573b4d 100755 --- a/frontend/app/views/partials/case/case.tasks.html +++ b/frontend/app/views/partials/case/case.tasks.html @@ -1,6 +1,7 @@
-
+
@@ -11,8 +12,7 @@

-
@@ -28,24 +28,25 @@

-
- -
+
+ +
-
- - - - - -
+
+ + + + + +
@@ -61,7 +62,8 @@

- + Group @@ -69,50 +71,57 @@

Date Assignee - Actions + Actions - + - + {{task.group}} -
- Closed after {{(task.endDate - task.startDate) | amDurationFormat : 'milliseconds'}} -
-
- Started -
+ + + + + {{task.title}} + +

+
+ Closed after {{(task.endDate - task.startDate) | amDurationFormat : + 'milliseconds'}} +
+
+ Started +
@@ -128,10 +137,12 @@

{{task.startDate | shortDate}} - + - + @@ -148,16 +159,21 @@

- + - + - + - + @@ -189,7 +205,8 @@

- + Group @@ -197,55 +214,65 @@

Date Assignee - Actions + Actions - + - + + - + {{task.group}} - -
- Closed after {{(task.endDate - task.startDate) | amDurationFormat : 'milliseconds'}} -
-
- Started -
+ + + + + + + + {{task.title}} + +

+
+ Closed after {{(task.endDate - task.startDate) | amDurationFormat : + 'milliseconds'}} +
+
+ Started +
- + ({{task.extraData.shareCount || 0}}) @@ -255,10 +282,14 @@

{{task.startDate | shortDate}} - + - + @@ -270,21 +301,27 @@

- + - + - + - + - + diff --git a/frontend/app/views/partials/case/case.tasks.item.html b/frontend/app/views/partials/case/case.tasks.item.html index 6190531c0e..ef8d9ad1ad 100644 --- a/frontend/app/views/partials/case/case.tasks.item.html +++ b/frontend/app/views/partials/case/case.tasks.item.html @@ -6,7 +6,8 @@

This task requires an action from your organisation. - +
@@ -55,7 +56,8 @@

- + | @@ -88,7 +90,7 @@

Title
- +
{{task.title}} @@ -98,7 +100,7 @@

Group
- +
{{task.group}} @@ -108,7 +110,8 @@

Assignee
- +
@@ -120,7 +123,7 @@

Start date
- +
@@ -162,14 +165,15 @@

-

Description

-
- -
-
-
- Not specified -
+

Description

+
+ +
+
+
+ Not specified +
@@ -206,17 +210,24 @@

Task logs

- Markdown Reference + Markdown Reference
- -
+ +
- - + - -
-
-
- -
+

Task sharing

+ +
+ +
From 1d2d0dff798fa12c3d6d04fc1baaaacaaf83bc3d Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 9 Apr 2021 11:18:29 +0200 Subject: [PATCH 18/63] #1946 Fix stream --- .../org/thp/thehive/services/AuditSrv.scala | 92 +++++++++---------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index a88bf5573f..39d89adc80 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -36,33 +36,33 @@ class AuditSrv @Inject() ( eventSrv: EventSrv, db: Database ) extends VertexSrv[Audit] { auditSrv => - lazy val userSrv: UserSrv = userSrvProvider.get - val auditUserSrv = new EdgeSrv[AuditUser, Audit, User] - val auditedSrv = new EdgeSrv[Audited, Audit, Product] - val auditContextSrv = new EdgeSrv[AuditContext, Audit, Product] - val `case` = new SelfContextObjectAudit[Case] - val task = new SelfContextObjectAudit[Task] - val observable = new SelfContextObjectAudit[Observable] - val log = new ObjectAudit[Log, Task] - val caseTemplate = new SelfContextObjectAudit[CaseTemplate] - val taskInTemplate = new ObjectAudit[Task, CaseTemplate] - val alert = new AlertAudit - val share = new ShareAudit - val observableInAlert = new ObjectAudit[Observable, Alert] - val user = new UserAudit - val dashboard = new SelfContextObjectAudit[Dashboard] - val organisation = new SelfContextObjectAudit[Organisation] - val profile = new SelfContextObjectAudit[Profile] - val pattern = new SelfContextObjectAudit[Pattern] - val procedure = new ObjectAudit[Procedure, Case] - val customField = new SelfContextObjectAudit[CustomField] - val page = new SelfContextObjectAudit[Page] - private val pendingAuditsLock = new Object - private val transactionAuditIdsLock = new Object - private val unauditedTransactionsLock = new Object - private var pendingAudits: Map[AnyRef, PendingAudit] = Map.empty - private var transactionAuditIds: List[(AnyRef, EntityId)] = Nil - private var unauditedTransactions: Set[AnyRef] = Set.empty + lazy val userSrv: UserSrv = userSrvProvider.get + val auditUserSrv = new EdgeSrv[AuditUser, Audit, User] + val auditedSrv = new EdgeSrv[Audited, Audit, Product] + val auditContextSrv = new EdgeSrv[AuditContext, Audit, Product] + val `case` = new SelfContextObjectAudit[Case] + val task = new SelfContextObjectAudit[Task] + val observable = new SelfContextObjectAudit[Observable] + val log = new ObjectAudit[Log, Task] + val caseTemplate = new SelfContextObjectAudit[CaseTemplate] + val taskInTemplate = new ObjectAudit[Task, CaseTemplate] + val alert = new AlertAudit + val share = new ShareAudit + val observableInAlert = new ObjectAudit[Observable, Alert] + val user = new UserAudit + val dashboard = new SelfContextObjectAudit[Dashboard] + val organisation = new SelfContextObjectAudit[Organisation] + val profile = new SelfContextObjectAudit[Profile] + val pattern = new SelfContextObjectAudit[Pattern] + val procedure = new ObjectAudit[Procedure, Case] + val customField = new SelfContextObjectAudit[CustomField] + val page = new SelfContextObjectAudit[Page] + private val pendingAuditsLock = new Object + private val transactionAuditIdsLock = new Object + private val unauditedTransactionsLock = new Object + private var pendingAudits: Map[Graph, PendingAudit] = Map.empty + private var transactionAuditIds: List[(Graph, EntityId)] = Nil + private var unauditedTransactions: Set[Graph] = Set.empty /** * Gets the main action Audits by ids sorted by date @@ -77,29 +77,26 @@ class AuditSrv @Inject() ( .sort(_.by("_createdAt", order)) def mergeAudits[R](body: => Try[R])(auditCreator: R => Try[Unit])(implicit graph: Graph): Try[R] = { - val tx = db.currentTransactionId(graph) unauditedTransactionsLock.synchronized { - unauditedTransactions = unauditedTransactions + tx + unauditedTransactions = unauditedTransactions + graph } val result = body unauditedTransactionsLock.synchronized { - unauditedTransactions = unauditedTransactions - tx + unauditedTransactions = unauditedTransactions - graph } result.flatMap { r => auditCreator(r).map(_ => r) } } - def flushPendingAudit()(implicit graph: Graph, authContext: AuthContext): Try[Unit] = flushPendingAudit(db.currentTransactionId(graph)) - - def flushPendingAudit(tx: AnyRef)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { + def flushPendingAudit()(implicit graph: Graph, authContext: AuthContext): Try[Unit] = { logger.debug("Store last audit") - pendingAudits.get(tx).fold[Try[Unit]](Success(())) { p => + pendingAudits.get(graph).fold[Try[Unit]](Success(())) { p => pendingAuditsLock.synchronized { - pendingAudits = pendingAudits - tx + pendingAudits = pendingAudits - graph } - createFromPending(tx, p.audit.copy(mainAction = true), p.context, p.`object`).map { _ => - val (ids, otherTxIds) = transactionAuditIds.partition(_._1 == tx) + createFromPending(p.audit.copy(mainAction = true), p.context, p.`object`).map { _ => + val (ids, otherTxIds) = transactionAuditIds.partition(_._1 == graph) transactionAuditIdsLock.synchronized { transactionAuditIds = otherTxIds } @@ -115,7 +112,7 @@ class AuditSrv @Inject() ( } } - private def createFromPending(tx: AnyRef, audit: Audit, context: Product with Entity, `object`: Option[Product with Entity])(implicit + private def createFromPending(audit: Audit, context: Product with Entity, `object`: Option[Product with Entity])(implicit graph: Graph, authContext: AuthContext ): Try[Unit] = { @@ -127,7 +124,7 @@ class AuditSrv @Inject() ( _ <- `object`.map(auditedSrv.create(Audited(), createdAudit, _)).flip _ = auditContextSrv.create(AuditContext(), createdAudit, context) // this could fail on delete (context doesn't exist) } yield transactionAuditIdsLock.synchronized { - transactionAuditIds = (tx -> createdAudit._id) :: transactionAuditIds + transactionAuditIds = (graph -> createdAudit._id) :: transactionAuditIds } } @@ -135,33 +132,32 @@ class AuditSrv @Inject() ( graph: Graph, authContext: AuthContext ): Try[Unit] = { - def setupCallbacks(tx: AnyRef): Try[Unit] = { + def setupCallbacks(): Try[Unit] = { logger.debug("Setup callbacks for the current transaction") db.addTransactionListener { case Status.ROLLBACK => pendingAuditsLock.synchronized { - pendingAudits = pendingAudits - tx + pendingAudits = pendingAudits - graph } transactionAuditIdsLock.synchronized { - transactionAuditIds = transactionAuditIds.filterNot(_._1 == tx) + transactionAuditIds = transactionAuditIds.filterNot(_._1 == graph) } case _ => } - db.addCallback(() => flushPendingAudit(tx)) + db.addCallback(() => flushPendingAudit()) Success(()) } - val tx = db.currentTransactionId(graph) - if (unauditedTransactions.contains(tx)) { + if (unauditedTransactions.contains(graph)) { logger.debug(s"Audit is disable to the current transaction, $audit ignored.") Success(()) } else { logger.debug(s"Hold $audit, store previous audit if any") - val p = pendingAudits.get(tx) + val p = pendingAudits.get(graph) pendingAuditsLock.synchronized { - pendingAudits = pendingAudits + (tx -> PendingAudit(audit, context, `object`)) + pendingAudits = pendingAudits + (graph -> PendingAudit(audit, context, `object`)) } - p.fold(setupCallbacks(tx))(p => createFromPending(tx, p.audit, p.context, p.`object`)) + p.fold(setupCallbacks())(p => createFromPending(p.audit, p.context, p.`object`)) } } From 18d297dd84a6ae8c0a3279fda48fe3e85409fcbc Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 9 Apr 2021 11:19:00 +0200 Subject: [PATCH 19/63] #1946 Improve log messages --- ScalliGraph | 2 +- conf/logback.xml | 19 +++++-------------- package/logback.xml | 4 ++++ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index e995731253..268150cf26 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit e9957312530f3925a90de47d3e2fb407df805a6e +Subproject commit 268150cf261cd14f025f2c1dfc35caae38f02802 diff --git a/conf/logback.xml b/conf/logback.xml index 2d299bb9ce..e72d600cf0 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -34,21 +34,12 @@ - + + - - + + + diff --git a/package/logback.xml b/package/logback.xml index 4b0ef2a674..af4ead5d9d 100644 --- a/package/logback.xml +++ b/package/logback.xml @@ -34,6 +34,10 @@ + + + + From b685b77d3abb5978b64c2b478947700972577d04 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 9 Apr 2021 12:06:48 +0200 Subject: [PATCH 20/63] #1946 Add limited count query --- ScalliGraph | 2 +- .../cortex/controllers/v0/CortexQueryExecutor.scala | 7 ++++++- .../thp/thehive/controllers/v0/TheHiveQueryExecutor.scala | 5 +++++ .../thp/thehive/controllers/v1/TheHiveQueryExecutor.scala | 5 +++++ thehive/conf/reference.conf | 2 ++ .../test/org/thp/thehive/controllers/v0/QueryTest.scala | 7 ++++--- 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 268150cf26..0fa2e37c38 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 268150cf261cd14f025f2c1dfc35caae38f02802 +Subproject commit 0fa2e37c388a599d918b7eda4711418e6c8640d3 diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala index 96e8390af9..87318660a9 100644 --- a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala +++ b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala @@ -5,6 +5,7 @@ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.FieldsParser import org.thp.scalligraph.models._ import org.thp.scalligraph.query._ +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.{BadRequestError, EntityIdOrName} @@ -18,7 +19,8 @@ import javax.inject.{Inject, Singleton} import scala.reflect.runtime.{universe => ru} @Singleton -class CortexQueryExecutor @Inject() (implicit +class CortexQueryExecutor @Inject() ( + appConfig: ApplicationConfig, override val db: Database, job: PublicJob, report: PublicAnalyzerTemplate, @@ -27,6 +29,9 @@ class CortexQueryExecutor @Inject() (implicit ) extends QueryExecutor { lazy val controllers: List[PublicData] = action :: report :: job :: analyzerTemplate :: Nil + val limitedCountThresholdConfig: ConfigItem[Long, Long] = appConfig.item[Long]("query.limitedCountThreshold", "Maximum number returned by a count") + override val limitedCountThreshold: Long = limitedCountThresholdConfig.get + override lazy val publicProperties: PublicProperties = controllers.map(_.publicProperties).reduce(_ ++ _) override lazy val queries: Seq[ParamQuery[_]] = diff --git a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala index a4f81dbb79..5db1c5c90e 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala @@ -5,6 +5,7 @@ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.controllers.{FObject, Field, FieldsParser} import org.thp.scalligraph.models._ import org.thp.scalligraph.query._ +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.Traversal import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.utils.RichType @@ -37,6 +38,7 @@ object OutputParam { @Singleton class TheHiveQueryExecutor @Inject() ( override val db: Database, + appConfig: ApplicationConfig, alert: PublicAlert, audit: PublicAudit, `case`: PublicCase, @@ -57,6 +59,9 @@ class TheHiveQueryExecutor @Inject() ( lazy val publicDatas: Seq[PublicData] = Seq(alert, audit, `case`, caseTemplate, customField, dashboard, log, observable, observableType, organisation, page, profile, tag, task, user) + val limitedCountThresholdConfig: ConfigItem[Long, Long] = appConfig.item[Long]("query.limitedCountThreshold", "Maximum number returned by a count") + override val limitedCountThreshold: Long = limitedCountThresholdConfig.get + def metaProperties: PublicProperties = PublicPropertyListBuilder .forType[Product](_ => true) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index b11fcace4a..f6c0567967 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -3,6 +3,7 @@ package org.thp.thehive.controllers.v1 import org.thp.scalligraph.controllers.{FObject, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import javax.inject.{Inject, Singleton} @@ -21,6 +22,7 @@ object OutputParam { @Singleton class TheHiveQueryExecutor @Inject() ( + appConfig: ApplicationConfig, alertCtrl: AlertCtrl, auditCtrl: AuditCtrl, caseCtrl: CaseCtrl, @@ -66,6 +68,9 @@ class TheHiveQueryExecutor @Inject() ( taxonomyCtrl ) + val limitedCountThresholdConfig: ConfigItem[Long, Long] = appConfig.item[Long]("query.limitedCountThreshold", "Maximum number returned by a count") + override val limitedCountThreshold: Long = limitedCountThresholdConfig.get + override val version: (Int, Int) = 1 -> 1 override lazy val publicProperties: PublicProperties = controllers.foldLeft(properties.metaProperties)(_ ++ _.publicProperties) diff --git a/thehive/conf/reference.conf b/thehive/conf/reference.conf index fd90ce7b82..0bcf7561f6 100644 --- a/thehive/conf/reference.conf +++ b/thehive/conf/reference.conf @@ -12,6 +12,8 @@ db { initialisationTimeout: 1 hour } +query.limitedCountThreshold: 1000 + storage { provider: localfs localfs.directory: /opt/thp/thehive/files diff --git a/thehive/test/org/thp/thehive/controllers/v0/QueryTest.scala b/thehive/test/org/thp/thehive/controllers/v0/QueryTest.scala index ab415bebd3..14cbcca708 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/QueryTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/QueryTest.scala @@ -13,8 +13,9 @@ class QueryTest extends PlaySpecification with Mockito { val publicTask = new PublicTask(mock[TaskSrv], mock[OrganisationSrv], mock[UserSrv]) val queryExecutor: QueryExecutor = new QueryExecutor { - override val db: Database = mock[Database] - override val version: (Int, Int) = 0 -> 0 + override val limitedCountThreshold: Long = 1000 + override val db: Database = mock[Database] + override val version: (Int, Int) = 0 -> 0 override lazy val queries: Seq[ParamQuery[_]] = publicTask.initialQuery +: publicTask.getQuery +: publicTask.outputQuery +: publicTask.outputQuery +: publicTask.extraQueries override lazy val publicProperties: PublicProperties = publicTask.publicProperties @@ -56,7 +57,7 @@ class QueryTest extends PlaySpecification with Mockito { val queryOrError = taskCtrl.statsParser(Field(input)) queryOrError.isGood must beTrue.updateMessage(s => s"$s\n$queryOrError") - queryOrError.get must not be empty + queryOrError.get must not be empty } } } From 4e1e25a0106f92a7909dec4d2e879202b9b39d17 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 9 Apr 2021 11:34:31 +0200 Subject: [PATCH 21/63] #1925 Sort case templates in case creation dialog --- .../case/case.templates.selector.html | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/frontend/app/views/partials/case/case.templates.selector.html b/frontend/app/views/partials/case/case.templates.selector.html index 839661d649..63830c19e7 100644 --- a/frontend/app/views/partials/case/case.templates.selector.html +++ b/frontend/app/views/partials/case/case.templates.selector.html @@ -3,42 +3,42 @@

Create new Case

- + -

OR

+

OR

-
-

Select a template

-
-
- - +
+

Select a template

+
+
+ + +
-
-
-
- No templates found -
- - - - - - -
- {{template.displayName || template.name}}
-

{{template.description | ellipsis:200}}

-
+
+
+ No templates found +
+ + + + + + +
+ {{template.displayName || template.name}}
+

{{template.description | ellipsis:200}}

+
-
+
From df58be82bc095a8206cba4eee8adc85d850543b3 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 9 Apr 2021 14:37:44 +0200 Subject: [PATCH 22/63] #1950 Use tag colour in tag-list component and refresh custom tags cache on tag update --- .../organisation/OrgCustomTagsListCmp.js | 44 ++++++++++--------- frontend/app/scripts/directives/tag-colour.js | 15 ++++--- .../scripts/services/api/TaxonomyCacheSrv.js | 7 +++ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js b/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js index 9025b3af1b..a61b8e5ca2 100644 --- a/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js +++ b/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js @@ -1,15 +1,15 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveComponents') .component('orgCustomTagsList', { - controller: function($scope, PaginatedQuerySrv, FilteringSrv, TagSrv, UserSrv, ModalUtilsSrv, NotificationSrv) { + controller: function ($scope, PaginatedQuerySrv, FilteringSrv, TaxonomyCacheSrv, TagSrv, UserSrv, ModalUtilsSrv, NotificationSrv) { var self = this; self.tags = []; self.getUserInfo = UserSrv.getCache; - this.$onInit = function() { + this.$onInit = function () { // TODO: FIXME self.filtering = new FilteringSrv('tag', 'custom-tags.list', { version: 'v1', @@ -23,7 +23,7 @@ }); self.filtering.initContext(self.organisation.name) - .then(function() { + .then(function () { self.load(); $scope.$watch('$vm.list.pageSize', function (newValue) { @@ -32,7 +32,7 @@ }); }; - this.load = function() { + this.load = function () { self.list = new PaginatedQuerySrv({ name: 'organisation-custom-tags', @@ -48,8 +48,8 @@ } ], extraData: ['usage'], - onFailure: function(err) { - if(err && err.status === 400) { + onFailure: function (err) { + if (err && err.status === 400) { self.filtering.resetContext(); self.load(); } @@ -74,22 +74,24 @@ }); }; - self.updateColour = function(id, colour) { - TagSrv.updateTag(id, {colour: colour}) - .then(function(/*response*/) { + self.updateColour = function (id, colour) { + TagSrv.updateTag(id, { colour: colour }) + .then(function (/*response*/) { NotificationSrv.success('Tag colour updated successfully'); + TaxonomyCacheSrv.refreshFreeTags(); }) - .catch(function(err) { + .catch(function (err) { NotificationSrv.error('Tag list', err.data, err.status); }) } - self.updateTag = function(id, value) { - TagSrv.updateTag(id, {predicate: value}) - .then(function(/*response*/) { + self.updateTag = function (id, value) { + TagSrv.updateTag(id, { predicate: value }) + .then(function (/*response*/) { NotificationSrv.success('Tag value updated successfully'); + TaxonomyCacheSrv.refreshFreeTags(); }) - .catch(function(err) { + .catch(function (err) { NotificationSrv.error('Tag list', err.data, err.status); }) } @@ -122,28 +124,28 @@ this.search(); }; - this.filterBy = function(field, value) { + this.filterBy = function (field, value) { self.filtering.clearFilters() - .then(function(){ + .then(function () { self.addFilterValue(field, value); }); }; - this.sortBy = function(sort) { + this.sortBy = function (sort) { self.list.sort = sort; self.list.update(); self.filtering.setSort(sort); }; - this.sortByField = function(field) { + this.sortByField = function (field) { var context = this.filtering.context; var currentSort = Array.isArray(context.sort) ? context.sort[0] : context.sort; var sort = null; - if(currentSort.substr(1) !== field) { + if (currentSort.substr(1) !== field) { sort = ['+' + field]; } else { - sort = [(currentSort === '+' + field) ? '-'+field : '+'+field]; + sort = [(currentSort === '+' + field) ? '-' + field : '+' + field]; } self.list.sort = sort; diff --git a/frontend/app/scripts/directives/tag-colour.js b/frontend/app/scripts/directives/tag-colour.js index c6fb358884..e17d9051e8 100644 --- a/frontend/app/scripts/directives/tag-colour.js +++ b/frontend/app/scripts/directives/tag-colour.js @@ -1,19 +1,22 @@ -(function() { +(function () { 'use strict'; - angular.module('theHiveDirectives').directive('tagColour', function($timeout, TaxonomyCacheSrv) { + angular.module('theHiveDirectives').directive('tagColour', function ($timeout, TaxonomyCacheSrv, TagSrv) { return { restrict: 'A', scope: { tag: '=' }, - link: function(scope, element/*, attrs*/) { - if(!scope.tag) { + link: function (scope, element/*, attrs*/) { + if (!scope.tag) { return; } - scope.bgColour = TaxonomyCacheSrv.getColour(scope.tag) || TaxonomyCacheSrv.getColour('_freetags_:' + scope.tag) || '#3c8dbc'; + scope.bgColour = TaxonomyCacheSrv.getColour(scope.tag) || + TaxonomyCacheSrv.getColour('_freetags_:' + scope.tag) || + TagSrv.tagsDefaultColour || + '#000000'; - $timeout(function() { + $timeout(function () { angular.element(element[0]).attr('style', 'background-color:' + scope.bgColour); }); } diff --git a/frontend/app/scripts/services/api/TaxonomyCacheSrv.js b/frontend/app/scripts/services/api/TaxonomyCacheSrv.js index cfae25df92..b7d861be36 100644 --- a/frontend/app/scripts/services/api/TaxonomyCacheSrv.js +++ b/frontend/app/scripts/services/api/TaxonomyCacheSrv.js @@ -44,6 +44,13 @@ }); }; + this.refreshFreeTags = function () { + return TagSrv.getFreeTags() + .then(function (freeTags) { + self.cacheTagColors(freeTags); + }); + } + this.all = function (reload) { var deferred = $q.defer(); From 29615a5a01a766f9f9c64f9b60912a35d8766152 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 9 Apr 2021 14:40:26 +0200 Subject: [PATCH 23/63] #1931 Fix a regression on waiting task list --- .../partials/main/index-waitingtasks.html | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/frontend/app/views/partials/main/index-waitingtasks.html b/frontend/app/views/partials/main/index-waitingtasks.html index 15265aa687..95151a1a0f 100644 --- a/frontend/app/views/partials/main/index-waitingtasks.html +++ b/frontend/app/views/partials/main/index-waitingtasks.html @@ -3,48 +3,51 @@
No records
- - + + - - - - - - + + + + + + - - + + - - - + + - - - + + + + + {{item.title}} + + +
+ + + +
+ + + +
SeverityGroupTaskAction
SeverityGroupTaskAction
- - {{item.group}} -
- - - +
+ + {{item.group}} + -
- -
- 		- -
+ +
From a1138120825e8511ada89d9a81375d379aaa6379 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 9 Apr 2021 15:08:50 +0200 Subject: [PATCH 24/63] #1946 Fix index name in status --- thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index 0bb548f6b2..ecc2fdb1c2 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -87,7 +87,7 @@ class AdminCtrl @Inject() ( } } - private val indexedModels = Seq("Alert", "Attachment", "Audit", "Case", "Log", "Observable", "Tag", "Task") + private val indexedModels = Seq("global") def indexStatus: Action[AnyContent] = entrypoint("Get index status") .authPermittedRoTransaction(db, Permissions.managePlatform) { _ => graph => From f74ff23793786d394bb44c89d949054e085b16a8 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 9 Apr 2021 15:09:29 +0200 Subject: [PATCH 25/63] #1946 Use direct index for count --- ScalliGraph | 2 +- frontend/app/scripts/controllers/RootCtrl.js | 4 +--- .../org/thp/thehive/controllers/v1/AlertCtrl.scala | 10 ++++++++++ .../org/thp/thehive/controllers/v1/CaseCtrl.scala | 5 +++++ .../thp/thehive/controllers/v1/ObservableCtrl.scala | 7 +++++++ .../app/org/thp/thehive/controllers/v1/TagCtrl.scala | 5 +++++ .../org/thp/thehive/controllers/v1/TaskCtrl.scala | 12 ++++++++++-- .../controllers/v1/TheHiveQueryExecutor.scala | 3 +++ 8 files changed, 42 insertions(+), 6 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 7b27220f2c..e26dc5cd8d 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 7b27220f2cc0ad58f2ea5d28be405756d69d19e9 +Subproject commit e26dc5cd8d97527184c46c805ce50d596b9cce35 diff --git a/frontend/app/scripts/controllers/RootCtrl.js b/frontend/app/scripts/controllers/RootCtrl.js index 609ce267b3..3528ea8b11 100644 --- a/frontend/app/scripts/controllers/RootCtrl.js +++ b/frontend/app/scripts/controllers/RootCtrl.js @@ -95,9 +95,7 @@ angular.module('theHiveControllers').controller('RootCtrl', }); StreamQuerySrv('v1', [ - {_name: 'listAlert'}, - {_name: 'filter', _field: 'read', _value: false}, - {_name: 'count'} + {_name: 'countUnreadAlert'}, ], { scope: $scope, rootId: 'any', diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 53ff93d07d..90d7462dc3 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -56,6 +56,16 @@ class AlertCtrl @Inject() ( implicit val caseFilterParser: FieldsParser[Option[InputQuery[Traversal.Unk, Traversal.Unk]]] = FilterQuery.default(caseProperties).paramParser(ru.typeOf[Traversal.V[Case]]).optional.on("caseFilter") override val extraQueries: Seq[ParamQuery[_]] = Seq( + Query.init[Long]( + "countAlert", + (graph, authContext) => + graph.indexCountQuery(s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value}""") + ), + Query.init[Long]( + "countUnreadAlert", + (graph, authContext) => + graph.indexCountQuery(s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND v.read:false""") + ), Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables), Query[Traversal.V[Alert], Traversal.V[Case]]("case", (alertSteps, _) => alertSteps.`case`), Query.withParam[Option[InputQuery[Traversal.Unk, Traversal.Unk]], Traversal.V[Alert], Traversal[ diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index c7af97fed4..b376bbecbd 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -56,6 +56,11 @@ class CaseCtrl @Inject() ( ) override val outputQuery: Query = Query.outputWithContext[RichCase, Traversal.V[Case]]((caseSteps, authContext) => caseSteps.richCase(authContext)) override val extraQueries: Seq[ParamQuery[_]] = Seq( + Query.init[Long]( + "countCase", + (graph, authContext) => + graph.indexCountQuery(s"""v."_label":Case AND v.organisationIds:${organisationSrv.currentId(graph, authContext).value} """) + ), Query[Traversal.V[Case], Traversal.V[Observable]]( "observables", (caseSteps, authContext) => diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index 3c8f8aa531..53df4dcc8a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -74,6 +74,13 @@ class ObservableCtrl @Inject() ( override val outputQuery: Query = Query.output[RichObservable, Traversal.V[Observable]](_.richObservable) override val extraQueries: Seq[ParamQuery[_]] = Seq( + Query.initWithParam[InCase, Long]( + "countObservable", + (inCase, graph, authContext) => + graph.indexCountQuery( + s"""v."_label":Observable AND relatedId:${inCase.caseId.value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" + ) + ), Query[Traversal.V[Observable], Traversal.V[Organisation]]( "organisations", (observableSteps, authContext) => observableSteps.organisations.visible(authContext) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala index 94841e27bd..e35ef1545f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala @@ -45,6 +45,11 @@ class TagCtrl @Inject() ( ) override val extraQueries: Seq[ParamQuery[_]] = Seq( Query.init[Traversal.V[Tag]]("freetags", (graph, authContext) => tagSrv.startTraversal(graph).freetags(organisationSrv)(authContext)), + Query.init[Long]( + "countFreetags", + (graph, authContext) => + graph.indexCountQuery(s"""v."_label":Tag AND v.namespace:_freetags_${organisationSrv.currentId(graph, authContext).value}""") + ), Query[Traversal.V[Tag], Traversal.V[Tag]]("freetags", (tagSteps, authContext) => tagSteps.freetags(organisationSrv)(authContext)), Query.initWithParam[TagHint, Traversal[String, Vertex, Converter[String, Vertex]]]( "tagAutoComplete", diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index ce1450a715..c99aa1f038 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -53,13 +53,20 @@ class TaskCtrl @Inject() ( override val outputQuery: Query = Query.outputWithContext[RichTask, Traversal.V[Task]]((taskSteps, _) => taskSteps.richTask) override val extraQueries: Seq[ParamQuery[_]] = Seq( + Query.initWithParam[InCase, Long]( + "countTask", + (inCase, graph, authContext) => + graph.indexCountQuery( + s"""v."_label":Task AND relatedId:${inCase.caseId.value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" + ) + ), Query.init[Traversal.V[Task]]( "waitingTasks", - (graph, authContext) => taskSrv.startTraversal(graph).has(_.status, TaskStatus.Waiting).inCase.visible(organisationSrv)(authContext) + (graph, authContext) => taskSrv.startTraversal(graph).has(_.status, TaskStatus.Waiting).visible(organisationSrv)(authContext).inCase ), Query.init[Traversal.V[Task]]( // DEPRECATED "waitingTask", - (graph, authContext) => taskSrv.startTraversal(graph).has(_.status, TaskStatus.Waiting).inCase.visible(organisationSrv)(authContext) + (graph, authContext) => taskSrv.startTraversal(graph).has(_.status, TaskStatus.Waiting).visible(organisationSrv)(authContext).inCase ), Query.init[Traversal.V[Task]]( "myTasks", @@ -68,6 +75,7 @@ class TaskCtrl @Inject() ( .startTraversal(graph) .assignTo(authContext.userId) .visible(organisationSrv)(authContext) + .inCase ), Query[Traversal.V[Task], Traversal.V[User]]("assignableUsers", (taskSteps, authContext) => taskSteps.assignableUsers(authContext)), Query[Traversal.V[Task], Traversal.V[Log]]("logs", (taskSteps, _) => taskSteps.logs), diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index f6c0567967..33833f4eb7 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -1,5 +1,6 @@ package org.thp.thehive.controllers.v1 +import org.thp.scalligraph.EntityId import org.thp.scalligraph.controllers.{FObject, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query._ @@ -7,6 +8,8 @@ import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import javax.inject.{Inject, Singleton} +case class InCase(caseId: EntityId) + case class OutputParam(from: Long, to: Long, extraData: Set[String]) object OutputParam { From 25d1dbc87072ebdfac46537cb39f75cde3da08c4 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Fri, 9 Apr 2021 15:30:54 +0200 Subject: [PATCH 26/63] #1951 Use pollingDuration config in the UI --- frontend/app/scripts/app.js | 218 +++++++++--------- .../scripts/services/common/data/StreamSrv.js | 72 +++--- 2 files changed, 150 insertions(+), 140 deletions(-) diff --git a/frontend/app/scripts/app.js b/frontend/app/scripts/app.js index 0f6ebe764c..de55dce768 100644 --- a/frontend/app/scripts/app.js +++ b/frontend/app/scripts/app.js @@ -36,22 +36,22 @@ angular.module('thehive', [ 'theHiveFilters', 'theHiveDirectives', 'theHiveComponents' - ]) - .config(function($resourceProvider) { +]) + .config(function ($resourceProvider) { 'use strict'; $resourceProvider.defaults.stripTrailingSlashes = true; }) - .config(function($compileProvider) { + .config(function ($compileProvider) { 'use strict'; $compileProvider.debugInfoEnabled(false); }) - .config(function($stateProvider, $urlRouterProvider) { + .config(function ($stateProvider, $urlRouterProvider) { 'use strict'; //$urlRouterProvider.otherwise('/index'); - $urlRouterProvider.otherwise(function($injector){ - $injector.invoke(['$state', function($state) { + $urlRouterProvider.otherwise(function ($injector) { + $injector.invoke(['$state', function ($state) { $state.go('app.index'); }]); }); @@ -62,8 +62,8 @@ angular.module('thehive', [ controller: 'AuthenticationCtrl', templateUrl: 'views/login.html', resolve: { - appConfig: function(VersionSrv) { - return VersionSrv.get(); + appConfig: function (VersionSrv) { + return VersionSrv.get(); } }, params: { @@ -89,43 +89,43 @@ angular.module('thehive', [ templateUrl: 'views/app.html', controller: 'RootCtrl', resolve: { - currentUser: function($q, $state, UserSrv, AuthenticationSrv, NotificationSrv) { + currentUser: function ($q, $state, UserSrv, AuthenticationSrv, NotificationSrv) { var deferred = $q.defer(); AuthenticationSrv.current() - .then(function(userData) { - // Prepare user cache - UserSrv.loadCache(userData.organisation); - - return deferred.resolve(userData); - }) - .catch( function(err) { - NotificationSrv.error('App', err.data, err.status); - return deferred.reject(err); - //return deferred.resolve(err.status === 520 ? err.status : null); - }); + .then(function (userData) { + // Prepare user cache + UserSrv.loadCache(userData.organisation); + + return deferred.resolve(userData); + }) + .catch(function (err) { + NotificationSrv.error('App', err.data, err.status); + return deferred.reject(err); + //return deferred.resolve(err.status === 520 ? err.status : null); + }); return deferred.promise; }, - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); }, - appLayout: function($q, $rootScope, AppLayoutSrv) { + appLayout: function ($q, $rootScope, AppLayoutSrv) { AppLayoutSrv.init(); return $q.resolve(); }, - uiConfig: function($q, UiSettingsSrv) { + uiConfig: function ($q, UiSettingsSrv) { return UiSettingsSrv.all(); }, - taxonomyCache: function(TaxonomyCacheSrv) { + taxonomyCache: function (TaxonomyCacheSrv) { return TaxonomyCacheSrv.all(); } } }) .state('app.index', { url: 'index', - onEnter: function($state, AuthenticationSrv) { - $state.go(AuthenticationSrv.getHomePage(), {}, {reload: true}); + onEnter: function ($state, AuthenticationSrv) { + $state.go(AuthenticationSrv.getHomePage(), {}, { reload: true }); } }) .state('app.main', { @@ -156,13 +156,13 @@ angular.module('thehive', [ controller: 'SearchCtrl', title: 'Search', resolve: { - metadata: function($q, DashboardSrv, NotificationSrv) { + metadata: function ($q, DashboardSrv, NotificationSrv) { var defer = $q.defer(); DashboardSrv.getMetadata() - .then(function(response) { + .then(function (response) { defer.resolve(response); - }, function(err) { + }, function (err) { NotificationSrv.error('DashboardViewCtrl', err.data, err.status); defer.reject(err); }); @@ -180,22 +180,22 @@ angular.module('thehive', [ controller: 'SettingsCtrl', title: 'Personal settings', resolve: { - currentUser: function($q, $state, $timeout, AuthenticationSrv, NotificationSrv) { + currentUser: function ($q, $state, $timeout, AuthenticationSrv, NotificationSrv) { var deferred = $q.defer(); AuthenticationSrv.current() - .then(function(userData) { - return deferred.resolve(userData); - }) - .catch( function(err) { - NotificationSrv.error('SettingsCtrl', err.data, err.status); + .then(function (userData) { + return deferred.resolve(userData); + }) + .catch(function (err) { + NotificationSrv.error('SettingsCtrl', err.data, err.status); - return deferred.reject(err); - }); + return deferred.reject(err); + }); return deferred.promise; }, - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); } } @@ -212,7 +212,7 @@ angular.module('thehive', [ controllerAs: '$vm', title: 'Platform administration', resolve: { - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); } }, @@ -227,7 +227,7 @@ angular.module('thehive', [ controllerAs: '$vm', title: 'Profiles administration', resolve: { - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); } }, @@ -242,7 +242,7 @@ angular.module('thehive', [ controllerAs: '$vm', title: 'Taxonomies administration', resolve: { - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); } }, @@ -257,7 +257,7 @@ angular.module('thehive', [ controllerAs: '$vm', title: 'ATT&CK patterns administration', resolve: { - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); } } @@ -272,7 +272,7 @@ angular.module('thehive', [ controllerAs: '$vm', title: 'Organisations administration', resolve: { - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); } }, @@ -286,16 +286,16 @@ angular.module('thehive', [ controller: 'OrgDetailsCtrl', controllerAs: '$vm', resolve: { - organisation: function($stateParams, OrganisationSrv) { + organisation: function ($stateParams, OrganisationSrv) { return OrganisationSrv.get($stateParams.organisation); }, - fields: function(CustomFieldsSrv){ + fields: function (CustomFieldsSrv) { return CustomFieldsSrv.all(); }, - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); }, - uiConfig: function($q, UiSettingsSrv) { + uiConfig: function ($q, UiSettingsSrv) { return UiSettingsSrv.all(true); } }, @@ -310,13 +310,13 @@ angular.module('thehive', [ controllerAs: 'vm', title: 'Analyzer templates administration', resolve: { - appConfig: function($q, VersionSrv) { + appConfig: function ($q, VersionSrv) { var defer = $q.defer(); VersionSrv.get() - .then(function(config) { + .then(function (config) { // Check if Cortex is enabled - if(VersionSrv.hasCortexConnector()) { + if (VersionSrv.hasCortexConnector()) { defer.resolve(config); } else { defer.reject(); @@ -347,9 +347,9 @@ angular.module('thehive', [ controllerAs: '$vm', title: 'Observable administration', resolve: { - types: function(ObservableTypeSrv, $q) { + types: function (ObservableTypeSrv, $q) { return ObservableTypeSrv.list() - .then(function(response) { + .then(function (response) { return $q.resolve(response.data); }); } @@ -365,13 +365,13 @@ angular.module('thehive', [ controller: 'CaseMainCtrl', title: 'Case', resolve: { - caze: function($q, $rootScope, $stateParams, CaseSrv, NotificationSrv) { + caze: function ($q, $stateParams, CaseSrv, NotificationSrv) { var deferred = $q.defer(); CaseSrv.getById($stateParams.caseId, true) - .then(function(data) { + .then(function (data) { deferred.resolve(data); - }).catch(function(response) { + }).catch(function (response) { deferred.reject(response); NotificationSrv.error('CaseMainCtrl', response.data, response.status); }); @@ -419,29 +419,29 @@ angular.module('thehive', [ controller: 'CaseSharingCtrl', controllerAs: '$vm', resolve: { - organisations: function(AuthenticationSrv, OrganisationSrv) { + organisations: function (AuthenticationSrv, OrganisationSrv) { return AuthenticationSrv.current() - .then(function(user) { + .then(function (user) { return OrganisationSrv.links(user.organisation); }) - .then(function(response) { - return _.map(response, function(item) { + .then(function (response) { + return _.map(response, function (item) { return _.pick(item, 'name', 'description'); }); }); }, - profiles: function(ProfileSrv) { + profiles: function (ProfileSrv) { return ProfileSrv.list() - .then(function(response) { + .then(function (response) { - return _.map(_.filter(response.data, function(item) { + return _.map(_.filter(response.data, function (item) { return !item.isAdmin; }), 'name'); }); }, - shares: function(CaseSrv, $stateParams) { + shares: function (CaseSrv, $stateParams) { return CaseSrv.getShares($stateParams.caseId) - .then(function(response) { + .then(function (response) { return response.data; }); } @@ -455,7 +455,7 @@ angular.module('thehive', [ templateUrl: 'views/partials/case/case.alerts.html', controller: 'CaseAlertsCtrl', resolve: { - alerts: function($stateParams, CaseSrv) { + alerts: function ($stateParams, CaseSrv) { return CaseSrv.alerts($stateParams.caseId); } }, @@ -468,14 +468,14 @@ angular.module('thehive', [ templateUrl: 'views/partials/case/case.tasks.item.html', controller: 'CaseTasksItemCtrl', resolve: { - task: function($q, $stateParams, CaseTaskSrv, NotificationSrv) { + task: function ($q, $stateParams, CaseTaskSrv, NotificationSrv) { var deferred = $q.defer(); CaseTaskSrv.getById($stateParams.itemId) - .then(function(data) { + .then(function (data) { deferred.resolve(data); }) - .catch(function(response) { + .catch(function (response) { deferred.reject(response); NotificationSrv.error('taskDetails', response.data, response.status); }); @@ -503,17 +503,17 @@ angular.module('thehive', [ templateUrl: 'views/partials/case/case.observables.item.html', controller: 'CaseObservablesItemCtrl', resolve: { - appConfig: function(VersionSrv) { + appConfig: function (VersionSrv) { return VersionSrv.get(); }, - artifact: function($q, $stateParams, CaseArtifactSrv, NotificationSrv) { + artifact: function ($q, $stateParams, CaseArtifactSrv, NotificationSrv) { var deferred = $q.defer(); CaseArtifactSrv.api().get({ 'artifactId': $stateParams.itemId - }).$promise.then(function(data) { + }).$promise.then(function (data) { deferred.resolve(data); - }).catch(function(response) { + }).catch(function (response) { deferred.reject(response); NotificationSrv.error('Observable Details', response.data, response.status); }); @@ -561,26 +561,26 @@ angular.module('thehive', [ controller: 'DashboardViewCtrl', controllerAs: '$vm', resolve: { - dashboard: function(NotificationSrv, DashboardSrv, $stateParams, $q) { + dashboard: function (NotificationSrv, DashboardSrv, $stateParams, $q) { var defer = $q.defer(); DashboardSrv.get($stateParams.id) - .then(function(response) { + .then(function (response) { defer.resolve(response.data); - }, function(err) { + }, function (err) { NotificationSrv.error('DashboardViewCtrl', err.data, err.status); defer.reject(err); }); return defer.promise; }, - metadata: function($q, DashboardSrv, NotificationSrv) { + metadata: function ($q, DashboardSrv, NotificationSrv) { var defer = $q.defer(); DashboardSrv.getMetadata() - .then(function(response) { + .then(function (response) { defer.resolve(response); - }, function(err) { + }, function (err) { NotificationSrv.error('DashboardViewCtrl', err.data, err.status); defer.reject(err); }); @@ -593,30 +593,30 @@ angular.module('thehive', [ } }); }) - .config(function($httpProvider) { + .config(function ($httpProvider) { 'use strict'; $httpProvider.defaults.xsrfCookieName = 'THEHIVE-XSRF-TOKEN'; $httpProvider.defaults.xsrfHeaderName = 'X-THEHIVE-XSRF-TOKEN'; - $httpProvider.interceptors.push(function($rootScope, $q) { - var isApiCall = function(url) { + $httpProvider.interceptors.push(function ($rootScope, $q) { + var isApiCall = function (url) { return url && url.startsWith('./api') && !url.startsWith('./api/stream'); }; return { - request: function(config) { + request: function (config) { if (isApiCall(config.url)) { $rootScope.async += 1; } return config; }, - response: function(response) { + response: function (response) { if (isApiCall(response.config.url)) { $rootScope.async -= 1; } return response; }, - responseError: function(rejection) { + responseError: function (rejection) { if (isApiCall(rejection.config.url)) { $rootScope.async -= 1; } @@ -625,7 +625,7 @@ angular.module('thehive', [ }; }); }) - .config(function(localStorageServiceProvider) { + .config(function (localStorageServiceProvider) { 'use strict'; localStorageServiceProvider @@ -633,7 +633,7 @@ angular.module('thehive', [ .setStorageType('localStorage') .setNotify(false, false); }) - .config(function(NotificationProvider) { + .config(function (NotificationProvider) { 'use strict'; NotificationProvider.setOptions({ @@ -647,14 +647,14 @@ angular.module('thehive', [ maxCount: 5 }); }) - .config(function($provide, markedProvider, hljsServiceProvider) { + .config(function ($provide, markedProvider, hljsServiceProvider) { 'use strict'; markedProvider.setOptions({ gfm: true, tables: true, sanitize: true, - highlight: function(code, lang) { + highlight: function (code, lang) { if (lang) { return hljs.highlight(lang, code, true).value; } else { @@ -672,55 +672,55 @@ angular.module('thehive', [ $provide.decorator('marked', [ '$delegate', function markedDecorator($delegate) { - // Credits: https://github.com/markedjs/marked/issues/655#issuecomment-383226346 - var defaults = markedProvider.defaults; + // Credits: https://github.com/markedjs/marked/issues/655#issuecomment-383226346 + var defaults = markedProvider.defaults; - var renderer = defaults.renderer; - var linkRenderer = _.wrap(renderer.link, function(originalLink, href, title, text) { - var html = originalLink.call(renderer, href, title, text); - return html.replace(/^scope {Object} *
  • callback {Function}
    • */ - addListener: function(config) { - if(!config.scope) { + addListener: function (config) { + if (!config.scope) { console.error('No scope provided, use the old listen method', config); self.listen(config.rootId, config.objectType, config.callback); return; } var eventName = 'stream:' + config.rootId + '-' + config.objectType; - config.scope.$on(eventName, function(event, data) { - if(!self.disabled) { + config.scope.$on(eventName, function (event, data) { + if (!self.disabled) { config.callback(data); } }); From abcccbd15a7eca1558cb8ba6228d6b04d02439d2 Mon Sep 17 00:00:00 2001 From: To-om Date: Fri, 9 Apr 2021 15:42:22 +0200 Subject: [PATCH 27/63] #1946 Use limited count in case extraData --- ScalliGraph | 2 +- thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala | 7 ++++++- .../app/org/thp/thehive/controllers/v1/CaseRenderer.scala | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index e26dc5cd8d..6a1a6e77bf 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit e26dc5cd8d97527184c46c805ce50d596b9cce35 +Subproject commit 6a1a6e77bf75933aed4c48cdeaf426f984a43081 diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index b376bbecbd..7fb243c841 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -4,6 +4,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.P import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.Database import org.thp.scalligraph.query.{ParamQuery, PropertyUpdater, PublicProperties, Query} +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{IteratorOutput, Traversal} import org.thp.scalligraph.{EntityIdOrName, RichOptionTry, RichSeq} @@ -33,10 +34,14 @@ class CaseCtrl @Inject() ( userSrv: UserSrv, taskSrv: TaskSrv, organisationSrv: OrganisationSrv, - db: Database + db: Database, + appConfig: ApplicationConfig ) extends QueryableCtrl with CaseRenderer { + val limitedCountThresholdConfig: ConfigItem[Long, Long] = appConfig.item[Long]("query.limitedCountThreshold", "Maximum number returned by a count") + val limitedCountThreshold: Long = limitedCountThresholdConfig.get + override val entityName: String = "case" override val publicProperties: PublicProperties = properties.`case` override val initialQuery: Query = diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala index 60bea093d5..1fb00388e0 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala @@ -15,11 +15,12 @@ import java.lang.{Long => JLong} import java.util.{Collection => JCollection, List => JList, Map => JMap} trait CaseRenderer extends BaseRenderer[Case] { + val limitedCountThreshold: Long def observableStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = _.share .observables - .count + .limitedCount(limitedCountThreshold) .domainMap(count => Json.obj("total" -> count)) def taskStats(implicit From 8edfe5f5a13b7c2955a8f066079ff3c505a584b5 Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 08:19:42 +0200 Subject: [PATCH 28/63] Remove useless conversion --- thehive/app/org/thp/thehive/controllers/v0/Conversion.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala index 433517119d..5c4047304c 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/Conversion.scala @@ -46,7 +46,7 @@ object Conversion { .withFieldComputed(_.id, _._id.toString) .withFieldComputed(_.id, _._id.toString) .withFieldConst(_._type, "alert") - .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) + .withFieldComputed(_.tags, _.tags.toSet) .withFieldComputed(_.`case`, _.caseId.map(_.toString)) .withFieldComputed( _.status, @@ -74,7 +74,7 @@ object Conversion { .withFieldRenamed(_._createdAt, _.createdAt) .withFieldRenamed(_._createdBy, _.createdBy) .withFieldConst(_._type, "alert") - .withFieldComputed(_.tags, _.tags.map(_.toString).toSet) + .withFieldComputed(_.tags, _.tags.toSet) .withFieldComputed(_.`case`, _.caseId.map(_.toString)) .withFieldComputed( _.status, From 875951df1d50dd89e9f653db0adaca2c007a66fe Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 08:20:22 +0200 Subject: [PATCH 29/63] #1946 Update frontend to use limited count in case extraData --- frontend/app/views/partials/case/case.list.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 2ca39cecd8..17aa70bc9a 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -255,8 +255,9 @@

    List of cases ({{$vm.list.values.length || 0}} of {{$vm.li Observables - {{currentCase.extraData.observableStats.total - || 0}} + {{currentCase.extraData.observableStats.total < 0 ? + -currentCase.extraData.observableStats.total + "+" : + currentCase.extraData.observableStats.total || 0}}

    From 9c31a110f405e263affb41bcfe859c2bf45de325 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 17:29:52 +0200 Subject: [PATCH 30/63] #1948 WIP: Add support to limitedCount, improve unread alerts count refresh, improve alert list count refresh --- frontend/app/scripts/controllers/RootCtrl.js | 148 +++++++++--------- .../controllers/alert/AlertListCtrl.js | 23 ++- .../app/scripts/services/api/AlertingSrv.js | 49 +++--- .../app/scripts/services/common/QuerySrv.js | 42 ++--- .../app/scripts/services/common/UtilsSrv.js | 51 +++--- .../services/common/data/PaginatedQuerySrv.js | 44 +++--- .../services/common/data/StreamQuerySrv.js | 22 +-- .../scripts/services/common/data/StreamSrv.js | 2 +- frontend/app/views/partials/alert/list.html | 2 +- .../app/views/partials/case/case.list.html | 11 +- 10 files changed, 217 insertions(+), 177 deletions(-) diff --git a/frontend/app/scripts/controllers/RootCtrl.js b/frontend/app/scripts/controllers/RootCtrl.js index 3528ea8b11..6abb174e2a 100644 --- a/frontend/app/scripts/controllers/RootCtrl.js +++ b/frontend/app/scripts/controllers/RootCtrl.js @@ -2,14 +2,14 @@ * Controller for main page */ angular.module('theHiveControllers').controller('RootCtrl', - function($scope, $rootScope, $timeout, $uibModal, $location, $state, AuthenticationSrv, AlertingSrv, StreamSrv, StreamQuerySrv, CaseSrv, CaseTemplateSrv, CustomFieldsSrv, NotificationSrv, AppLayoutSrv, VersionSrv, currentUser, appConfig) { + function ($scope, $rootScope, $timeout, $uibModal, $location, $state, AuthenticationSrv, UtilsSrv, StreamSrv, StreamQuerySrv, CaseSrv, CaseTemplateSrv, CustomFieldsSrv, NotificationSrv, AppLayoutSrv, VersionSrv, currentUser, appConfig) { 'use strict'; - if(currentUser === 520) { + if (currentUser === 520) { $state.go('maintenance'); return; - }else if(!currentUser || !currentUser.login) { - $state.go('login', {autoLogin: appConfig.config.ssoAutoLogin }); + } else if (!currentUser || !currentUser.login) { + $state.go('login', { autoLogin: appConfig.config.ssoAutoLogin }); return; } @@ -26,43 +26,44 @@ angular.module('theHiveControllers').controller('RootCtrl', $scope.currentUser = currentUser; StreamSrv.init(); - VersionSrv.startMonitoring(function(conf) { - var connectors = ['misp', 'cortex']; - - _.each(connectors, function(connector) { - var currentStatus = $scope.appConfig.connectors[connector]; - var newStatus = conf.connectors[connector]; - if(currentStatus && currentStatus.enabled === newStatus.enabled && - newStatus.enabled === true && - currentStatus.status !== newStatus.status) { - - if(newStatus.status === 'OK') { - NotificationSrv.log('The configured ' + connector.toUpperCase() + ' connections are now up.', 'success'); - } else if(newStatus.status === 'WARNING') { - NotificationSrv.log('Some of the configured ' + connector.toUpperCase() + ' connections have errors. Please check your configuration.', 'warning'); - } else { - NotificationSrv.log('The configured ' + connector.toUpperCase() + ' connections have errors. Please check your configuration.', 'error'); - } - } - }); - - $scope.appConfig = conf; - $scope.hasCortexConnector = VersionSrv.hasCortexConnector(); + VersionSrv.startMonitoring(function (conf) { + var connectors = ['misp', 'cortex']; + + _.each(connectors, function (connector) { + var currentStatus = $scope.appConfig.connectors[connector]; + var newStatus = conf.connectors[connector]; + if (currentStatus && currentStatus.enabled === newStatus.enabled && + newStatus.enabled === true && + currentStatus.status !== newStatus.status) { + + if (newStatus.status === 'OK') { + NotificationSrv.log('The configured ' + connector.toUpperCase() + ' connections are now up.', 'success'); + } else if (newStatus.status === 'WARNING') { + NotificationSrv.log('Some of the configured ' + connector.toUpperCase() + ' connections have errors. Please check your configuration.', 'warning'); + } else { + NotificationSrv.log('The configured ' + connector.toUpperCase() + ' connections have errors. Please check your configuration.', 'error'); + } + } + }); + + $scope.appConfig = conf; + $scope.hasCortexConnector = VersionSrv.hasCortexConnector(); }); - CaseTemplateSrv.list().then(function(templates) { + CaseTemplateSrv.list().then(function (templates) { $scope.templates = templates; }); StreamQuerySrv('v1', [ - {_name: 'myTasks'}, - {_name: 'filter', + { _name: 'myTasks' }, + { + _name: 'filter', _in: { _field: 'status', _values: ['Waiting', 'InProgress'] } }, - {_name: 'count'} + { _name: 'count' } ], { scope: $scope, rootId: 'any', @@ -72,14 +73,14 @@ angular.module('theHiveControllers').controller('RootCtrl', name: 'my-tasks.stats' } }, - onUpdate: function(data) { + onUpdate: function (data) { $scope.myCurrentTasksCount = data; } }); StreamQuerySrv('v1', [ - {_name: 'waitingTasks'}, - {_name: 'count'} + { _name: 'waitingTasks' }, + { _name: 'count' } ], { scope: $scope, rootId: 'any', @@ -89,13 +90,13 @@ angular.module('theHiveControllers').controller('RootCtrl', name: 'waiting-tasks.stats' } }, - onUpdate: function(data) { + onUpdate: function (data) { $scope.waitingTasksCount = data; } }); StreamQuerySrv('v1', [ - {_name: 'countUnreadAlert'}, + { _name: 'countUnreadAlert' }, ], { scope: $scope, rootId: 'any', @@ -105,18 +106,19 @@ angular.module('theHiveControllers').controller('RootCtrl', name: 'unread-alert-count' } }, - onUpdate: function(data) { + guard: UtilsSrv.hasAddDeleteEvents, + onUpdate: function (data) { $scope.unreadAlertCount = data; } }); - $scope.$on('templates:refresh', function(){ - CaseTemplateSrv.list().then(function(templates) { + $scope.$on('templates:refresh', function () { + CaseTemplateSrv.list().then(function (templates) { $scope.templates = templates; }); }); - $scope.$on('custom-fields:refresh', function() { + $scope.$on('custom-fields:refresh', function () { // Get custom fields cache $scope.initCustomFieldsCache(); }); @@ -134,33 +136,33 @@ angular.module('theHiveControllers').controller('RootCtrl', // $scope.mispEnabled = enabled; // }); - $scope.initCustomFieldsCache = function() { - CustomFieldsSrv.all().then(function(list) { + $scope.initCustomFieldsCache = function () { + CustomFieldsSrv.all().then(function (list) { $scope.customFieldsCache = list; }); }; $scope.initCustomFieldsCache(); - $scope.isSuperAdmin = function() { + $scope.isSuperAdmin = function () { return AuthenticationSrv.isSuperAdmin(); }; - $scope.selectView = function(name) { + $scope.selectView = function (name) { $state.go('app.main', { viewId: name }); $scope.view.data = name; }; - $scope.logout = function() { - AuthenticationSrv.logout(function() { + $scope.logout = function () { + AuthenticationSrv.logout(function () { $state.go('login'); - }, function(data, status) { + }, function (data, status) { NotificationSrv.error('RootCtrl', data, status); }); }; - $scope.switchOrg = function() { + $scope.switchOrg = function () { var modal = $uibModal.open({ templateUrl: 'views/components/org/orgSwitch.modal.html', controller: 'OrgSwitchCtrl', @@ -171,25 +173,25 @@ angular.module('theHiveControllers').controller('RootCtrl', }); modal.result - .then(function(organisation) { + .then(function (organisation) { $rootScope.isLoading = true; return AuthenticationSrv.current(organisation) - .then(function(userData) { + .then(function (userData) { $scope.currentUser = userData; StreamSrv.cancelPoll(); }); }) - .then(function() { - $state.go('app.index', {}, {reload:true}); + .then(function () { + $state.go('app.index', {}, { reload: true }); }) - .catch(function(err) { - if(err && !_.isString(err)) { + .catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Switch organisation', err.data, err.status); } }) - .finally(function() { - $timeout(function() { + .finally(function () { + $timeout(function () { $rootScope.isLoading = false; }, 500); @@ -197,7 +199,7 @@ angular.module('theHiveControllers').controller('RootCtrl', }; - $scope.createNewCase = function(template) { + $scope.createNewCase = function (template) { var modal = $uibModal.open({ templateUrl: 'views/partials/case/case.creation.html', controller: 'CaseCreationCtrl', @@ -208,46 +210,46 @@ angular.module('theHiveControllers').controller('RootCtrl', }); modal.result - .then(function(data) { + .then(function (data) { $state.go('app.case.details', { caseId: data.id }); }) - .catch(function(err) { - if(err && !_.isString(err)) { + .catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('CaseCreationCtrl', err.data, err.status); } }); }; - $scope.openTemplateSelector = function() { + $scope.openTemplateSelector = function () { var modal = $uibModal.open({ templateUrl: 'views/partials/case/case.templates.selector.html', controller: 'CaseTemplatesDialogCtrl', controllerAs: 'dialog', size: 'lg', resolve: { - templates: function(){ + templates: function () { return $scope.templates; }, - uiSettings: ['UiSettingsSrv', function(UiSettingsSrv) { + uiSettings: ['UiSettingsSrv', function (UiSettingsSrv) { return UiSettingsSrv.all(); }] } }); modal.result - .then(function(template) { + .then(function (template) { $scope.createNewCase(template); }) - .catch(function(err) { - if(err && !_.isString(err)) { + .catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Template Selection', err.data, err.status); } }); }; - $scope.aboutTheHive = function() { + $scope.aboutTheHive = function () { $uibModal.open({ templateUrl: 'views/partials/about.html', controller: 'AboutCtrl', @@ -255,8 +257,8 @@ angular.module('theHiveControllers').controller('RootCtrl', }); }; - $scope.search = function(caseId) { - if(!caseId || !_.isNumber(caseId) || caseId <= 0) { + $scope.search = function (caseId) { + if (!caseId || !_.isNumber(caseId) || caseId <= 0) { return; } @@ -265,19 +267,19 @@ angular.module('theHiveControllers').controller('RootCtrl', caseId: caseId }, range: '0-1' - }, function(response) { - if(response.length === 1) { - $state.go('app.case.details', {caseId: response[0].id}, {reload: true}); + }, function (response) { + if (response.length === 1) { + $state.go('app.case.details', { caseId: response[0].id }, { reload: true }); } else { NotificationSrv.log('Unable to find the case with number ' + caseId, 'error'); } - }, function(err) { + }, function (err) { NotificationSrv.error('Case search', err.data, err.status); }); }; // Used to show spinning refresh icon n times - $scope.getNumber = function(num) { + $scope.getNumber = function (num) { return new Array(num); }; } diff --git a/frontend/app/scripts/controllers/alert/AlertListCtrl.js b/frontend/app/scripts/controllers/alert/AlertListCtrl.js index da5db5a052..3380f64999 100755 --- a/frontend/app/scripts/controllers/alert/AlertListCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertListCtrl.js @@ -2,7 +2,7 @@ (function () { 'use strict'; angular.module('theHiveControllers') - .controller('AlertListCtrl', function ($rootScope, $scope, $q, $state, $uibModal, TagSrv, StreamQuerySrv, CaseTemplateSrv, ModalUtilsSrv, AlertingSrv, NotificationSrv, FilteringSrv, CortexSrv, Severity, VersionSrv) { + .controller('AlertListCtrl', function ($rootScope, $scope, $q, $state, $uibModal, TagSrv, StreamQuerySrv, CaseTemplateSrv, UtilsSrv, ModalUtilsSrv, AlertingSrv, NotificationSrv, FilteringSrv, CortexSrv, Severity, VersionSrv) { var self = this; self.urls = VersionSrv.mispUrls(); @@ -44,6 +44,24 @@ self.filtering.setPageSize(newValue); }); }); + + + StreamQuerySrv('v1', [ + { _name: 'countAlert' } + ], { + scope: $scope, + rootId: 'any', + objectType: 'alert', + query: { + params: { + name: 'alert-count-all' + } + }, + guard: UtilsSrv.hasAddDeleteEvents, + onUpdate: function (data) { + self.alertCountAll = data; + } + }); }; self.load = function () { @@ -51,6 +69,7 @@ scope: $scope, filter: this.filtering.buildQuery(), loadAll: false, + limitedCount: true, sort: self.filtering.context.sort, pageSize: self.filtering.context.pageSize, }; @@ -104,7 +123,7 @@ }; self.bulkFollow = function (follow) { - var ids = _.pluck(self.selection, 'id'); + var ids = _.pluck(self.selection, '_id'); var fn = angular.noop; if (follow === true) { diff --git a/frontend/app/scripts/services/api/AlertingSrv.js b/frontend/app/scripts/services/api/AlertingSrv.js index 7bea51dfe3..ed28e2662f 100644 --- a/frontend/app/scripts/services/api/AlertingSrv.js +++ b/frontend/app/scripts/services/api/AlertingSrv.js @@ -1,7 +1,7 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveServices') - .factory('AlertingSrv', function($q, $http, $rootScope, StatSrv, StreamSrv, PSearchSrv, PaginatedQuerySrv) { + .factory('AlertingSrv', function ($q, $http, $rootScope, StatSrv, StreamSrv, PSearchSrv, PaginatedQuerySrv) { var baseUrl = './api/alert'; @@ -123,13 +123,13 @@ }; var factory = { - getSimilarityFilters: function() { + getSimilarityFilters: function () { return similarityFilters; }, - getSimilarityFilter: function(name) { + getSimilarityFilter: function (name) { return (similarityFilters[name] || {}).filters; }, - list: function(config, callback) { + list: function (config, callback) { return new PaginatedQuerySrv({ name: 'alerts', root: undefined, @@ -138,67 +138,68 @@ scope: config.scope, sort: config.sort || ['-date'], loadAll: config.loadAll || false, - pageSize: config.pageSize || 10, - filter: config.filter || undefined, + pageSize: config.pageSize || 10, + filter: config.filter || undefined, onUpdate: callback || undefined, + limitedCount: config.limitedCount || false, operations: [ - {'_name': 'listAlert'} + { '_name': 'listAlert' } ], extraData: ['importDate'] }); }, - get: function(alertId) { + get: function (alertId) { return $http.get('./api/v1/alert/' + alertId) - .then(function(response) { + .then(function (response) { return response.data; }); }, - create: function(alertId, data) { + create: function (alertId, data) { return $http.post(baseUrl + '/' + alertId + '/createCase', data || {}); }, - update: function(alertId, updates) { + update: function (alertId, updates) { return $http.patch(baseUrl + '/' + alertId, updates); }, - mergeInto: function(alertId, caseId) { + mergeInto: function (alertId, caseId) { return $http.post(baseUrl + '/' + alertId + '/merge/' + caseId); }, - bulkMergeInto: function(alertIds, caseId) { + bulkMergeInto: function (alertIds, caseId) { return $http.post(baseUrl + '/merge/_bulk', { caseId: caseId, alertIds: alertIds }); }, - canMarkAsRead: function(event) { + canMarkAsRead: function (event) { return !!!event.read; }, - canMarkAsUnread: function(event) { + canMarkAsUnread: function (event) { return !!event.read; }, - markAsRead: function(alertId) { + markAsRead: function (alertId) { return $http.post(baseUrl + '/' + alertId + '/markAsRead'); }, - markAsUnread: function(alertId) { + markAsUnread: function (alertId) { return $http.post(baseUrl + '/' + alertId + '/markAsUnread'); }, - follow: function(alertId) { + follow: function (alertId) { return $http.post(baseUrl + '/' + alertId + '/follow'); }, - unfollow: function(alertId) { + unfollow: function (alertId) { return $http.post(baseUrl + '/' + alertId + '/unfollow'); }, - forceRemove: function(alertId) { + forceRemove: function (alertId) { return $http.delete(baseUrl + '/' + alertId, { params: { force: 1 @@ -206,7 +207,7 @@ }); }, - bulkRemove: function(alertIds) { + bulkRemove: function (alertIds) { return $http.post(baseUrl + '/delete/_bulk', { ids: alertIds }, { @@ -216,7 +217,7 @@ }); }, - stats: function(scope) { + stats: function (scope) { var field = 'status', result = {}, statConfig = { @@ -230,7 +231,7 @@ rootId: 'any', objectType: 'alert', scope: scope, - callback: function() { + callback: function () { StatSrv.get(statConfig); } }); diff --git a/frontend/app/scripts/services/common/QuerySrv.js b/frontend/app/scripts/services/common/QuerySrv.js index fbd83b143f..ddbc36600b 100644 --- a/frontend/app/scripts/services/common/QuerySrv.js +++ b/frontend/app/scripts/services/common/QuerySrv.js @@ -1,17 +1,17 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveServices') - .service('QuerySrv', function($http, $q) { + .service('QuerySrv', function ($http, $q) { var self = this; - this._buildSort = function(sort) { + this._buildSort = function (sort) { var input = _.isArray(sort) ? sort : [sort]; - return _.map(input, function(item) { + return _.map(input, function (item) { var ret = {}; - if(item.startsWith('-')) { + if (item.startsWith('-')) { ret[item.slice(1)] = 'desc'; } else if (item.startsWith('+')) { ret[item.slice(1)] = 'asc'; @@ -21,7 +21,7 @@ }); }; - this.query = function(version, operations, config) { + this.query = function (version, operations, config) { return $http.post('./api/' + version + '/query', { query: operations }, config || {}); @@ -36,43 +36,43 @@ * @param {type} options Options object defining filters, sort, pagination and addition $http options * @return {type} Promise resolving to the list of the searched objects */ - this.call = function(version, selectorOperation, options) { + this.call = function (version, selectorOperation, options) { var operations = [].concat(selectorOperation); var config = {}; // Apply filter is defined if (options && options.filter && !_.isEmpty(options.filter)) { operations.push( - _.extend({'_name': 'filter'}, options.filter) + _.extend({ '_name': 'filter' }, options.filter) ); } // Apply sort is defined if (options && options.sort) { operations.push( - _.extend({'_name': 'sort'}, {'_fields': this._buildSort(options.sort)}) + _.extend({ '_name': 'sort' }, { '_fields': this._buildSort(options.sort) }) ); } // Apply pagination if isDefined - if(options && options.page) { + if (options && options.page) { operations.push( - _.extend({'_name': 'page'}, options.page) + _.extend({ '_name': 'page' }, options.page) ); } - if(options && options.name) { + if (options && options.name) { config.params = { name: options.name }; } - if(options && options.config) { + if (options && options.config) { config = _.extend({}, config, options.config); } return self.query(version, operations, config) - .then(function(response) { + .then(function (response) { return $q.resolve(response.data); }); }; @@ -86,32 +86,32 @@ * @param {type} options Options object defining filters and addition $http options * @return {type} Promise resolving to the total of the searched objects */ - this.count = function(version, selectorOperation, options) { + this.count = function (version, selectorOperation, options) { var operations = [].concat(selectorOperation); var config = {}; // Apply filter is defined if (options && options.filter && !_.isEmpty(options.filter)) { operations.push( - _.extend({'_name': 'filter'}, options.filter) + _.extend({ '_name': 'filter' }, options.filter) ); } - if(options && options.name) { + if (options && options.name) { config.params = { name: options.name + '.count' }; } - if(options && options.config) { + if (options && options.config) { config = _.extend({}, config, options.config); } - // Add filters - operations.push({'_name': 'count'}); + // Add count + operations.push({ '_name': options.limitedCount ? 'limitedCount' : 'count' }); return self.query(version, operations, config) - .then(function(response) { + .then(function (response) { return $q.resolve(response.data); }); }; diff --git a/frontend/app/scripts/services/common/UtilsSrv.js b/frontend/app/scripts/services/common/UtilsSrv.js index e63a89ee41..7365af6718 100644 --- a/frontend/app/scripts/services/common/UtilsSrv.js +++ b/frontend/app/scripts/services/common/UtilsSrv.js @@ -1,11 +1,11 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveServices') - .factory('UtilsSrv', function($location) { + .factory('UtilsSrv', function ($location) { var sensitiveTypes = ['url', 'ip', 'mail', 'domain', 'filename']; var service = { - guid: function() { + guid: function () { function s4() { return Math.floor((1 + Math.random()) * 0x10000) .toString(16) @@ -14,15 +14,15 @@ return s4() + s4() + '-' + s4() + '-' + s4() + '-' + s4() + '-' + s4() + s4() + s4(); }, - objectify: function(arr, property) { - return _.map(arr, function(str) { + objectify: function (arr, property) { + return _.map(arr, function (str) { var obj = {}; obj[property] = str; return obj; }); }, - fangValue: function(value) { + fangValue: function (value) { return value .replace(/\[\.\]/g, ".") .replace(/hxxp/gi, "http") @@ -30,7 +30,7 @@ .replace(/http/gi, "hxxp"); }, - fang: function(observable) { + fang: function (observable) { if (sensitiveTypes.indexOf(observable.dataType) === -1) { return observable.data; } @@ -38,16 +38,16 @@ return service.fangValue(observable.data); }, - unfang: function(observable) { + unfang: function (observable) { return observable.data .replace(/\[\.\]/g, ".") .replace(/hxxp/gi, "http"); }, - shallowClearAndCopy: function(src, dst) { + shallowClearAndCopy: function (src, dst) { dst = dst || {}; - angular.forEach(dst, function(value, key) { + angular.forEach(dst, function (value, key) { delete dst[key]; }); @@ -59,7 +59,7 @@ return dst; }, - updatableLink: function(scope, element, attrs) { + updatableLink: function (scope, element, attrs) { scope.updatable = { 'updating': false }; @@ -67,10 +67,10 @@ if (!angular.isDefined(scope.active)) { scope.active = false; } - scope.edit = function() { + scope.edit = function () { scope.updatable.updating = true; }; - scope.update = function(newValue) { + scope.update = function (newValue) { if (angular.isDefined(newValue)) { scope.value = newValue; } @@ -82,11 +82,11 @@ updateResult = updateResult.$promise; } if (angular.isObject(updateResult) && angular.isFunction(updateResult.then)) { - updateResult.then(function() { + updateResult.then(function () { scope.oldValue = scope.value; scope.active = false; scope.format = 'static'; - }, function() { + }, function () { scope.value = scope.oldValue; }); } else { @@ -97,16 +97,16 @@ } scope.updatable.updating = false; }; - scope.cancel = function() { + scope.cancel = function () { scope.value = scope.oldValue; scope.updatable.updating = false; }; - scope.clear = function() { + scope.clear = function () { scope.value = null; }; }, - extractQueryParam: function(paramName, queryString) { + extractQueryParam: function (paramName, queryString) { if (!queryString || !paramName) { return; } @@ -116,17 +116,17 @@ if (param) { return param; } else { - var parsedQuery = _.find(queryString.split('&'), function(str) { + var parsedQuery = _.find(queryString.split('&'), function (str) { return str.startsWith(paramName + '='); }); return parsedQuery ? parsedQuery.substr(paramName.length + 1) : undefined; } }, - getDateRange: function(operator) { + getDateRange: function (operator) { var from; - switch(operator) { + switch (operator) { case 'last7days': from = moment().subtract(7, 'days'); break; @@ -154,6 +154,15 @@ from: from.hour(0).minutes(0).seconds(0).toDate(), to: to.hour(23).minutes(59).seconds(59).toDate() }; + }, + + hasAddDeleteEvents: function (updates) { + if (!updates || updates.length === 0) + return; + + return _.without(_.map(updates, function (u) { + return u.base.operation; + }), 'Update').length > 0; } }; diff --git a/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js b/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js index 25cce42f48..827c058de3 100644 --- a/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js +++ b/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js @@ -1,9 +1,9 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveServices') - .service('PaginatedQuerySrv', function($filter, StreamSrv, QuerySrv) { + .service('PaginatedQuerySrv', function ($filter, StreamSrv, QuerySrv) { - return function(options) { + return function (options) { var self = this; // Internal fields @@ -22,6 +22,7 @@ this.loadAll = !!options.loadAll; this.pageSize = options.pageSize || 10; this.pageOptions = options.pageOptions || {}; + this.limitedCount = options.limitedCount || false; this.baseFilter = options.baseFilter; this.filter = options.filter; this.sort = options.sort; @@ -43,8 +44,8 @@ scope: self.scope, rootId: self.root, objectType: self.streamObjectType, - callback: function(updates) { - if(!self.guard || self.guard(updates)) { + callback: function (updates) { + if (!self.guard || self.guard(updates)) { self.update(updates, true); } } @@ -56,12 +57,12 @@ /* Function to change the page */ - this.changePage = function() { + this.changePage = function () { if (this.loadAll) { this.values.length = 0; var end = this.currentPage * this.pageSize; var start = end - this.pageSize; - angular.forEach(this.allValues.slice(start, end), function(d) { + angular.forEach(this.allValues.slice(start, end), function (d) { self.values.push(d); }); @@ -76,7 +77,7 @@ /* Function to compute the range of the page */ - this.getPage = function() { + this.getPage = function () { if (this.loadAll) { return; } @@ -90,7 +91,7 @@ from: from, to: to }, - self.extraData ? {extraData: self.extraData} : {}, + self.extraData ? { extraData: self.extraData } : {}, self.pageOptions ? self.pageOptions : {} ); }; @@ -101,20 +102,20 @@ * * @return {type} filters definition */ - this.getFilter = function() { - if(!this.baseFilter && !this.filter) { + this.getFilter = function () { + if (!this.baseFilter && !this.filter) { return; } - var predicates = _.filter([this.baseFilter, this.filter], function(item) { + var predicates = _.filter([this.baseFilter, this.filter], function (item) { return !_.isEmpty(item); }); - if(predicates.length === 0) { + if (predicates.length === 0) { return []; } - return predicates.length === 1 ? predicates[0] : {'_and': predicates}; + return predicates.length === 1 ? predicates[0] : { '_and': predicates }; }; /** @@ -122,14 +123,14 @@ * * @return {type} sort definition */ - this.getSort = function() { + this.getSort = function () { return self.sort; }; /* Function to change the page */ - this.update = function(updates, forceCount) { + this.update = function (updates, forceCount) { var filters = self.getFilter(); // Get the list @@ -140,7 +141,7 @@ config: self.config, withParent: false, name: self.name - }).then(function(data) { + }).then(function (data) { if (self.loadAll) { self.allValues = data; @@ -153,15 +154,15 @@ self.onUpdate(updates); } } - }).catch(function(err) { - if(self.onFailure) { + }).catch(function (err) { + if (self.onFailure) { self.onFailure(err); } }); // get the total if not cached var hash = $filter('md5')(JSON.stringify(this.filter)); - if(forceCount || (!!!this.loadAll && this.filterHash !== hash)) { + if (forceCount || (!!!this.loadAll && this.filterHash !== hash)) { this.filterHash = hash; // Compute the total again @@ -169,7 +170,8 @@ filter: filters, name: self.name, config: self.config, - }).then(function(total) { + limitedCount: self.limitedCount + }).then(function (total) { self.total = total; }); } diff --git a/frontend/app/scripts/services/common/data/StreamQuerySrv.js b/frontend/app/scripts/services/common/data/StreamQuerySrv.js index 7f08c58efd..174015bc95 100644 --- a/frontend/app/scripts/services/common/data/StreamQuerySrv.js +++ b/frontend/app/scripts/services/common/data/StreamQuerySrv.js @@ -1,22 +1,24 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveServices') - .factory('StreamQuerySrv', function($http, StreamSrv, QuerySrv) { - return function(version, operations, config) { + .factory('StreamQuerySrv', function ($http, StreamSrv, QuerySrv) { + return function (version, operations, config) { StreamSrv.addListener({ + scope: config.scope, rootId: config.rootId, objectType: config.objectType, - scope: config.scope, - callback:function() { - QuerySrv.query(version, operations, config.query) - .then(function(response) { - config.onUpdate(response.data); - }); + callback: function (updates) { + if (!config.guard || config.guard(updates)) { + QuerySrv.query(version, operations, config.query) + .then(function (response) { + config.onUpdate(response.data); + }); + } } }); QuerySrv.query(version, operations, config.query) - .then(function(response) { + .then(function (response) { config.onUpdate(response.data); }); }; diff --git a/frontend/app/scripts/services/common/data/StreamSrv.js b/frontend/app/scripts/services/common/data/StreamSrv.js index d7705e20dd..c1f8d9e2eb 100644 --- a/frontend/app/scripts/services/common/data/StreamSrv.js +++ b/frontend/app/scripts/services/common/data/StreamSrv.js @@ -131,7 +131,7 @@ var pollingDuration; try { pollingDuration = appConfig.config.pollingDuration - } catch { + } catch (error) { pollingDuration = 0 } diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html index e66f05426d..537f8df2b1 100644 --- a/frontend/app/views/partials/alert/list.html +++ b/frontend/app/views/partials/alert/list.html @@ -2,7 +2,7 @@
    -

    List of alerts ({{$vm.list.values.length || 0}} of {{$vm.list.total}})

    +

    List of alerts ({{$vm.list.total || 0}} of {{$vm.alertCountAll}})

    diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 17aa70bc9a..148fa13a3e 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -255,9 +255,14 @@

    List of cases ({{$vm.list.values.length || 0}} of {{$vm.li Observables - {{currentCase.extraData.observableStats.total < 0 ? - -currentCase.extraData.observableStats.total + "+" : - currentCase.extraData.observableStats.total || 0}} + + {{-currentCase.extraData.observableStats.total + '+'}} + + + {{currentCase.extraData.observableStats.total || 0}} +

    From f8c4c0fc1206ea01ea34fa08b02ffaee612d76ac Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 18:04:27 +0200 Subject: [PATCH 31/63] #1948 WIP: Add limitedCount filter --- frontend/app/index.html | 1 + .../scripts/controllers/case/CaseListCtrl.js | 18 +- .../scripts/controllers/case/CaseMainCtrl.js | 187 +++++++++--------- frontend/app/scripts/filters/limited-count.js | 16 ++ frontend/app/views/partials/alert/list.html | 4 +- .../app/views/partials/case/case.list.html | 18 +- 6 files changed, 137 insertions(+), 107 deletions(-) create mode 100644 frontend/app/scripts/filters/limited-count.js diff --git a/frontend/app/index.html b/frontend/app/index.html index 35e7900ca0..fd8df0c4c8 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -284,6 +284,7 @@ + diff --git a/frontend/app/scripts/controllers/case/CaseListCtrl.js b/frontend/app/scripts/controllers/case/CaseListCtrl.js index 16f3bdbcb3..fe6a49846e 100644 --- a/frontend/app/scripts/controllers/case/CaseListCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseListCtrl.js @@ -4,7 +4,7 @@ .controller('CaseListCtrl', CaseListCtrl) .controller('CaseBulkDeleteModalCtrl', CaseBulkDeleteModalCtrl); - function CaseListCtrl($scope, $rootScope, $q, $uibModal, StreamQuerySrv, FilteringSrv, SecuritySrv, ModalUtilsSrv, PaginatedQuerySrv, EntitySrv, CaseSrv, UserSrv, AuthenticationSrv, CaseResolutionStatus, CaseImpactStatus, NotificationSrv, CortexSrv) { + function CaseListCtrl($scope, $rootScope, $q, $uibModal, StreamQuerySrv, FilteringSrv, SecuritySrv, ModalUtilsSrv, PaginatedQuerySrv, EntitySrv, CaseSrv, UserSrv, AuthenticationSrv, CaseResolutionStatus, CaseImpactStatus, NotificationSrv, CortexSrv, UtilsSrv) { var self = this; this.openEntity = EntitySrv.open; @@ -49,6 +49,22 @@ }); }); + StreamQuerySrv('v1', [ + { _name: 'countCase' } + ], { + scope: $scope, + rootId: 'any', + objectType: 'case', + query: { + params: { + name: 'case-count-all' + } + }, + guard: UtilsSrv.hasAddDeleteEvents, + onUpdate: function (data) { + self.caseCountAll = data; + } + }); }; this.load = function () { diff --git a/frontend/app/scripts/controllers/case/CaseMainCtrl.js b/frontend/app/scripts/controllers/case/CaseMainCtrl.js index ebd98a62c1..1436efc0bb 100644 --- a/frontend/app/scripts/controllers/case/CaseMainCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseMainCtrl.js @@ -1,7 +1,7 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveControllers').controller('CaseMainCtrl', - function($scope, $rootScope, $state, $stateParams, $q, $uibModal, CaseTabsSrv, CaseSrv, UserSrv, MispSrv, StreamSrv, StreamQuerySrv, StreamStatSrv, NotificationSrv, UtilsSrv, CaseResolutionStatus, CaseImpactStatus, CortexSrv, caze) { + function ($scope, $rootScope, $state, $stateParams, $q, $uibModal, CaseTabsSrv, CaseSrv, UserSrv, StreamSrv, StreamQuerySrv, NotificationSrv, UtilsSrv, CaseResolutionStatus, CaseImpactStatus, CortexSrv, caze) { $scope.CaseResolutionStatus = CaseResolutionStatus; $scope.CaseImpactStatus = CaseImpactStatus; $scope.caseResponders = null; @@ -30,26 +30,26 @@ $scope.canEdit = caze.extraData.permissions.indexOf('manageCase') !== -1; - $scope.initExports = function() { - $scope.existingExports = _.filter($scope.caze.extraData.alerts || [], function(item) { + $scope.initExports = function () { + $scope.existingExports = _.filter($scope.caze.extraData.alerts || [], function (item) { return item.type === 'misp'; }).length; }; $scope.initExports(); - $scope.countIoc = function(link) { - return _.filter(link.linkedWith, function(l) { + $scope.countIoc = function (link) { + return _.filter(link.linkedWith, function (l) { return l.ioc; }).length; }; CaseSrv.links({ caseId: $scope.caseId - }, function(data) { - $scope.links = _.map(data, function(item){ - item.linksCount = item.linkedWith.length || 0; + }, function (data) { + $scope.links = _.map(data, function (item) { + item.linksCount = item.linkedWith.length || 0; - return item; + return item; }); if (data.length > 0) { @@ -67,15 +67,15 @@ scope: $scope, rootId: $scope.caseId, objectType: 'case', - callback: function(updates) { + callback: function (updates) { CaseSrv.getById($stateParams.caseId, true) - .then(function(data) { + .then(function (data) { $scope.caze = data; - if(updates.length === 1 && updates[0] && updates[0].base.details.customFields){ + if (updates.length === 1 && updates[0] && updates[0].base.details.customFields) { $scope.$broadcast('case:refresh-custom-fields'); } - }).catch(function(response) { + }).catch(function (response) { NotificationSrv.error('CaseMainCtrl', response.data, response.status); }); } @@ -83,15 +83,16 @@ // Stats for case tasks counter StreamQuerySrv('v1', [ - {_name: 'getCase', idOrName: caseId}, - {_name: 'tasks'}, - {_name: 'filter', - _not: { - '_field': 'status', - '_value': 'Cancel' - } - }, - {_name: 'count'} + { _name: 'countTask', caseId: caseId }, + // {_name: 'getCase', idOrName: caseId}, + // {_name: 'tasks'}, + // {_name: 'filter', + // _not: { + // '_field': 'status', + // '_value': 'Cancel' + // } + // }, + // {_name: 'count'} ], { scope: $scope, rootId: caseId, @@ -101,16 +102,15 @@ name: 'task-stats-' + caseId } }, - onUpdate: function(updates) { + guard: UtilsSrv.hasAddDeleteEvents, + onUpdate: function (updates) { $scope.tasksCount = updates; } }); // Stats for case observables counter StreamQuerySrv('v1', [ - {_name: 'getCase', idOrName: caseId}, - {_name: 'observables'}, - {_name: 'count'} + { _name: 'countObservable', caseId: caseId }, ], { scope: $scope, rootId: caseId, @@ -120,16 +120,17 @@ name: 'observable-stats-' + caseId } }, - onUpdate: function(updates) { + guard: UtilsSrv.hasAddDeleteEvents, + onUpdate: function (updates) { $scope.observableCount = updates; } }); - // Stats for case observables counter + // Stats for case alerts counter StreamQuerySrv('v1', [ - {_name: 'getCase', idOrName: caseId}, - {_name: 'alerts'}, - {_name: 'count'} + { _name: 'getCase', idOrName: caseId }, + { _name: 'alerts' }, + { _name: 'count' } ], { scope: $scope, rootId: caseId, @@ -139,34 +140,34 @@ name: 'alert-stats-' + caseId } }, - onUpdate: function(updates) { + onUpdate: function (updates) { $scope.alertCount = updates; } }); - $scope.$on('tasks:task-removed', function(event, task) { + $scope.$on('tasks:task-removed', function (event, task) { CaseTabsSrv.removeTab('task-' + task._id); }); - $scope.$on('observables:observable-removed', function(event, observable) { + $scope.$on('observables:observable-removed', function (event, observable) { CaseTabsSrv.removeTab('observable-' + observable._id); }); - $scope.openTab = function(tabName) { + $scope.openTab = function (tabName) { var tab = CaseTabsSrv.getTab(tabName), params = angular.extend({}, $state.params, tab.params || {}); $state.go(tab.state, params); }; - $scope.removeTab = function(tab) { + $scope.removeTab = function (tab) { var switchToDetails = CaseTabsSrv.removeTab(tab); - if(switchToDetails) { + if (switchToDetails) { $scope.openTab('details'); } }; - $scope.switchFlag = function() { + $scope.switchFlag = function () { if ($scope.caze.flag === true) { $scope.updateField('flag', false); } else { @@ -175,7 +176,7 @@ }; // update a specific case field - $scope.updateField = function(fieldName, newValue) { + $scope.updateField = function (fieldName, newValue) { var data = {}; if (angular.isString(fieldName)) { @@ -188,10 +189,10 @@ CaseSrv.update({ caseId: $scope.caseId - }, data, function(/*response*/) { + }, data, function (/*response*/) { //UtilsSrv.shallowClearAndCopy(response, $scope.caze); defer.resolve($scope.caze); - }, function(response) { + }, function (response) { NotificationSrv.error('caseDetails', response.data, response.status); defer.reject(response); }); @@ -199,33 +200,33 @@ return defer.promise; }; - $scope.isCaseClosed = function() { + $scope.isCaseClosed = function () { return $scope.caze.status === 'Resolved'; }; - $scope.isCaseTruePositive = function() { + $scope.isCaseTruePositive = function () { return $scope.caze.resolutionStatus === 'TruePositive'; }; - $scope.openCloseDialog = function() { + $scope.openCloseDialog = function () { var modalInstance = $uibModal.open({ scope: $scope, templateUrl: 'views/partials/case/case.close.html', controller: 'CaseCloseModalCtrl', size: 'lg', resolve: { - caze: function() { + caze: function () { return angular.copy($scope.caze); } } }); - modalInstance.result.then(function() { + modalInstance.result.then(function () { $state.go('app.cases'); }); }; - $scope.reopenCase = function() { + $scope.reopenCase = function () { $uibModal.open({ scope: $scope, templateUrl: 'views/partials/case/case.reopen.html', @@ -233,26 +234,26 @@ }); }; - $scope.mergeCase = function() { + $scope.mergeCase = function () { var caseModal = $uibModal.open({ templateUrl: 'views/partials/case/case.merge.html', controller: 'CaseMergeModalCtrl', controllerAs: 'dialog', size: 'lg', resolve: { - source: function() { + source: function () { return $scope.caze; }, - title: function() { + title: function () { return 'Merge Case #' + $scope.caze.number; }, - prompt: function() { + prompt: function () { return '#' + $scope.caze.number + ': ' + $scope.caze.title; } } }); - caseModal.result.then(function(selectedCase) { + caseModal.result.then(function (selectedCase) { CaseSrv.merge([$scope.caze._id, selectedCase._id]) .then(function (response) { var merged = response.data; @@ -272,15 +273,15 @@ // caseId: $scope.caze.id, // mergedCaseId: selectedCase.id // }, , ); - }).catch(function(err) { - if(err && !_.isString(err)) { + }).catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Case Merge', err.data, err.status); } }); }; - $scope.exportToMisp = function() { - if($scope.appConfig.connectors.misp && $scope.appConfig.connectors.misp.servers.length === 0) { + $scope.exportToMisp = function () { + if ($scope.appConfig.connectors.misp && $scope.appConfig.connectors.misp.servers.length === 0) { NotificationSrv.log('There are no MISP servers defined', 'error'); return; } @@ -291,71 +292,71 @@ controllerAs: 'dialog', size: 'lg', resolve: { - caze: function() { + caze: function () { return $scope.caze; }, - config: function() { + config: function () { return $scope.appConfig.connectors.misp; } } }); - modalInstance.result.then(function() { + modalInstance.result.then(function () { return CaseSrv.getById($scope.caseId, true); // return CaseSrv.get({ // 'caseId': $scope.caseId, // 'nstats': true // }).$promise; - }).then(function(data) { + }).then(function (data) { $scope.caze = data; $scope.initExports(); }); }; - $scope.removeCase = function() { - var modalInstance = $uibModal.open({ - templateUrl: 'views/partials/case/case.delete.confirm.html', - controller: 'CaseDeleteModalCtrl', - resolve: { - caze: function() { - return $scope.caze; - } - } - }); - - modalInstance.result.then(function() { - $state.go('app.cases'); - }) - .catch(function(err) { - if(err && !_.isString(err)) { - NotificationSrv.error('caseDetails', err.data, err.status); - } - }); + $scope.removeCase = function () { + var modalInstance = $uibModal.open({ + templateUrl: 'views/partials/case/case.delete.confirm.html', + controller: 'CaseDeleteModalCtrl', + resolve: { + caze: function () { + return $scope.caze; + } + } + }); + + modalInstance.result.then(function () { + $state.go('app.cases'); + }) + .catch(function (err) { + if (err && !_.isString(err)) { + NotificationSrv.error('caseDetails', err.data, err.status); + } + }); }; - $scope.getCaseResponders = function(force) { - if(!force && $scope.caseResponders !== null) { - return; + $scope.getCaseResponders = function (force) { + if (!force && $scope.caseResponders !== null) { + return; } $scope.caseResponders = null; CortexSrv.getResponders('case', $scope.caseId) - .then(function(responders){ + .then(function (responders) { $scope.caseResponders = responders; return CortexSrv.promntForResponder(responders); }) - .then(function(response) { - if(response && _.isString(response)) { + .then(function (response) { + if (response && _.isString(response)) { NotificationSrv.log(response, 'warning'); } else { return CortexSrv.runResponder(response.id, response.name, 'case', _.pick($scope.caze, '_id', 'tlp', 'pap')); } }) - .then(function(response){ + .then(function (response) { NotificationSrv.log(['Responder', response.data.responderName, 'started successfully on case', $scope.caze.title].join(' '), 'success'); }) - .catch(function(err) { - if(err && !_.isString(err)) { + .catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('caseDetails', err.data, err.status); } }); @@ -365,21 +366,21 @@ * A workaround filter to make sure the ngRepeat doesn't order the * object keys */ - $scope.notSorted = function(obj) { + $scope.notSorted = function (obj) { if (!obj) { return []; } return Object.keys(obj); }; - $scope.keys = function(obj) { + $scope.keys = function (obj) { return _.keys(obj); }; - $scope.getTags = function(selection) { + $scope.getTags = function (selection) { var tags = []; - angular.forEach(selection, function(tag) { + angular.forEach(selection, function (tag) { tags.push(tag.text); }); diff --git a/frontend/app/scripts/filters/limited-count.js b/frontend/app/scripts/filters/limited-count.js new file mode 100644 index 0000000000..1959e9f48d --- /dev/null +++ b/frontend/app/scripts/filters/limited-count.js @@ -0,0 +1,16 @@ +(function () { + 'use strict'; + + angular.module('theHiveFilters') + .filter('limitedCount', function () { + return function (count) { + if (isNaN(count)) + return 0 + + if (count < 0) + return (-1 * count) + '+'; + + return count; + }; + }); +})(); diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html index 537f8df2b1..e8fda587f3 100644 --- a/frontend/app/views/partials/alert/list.html +++ b/frontend/app/views/partials/alert/list.html @@ -2,7 +2,7 @@
    -

    List of alerts ({{$vm.list.total || 0}} of {{$vm.alertCountAll}})

    +

    List of alerts ({{$vm.list.total | limitedCount}} of {{$vm.alertCountAll}})

    @@ -27,7 +27,7 @@

    List of alerts ({{$vm.list.total || 0}} of {{$vm.alertCoun
    No records

    -
    +
    diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 148fa13a3e..1267e5d861 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -2,7 +2,8 @@
    -

    List of cases ({{$vm.list.values.length || 0}} of {{$vm.list.total}})

    + +

    List of alerts ({{$vm.list.total || 0}} of {{$vm.caseCountAll}})

    @@ -244,8 +245,8 @@

    List of cases ({{$vm.list.values.length || 0}} of {{$vm.li Tasks - {{currentCase.extraData.taskStats.total || - 0}} + {{currentCase.extraData.taskStats.total | + limitedCount}}

    List of cases ({{$vm.list.values.length || 0}} of {{$vm.li Observables - - {{-currentCase.extraData.observableStats.total + '+'}} - - - {{currentCase.extraData.observableStats.total || 0}} - + {{-currentCase.extraData.observableStats.total | + limitedCount}}
    From d34b08398b59e2fad0c2b0274c755c20dda01039 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 18:07:29 +0200 Subject: [PATCH 32/63] #1948 WIP: Use limited case in cases list --- frontend/app/scripts/controllers/case/CaseListCtrl.js | 1 + frontend/app/views/partials/case/case.list.html | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseListCtrl.js b/frontend/app/scripts/controllers/case/CaseListCtrl.js index fe6a49846e..dd94c272f9 100644 --- a/frontend/app/scripts/controllers/case/CaseListCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseListCtrl.js @@ -77,6 +77,7 @@ scope: $scope, sort: self.filtering.context.sort, loadAll: false, + limitedCount: true, pageSize: self.filtering.context.pageSize, filter: this.filtering.buildQuery(), operations: [ diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 1267e5d861..8a11563592 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -3,7 +3,7 @@
    -

    List of alerts ({{$vm.list.total || 0}} of {{$vm.caseCountAll}})

    +

    List of alerts ({{$vm.list.total | limitedCount}} of {{$vm.caseCountAll}})

    @@ -34,7 +34,7 @@

    List of alerts ({{$vm.list.total || 0}} of {{$vm.caseCount
    No records

    -
    +
    From 9b293588f92a7ade66f6535dcb2dddcda31cfb4b Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 18:30:56 +0200 Subject: [PATCH 33/63] #1946 Add countImportedAlert and ignore canceled task in countTask --- thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala | 7 +++++++ thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 90d7462dc3..eb1a15a69b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -66,6 +66,13 @@ class AlertCtrl @Inject() ( (graph, authContext) => graph.indexCountQuery(s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND v.read:false""") ), + Query.init[Long]( + "countImportedAlert", + (graph, authContext) => + graph.indexCountQuery( + s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND NOT v.caseId:[* TO 'ZZZZZZZZ']""" + ) + ), Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables), Query[Traversal.V[Alert], Traversal.V[Case]]("case", (alertSteps, _) => alertSteps.`case`), Query.withParam[Option[InputQuery[Traversal.Unk, Traversal.Unk]], Traversal.V[Alert], Traversal[ diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index c99aa1f038..08902e7c12 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -57,7 +57,10 @@ class TaskCtrl @Inject() ( "countTask", (inCase, graph, authContext) => graph.indexCountQuery( - s"""v."_label":Task AND relatedId:${inCase.caseId.value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" + s"""v."_label":Task AND """ + + "v.relatedId:${inCase.caseId.value} AND " + + "v.organisationIds:${organisationSrv.currentId(graph, authContext).value} AND " + + "NOT v.status:Cancel" ) ), Query.init[Traversal.V[Task]]( From 50386b207657ce79b9accf65271fa79c56a95339 Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 18:36:38 +0200 Subject: [PATCH 34/63] #1946 Add task status in describe API --- .../app/org/thp/thehive/controllers/v1/DescribeCtrl.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala index d89cac4343..c5c6ebf113 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/DescribeCtrl.scala @@ -163,6 +163,12 @@ class DescribeCtrl @Inject() ( case ("case", "resolutionStatus") => Some(Seq(resolutionStatus)) case ("dashboard", "status") => Some(Seq(PropertyDescription("status", "enumeration", Seq(JsString("Shared"), JsString("Private"), JsString("Deleted"))))) + case ("task", "status") => + Some( + Seq( + PropertyDescription("status", "enumeration", Seq(JsString("Waiting"), JsString("InProgress"), JsString("Completed"), JsString("Cancel"))) + ) + ) case (_, "tlp") => Some( Seq(PropertyDescription("tlp", "number", Seq(JsNumber(0), JsNumber(1), JsNumber(2), JsNumber(3)), Seq("white", "green", "amber", "red"))) From 213e3037001661060eeacbd8f685374156dc7006 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 18:36:50 +0200 Subject: [PATCH 35/63] #1948 WIP: Use limited count in case details (tasks, and observables) --- .../scripts/controllers/case/CaseMainCtrl.js | 2 +- .../controllers/case/CaseObservablesCtrl.js | 167 +++++++++--------- frontend/app/views/app.case.html | 12 +- .../app/views/partials/case/case.list.html | 5 +- .../app/views/partials/case/case.tasks.html | 2 +- .../observables/list/observables.html | 88 +++++---- 6 files changed, 151 insertions(+), 125 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseMainCtrl.js b/frontend/app/scripts/controllers/case/CaseMainCtrl.js index 1436efc0bb..de007c266a 100644 --- a/frontend/app/scripts/controllers/case/CaseMainCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseMainCtrl.js @@ -122,7 +122,7 @@ }, guard: UtilsSrv.hasAddDeleteEvents, onUpdate: function (updates) { - $scope.observableCount = updates; + $scope.observablesCount = updates; } }); diff --git a/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js b/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js index 4a59571002..95d3b631fb 100644 --- a/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseObservablesCtrl.js @@ -17,7 +17,7 @@ selectAll: false }; - this.$onInit = function() { + this.$onInit = function () { $scope.filtering = new FilteringSrv('observable', 'observable.list', { version: 'v1', defaults: { @@ -30,7 +30,7 @@ }); $scope.filtering.initContext($scope.caseId) - .then(function() { + .then(function () { $scope.load(); $scope.initAnalyzersList(); @@ -40,19 +40,19 @@ scope: $scope, rootId: $scope.caseId, objectType: 'case_artifact_job', - callback: function(data) { + callback: function (data) { var successFound = false; var i = 0; var ln = data.length; - while(!successFound && i < ln) { - if(data[i].base.operation === 'Update' && data[i].base.details.status === 'Success') { + while (!successFound && i < ln) { + if (data[i].base.operation === 'Update' && data[i].base.details.status === 'Success') { successFound = true; } i++; } - if(successFound) { + if (successFound) { $scope.artifacts.update(); } } @@ -64,7 +64,7 @@ }); }; - $scope.load = function() { + $scope.load = function () { $scope.artifacts = new PaginatedQuerySrv({ name: 'observables', root: $scope.caseId, @@ -72,22 +72,23 @@ streamObjectType: 'case_artifact', version: 'v1', scope: $scope, + limitedCount: true, sort: $scope.filtering.context.sort, pageSize: $scope.filtering.context.pageSize, filter: $scope.filtering.buildQuery(), extraData: ['seen', 'permissions', 'shareCount'], // extraData: ['seen', 'permissions', 'isOwner', 'shareCount'], operations: [ - {'_name': 'getCase', 'idOrName': $scope.caseId}, - {'_name': 'observables'} + { '_name': 'getCase', 'idOrName': $scope.caseId }, + { '_name': 'observables' } ], - onUpdate: function() { + onUpdate: function () { $scope.resetSelection(); } }); }; - $scope.resetSelection = function() { + $scope.resetSelection = function () { if ($scope.menu.selectAll) { $scope.selectAll(); } else { @@ -97,15 +98,15 @@ } }; - $scope.sortByField = function(field) { + $scope.sortByField = function (field) { var context = this.filtering.context; var currentSort = Array.isArray(context.sort) ? context.sort[0] : context.sort; var sort = null; - if(currentSort.substr(1) !== field) { + if (currentSort.substr(1) !== field) { sort = ['+' + field]; } else { - sort = [(currentSort === '+' + field) ? '-'+field : '+'+field]; + sort = [(currentSort === '+' + field) ? '-' + field : '+' + field]; } $scope.artifacts.sort = sort; @@ -113,7 +114,7 @@ $scope.filtering.setSort(sort); }; - $scope.keys = function(obj) { + $scope.keys = function (obj) { return _.keys(obj || {}); }; @@ -149,13 +150,13 @@ $scope.search(); }; - $scope.filterByTlp = function(value) { + $scope.filterByTlp = function (value) { $scope.addFilterValue('tlp', value); }; // *************************************************** - $scope.countReports = function(observable) { + $scope.countReports = function (observable) { return _.keys(observable.reports).length; }; @@ -174,7 +175,7 @@ // init lists // $scope.initAnalyzersList = function () { - if($scope.analysisEnabled) { + if ($scope.analysisEnabled) { AnalyzerSrv.query() .then(function (analyzers) { $scope.analyzersList.analyzers = analyzers; @@ -186,14 +187,14 @@ $scope.selectAll = function () { var selected = $scope.menu.selectAll; - _.each($scope.artifacts.values, function(item) { - if(SecuritySrv.checkPermissions(['manageObservable'], item.extraData.permissions)) { + _.each($scope.artifacts.values, function (item) { + if (SecuritySrv.checkPermissions(['manageObservable'], item.extraData.permissions)) { item.selected = selected; } }); if (selected) { - $scope.selection.artifacts = _.filter($scope.artifacts.values, function(item) { + $scope.selection.artifacts = _.filter($scope.artifacts.values, function (item) { return !!item.selected; }); } else { @@ -206,7 +207,7 @@ if (artifact.selected) { $scope.selection.artifacts.push(artifact); } else { - $scope.selection.artifacts = _.reject($scope.selection.artifacts, function(item) { + $scope.selection.artifacts = _.reject($scope.selection.artifacts, function (item) { return item._id === artifact._id; }); } @@ -220,17 +221,17 @@ controller: 'ObservableCreationCtrl', size: 'lg', resolve: { - params: function() { + params: function () { return null; }, - tags: function() { + tags: function () { return []; } } }); }; - $scope.bulkEdit = function() { + $scope.bulkEdit = function () { var modal = $uibModal.open({ animation: 'true', templateUrl: 'views/partials/observables/observable.update.html', @@ -238,22 +239,22 @@ controllerAs: '$dialog', size: 'lg', resolve: { - selection: function() { + selection: function () { return $scope.selection.artifacts; } } }); - modal.result.then(function(operations) { - $q.all(_.map(operations, function(operation) { + modal.result.then(function (operations) { + $q.all(_.map(operations, function (operation) { return CaseArtifactSrv.bulkUpdate(operation.ids, operation.patch); - })).then(function(/*responses*/) { + })).then(function (/*responses*/) { NotificationSrv.log('Selected observables have been updated successfully', 'success'); }); }); }; - $scope.bulkAnalyze = function() { + $scope.bulkAnalyze = function () { var modal = $uibModal.open({ animation: 'true', templateUrl: 'views/partials/observables/observable.analyze.html', @@ -261,22 +262,22 @@ controllerAs: '$dialog', size: 'lg', resolve: { - selection: function() { + selection: function () { return $scope.selection.artifacts; }, - analyzers: function() { + analyzers: function () { return $scope.analyzersList.analyzers; } } }); - modal.result.then(function(operations) { + modal.result.then(function (operations) { var analyzerNames = _.uniq(_.pluck(operations, 'analyzerName')); CortexSrv.getServers(analyzerNames) - .then(function(serverId) { + .then(function (serverId) { return $q.all( - _.map(operations, function(item) { + _.map(operations, function (item) { return CortexSrv.createJob({ cortexId: serverId, artifactId: item.observableId, @@ -285,19 +286,19 @@ }) ); }) - .then(function() { + .then(function () { NotificationSrv.log('Analyzers have been successfully started for the selected observables', 'success'); - }, function() { + }, function () { }); }); }; - $scope.showExport = function() { + $scope.showExport = function () { $scope.showExportPanel = true; }; - $scope.hideExport = function() { + $scope.hideExport = function () { $scope.showExportPanel = false; }; @@ -306,18 +307,18 @@ ModalUtilsSrv.confirm('Remove Observables', 'Are you sure you want to delete the selected Observables?', { okText: 'Yes, remove them', flavor: 'danger' - }).then(function() { + }).then(function () { - $q.all(_.map($scope.selection.artifacts, function(observable) { + $q.all(_.map($scope.selection.artifacts, function (observable) { return CaseArtifactSrv.api().delete({ artifactId: observable._id }, function () { $scope.$emit('observables:observable-removed', observable); }).$promise; })); - }).then(function(/*responses*/) { + }).then(function (/*responses*/) { NotificationSrv.log('The selected observables have been deleted', 'success'); - }).catch(function(/*err*/) { + }).catch(function (/*err*/) { //NotificationSrv.error('Observable deletion', response.data, response.status); }); }; @@ -341,9 +342,9 @@ var analyzerIds = _.uniq(_.pluck(toRun, 'analyzerId')); CortexSrv.getServers(analyzerIds) - .then(function(serverId) { + .then(function (serverId) { return $q.all( - _.map(toRun, function(item) { + _.map(toRun, function (item) { return CortexSrv.createJob({ cortexId: serverId, artifactId: item.artifact._id, @@ -352,9 +353,9 @@ }) ); }) - .then(function() { + .then(function () { NotificationSrv.log('Analyzers have been successfully started for ' + nbArtifacts + ' observables', 'success'); - }, function() { + }, function () { }); }; @@ -365,10 +366,10 @@ }); }; - $scope.showReport = function(observable, analyzerId) { + $scope.showReport = function (observable, analyzerId) { CortexSrv.getJobs($scope.caseId, observable._id, analyzerId, 1) - .then(function(response){ - if(!response.data || response.data.length !== 1) { + .then(function (response) { + if (!response.data || response.data.length !== 1) { return; } @@ -388,56 +389,56 @@ controllerAs: '$vm', size: 'max', resolve: { - report: function() { + report: function () { return report; }, - observable: function() { + observable: function () { return observable; } } }); }) - .catch(function(/*err*/) { + .catch(function (/*err*/) { NotificationSrv.error('Unable to fetch the analysis report'); }); }; - $scope.getObsResponders = function(observable, force) { - if(!force && $scope.obsResponders !== null) { - return; + $scope.getObsResponders = function (observable, force) { + if (!force && $scope.obsResponders !== null) { + return; } $scope.obsResponders = null; CortexSrv.getResponders('case_artifact', observable._id) - .then(function(responders) { - $scope.obsResponders = responders; - return CortexSrv.promntForResponder(responders); - }) - .then(function(response) { - if(response && _.isString(response)) { - NotificationSrv.log(response, 'warning'); - } else { - return CortexSrv.runResponder(response.id, response.name, 'case_artifact', _.pick(observable, '_id')); - } - }) - .then(function(response){ - var data = '['+$filter('fang')(observable.data || observable.attachment.name)+']'; - NotificationSrv.log(['Responder', response.data.responderName, 'started successfully on observable', data].join(' '), 'success'); - }) - .catch(function(err) { - if(err && !_.isString(err)) { - NotificationSrv.error('observablesList', err.data, err.status); - } - }); + .then(function (responders) { + $scope.obsResponders = responders; + return CortexSrv.promntForResponder(responders); + }) + .then(function (response) { + if (response && _.isString(response)) { + NotificationSrv.log(response, 'warning'); + } else { + return CortexSrv.runResponder(response.id, response.name, 'case_artifact', _.pick(observable, '_id')); + } + }) + .then(function (response) { + var data = '[' + $filter('fang')(observable.data || observable.attachment.name) + ']'; + NotificationSrv.log(['Responder', response.data.responderName, 'started successfully on observable', data].join(' '), 'success'); + }) + .catch(function (err) { + if (err && !_.isString(err)) { + NotificationSrv.error('observablesList', err.data, err.status); + } + }); }; } ) - .controller('JobReportModalCtrl', function($uibModalInstance, report, observable) { - this.report = report; - this.observable = observable; - this.close = function() { - $uibModalInstance.dismiss(); - }; - }); + .controller('JobReportModalCtrl', function ($uibModalInstance, report, observable) { + this.report = report; + this.observable = observable; + this.close = function () { + $uibModalInstance.dismiss(); + }; + }); })(); diff --git a/frontend/app/views/app.case.html b/frontend/app/views/app.case.html index e548b9ce1f..cd6114141b 100644 --- a/frontend/app/views/app.case.html +++ b/frontend/app/views/app.case.html @@ -20,21 +20,22 @@    Observables   - {{observableCount}} + {{observablesCount}}    TTPs   - -   {{tab.label | limitTo:25}}   +   {{tab.label | + limitTo:25}}   -   {{tab.label | fang | limitTo:25}}   + +   {{tab.label | fang | limitTo:25}}     {{tab.label | limitTo:25}}   @@ -42,7 +43,8 @@ - + diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 8a11563592..46ecede4b0 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -2,7 +2,6 @@
    -

    List of alerts ({{$vm.list.total | limitedCount}} of {{$vm.caseCountAll}})

    @@ -256,8 +255,8 @@

    List of alerts ({{$vm.list.total | limitedCount}} of {{$vm Observables - {{-currentCase.extraData.observableStats.total | + {{currentCase.extraData.observableStats.total + | limitedCount}}

    diff --git a/frontend/app/views/partials/case/case.tasks.html b/frontend/app/views/partials/case/case.tasks.html index 220c573b4d..fb998beb75 100755 --- a/frontend/app/views/partials/case/case.tasks.html +++ b/frontend/app/views/partials/case/case.tasks.html @@ -8,7 +8,7 @@

    - Tasks List ({{list.values.length || 0}} of {{list.total}}) + List of tasks ({{list.total | limitedCount}} of {{tasksCount}})

    diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index 1c10ddcd8d..898fb6af3a 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -1,15 +1,14 @@ -

    - Observable List ({{artifacts.total || 0}} of {{observableCount}}) + List of observables ({{artifacts.total | limitedCount}} of {{observablesCount}}) + ({{selection.artifacts.length}} selected)

    -
    @@ -18,7 +17,7 @@

    -
    +
    @@ -26,13 +25,16 @@

    -
    - + Flags - + Type - + @@ -40,11 +42,12 @@

    		Value/Filename - + - - + - + + - + @@ -99,35 +112,46 @@

    - None + None
    - +

    		- + ({{artifact.extraData.shareCount || 0}}) - + - + From 9112cc3ed0fa524f4699887d4070fdc61c30fb91 Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 19:06:39 +0200 Subject: [PATCH 36/63] #1946 Fix countTask --- thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index 08902e7c12..7aeb37e41c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -58,8 +58,8 @@ class TaskCtrl @Inject() ( (inCase, graph, authContext) => graph.indexCountQuery( s"""v."_label":Task AND """ + - "v.relatedId:${inCase.caseId.value} AND " + - "v.organisationIds:${organisationSrv.currentId(graph, authContext).value} AND " + + s"v.relatedId:${inCase.caseId.value} AND " + + s"v.organisationIds:${organisationSrv.currentId(graph, authContext).value} AND " + "NOT v.status:Cancel" ) ), From d92e402ff7db09fdca1118aeb140dc1f1eb92359 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 19:09:18 +0200 Subject: [PATCH 37/63] #1948 WIP: Use countImportedAlert in case details page --- frontend/app/scripts/controllers/case/CaseMainCtrl.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseMainCtrl.js b/frontend/app/scripts/controllers/case/CaseMainCtrl.js index de007c266a..defd55aed9 100644 --- a/frontend/app/scripts/controllers/case/CaseMainCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseMainCtrl.js @@ -128,9 +128,7 @@ // Stats for case alerts counter StreamQuerySrv('v1', [ - { _name: 'getCase', idOrName: caseId }, - { _name: 'alerts' }, - { _name: 'count' } + { _name: 'countImportedAlert', caseId: caseId }, ], { scope: $scope, rootId: caseId, From ac538fe5c9b8f4eb13ea606672faf22bf3839c72 Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 19:18:31 +0200 Subject: [PATCH 38/63] #1946 Improve performance of case->alert query --- thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index 7fb243c841..e88f443fdd 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -34,6 +34,7 @@ class CaseCtrl @Inject() ( userSrv: UserSrv, taskSrv: TaskSrv, organisationSrv: OrganisationSrv, + alertSrv: AlertSrv, db: Database, appConfig: ApplicationConfig ) extends QueryableCtrl @@ -80,7 +81,12 @@ class CaseCtrl @Inject() ( ), Query[Traversal.V[Case], Traversal.V[User]]("assignableUsers", (caseSteps, authContext) => caseSteps.assignableUsers(authContext)), Query[Traversal.V[Case], Traversal.V[Organisation]]("organisations", (caseSteps, authContext) => caseSteps.organisations.visible(authContext)), - Query[Traversal.V[Case], Traversal.V[Alert]]("alerts", (caseSteps, authContext) => caseSteps.alert.visible(organisationSrv)(authContext)), + Query[Traversal.V[Case], Traversal.V[Alert]]( + "alerts", + (caseSteps, authContext) => +// caseSteps.alert.visible(organisationSrv)(authContext) + alertSrv.startTraversal(caseSteps.graph).has(_.caseId, P.within(caseSteps._id.toSeq: _*)).visible(organisationSrv)(authContext) + ), Query[Traversal.V[Case], Traversal.V[Share]]("shares", (caseSteps, authContext) => caseSteps.shares.visible(authContext)), Query[Traversal.V[Case], Traversal.V[Procedure]]("procedures", (caseSteps, _) => caseSteps.procedure) ) From aaed1114c88a85b68f80edfd844effab8f537465 Mon Sep 17 00:00:00 2001 From: To-om Date: Sat, 10 Apr 2021 19:41:57 +0200 Subject: [PATCH 39/63] #1946 Apply limitedCount in tag usage --- thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala | 8 +++++++- .../app/org/thp/thehive/controllers/v1/TagRenderer.scala | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala index e35ef1545f..5f0fa4698b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TagCtrl.scala @@ -6,6 +6,7 @@ import org.thp.scalligraph.EntityIdOrName import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser} import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.query._ +import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, IteratorOutput, Traversal} import org.thp.scalligraph.utils.FunctionalCondition.When @@ -24,9 +25,14 @@ class TagCtrl @Inject() ( db: Database, tagSrv: TagSrv, val organisationSrv: OrganisationSrv, - properties: Properties + properties: Properties, + appConfig: ApplicationConfig ) extends QueryableCtrl with TagRenderer { + + val limitedCountThresholdConfig: ConfigItem[Long, Long] = appConfig.item[Long]("query.limitedCountThreshold", "Maximum number returned by a count") + val limitedCountThreshold: Long = limitedCountThresholdConfig.get + override val entityName: String = "Tag" override val publicProperties: PublicProperties = properties.tag override val initialQuery: Query = diff --git a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala index 3b665045c3..489887c737 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala @@ -10,15 +10,16 @@ import play.api.libs.json._ import java.util.{Map => JMap} trait TagRenderer extends BaseRenderer[Tag] { + val limitedCountThreshold: Long def usageStats(implicit authContext: AuthContext ): Traversal.V[Tag] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = _.project( - _.by(_.`case`.count) - .by(_.alert.count) - .by(_.observable.count) - .by(_.caseTemplate.count) + _.by(_.`case`.limitedCount(limitedCountThreshold)) + .by(_.alert.limitedCount(limitedCountThreshold)) + .by(_.observable.limitedCount(limitedCountThreshold)) + .by(_.caseTemplate.limitedCount(limitedCountThreshold)) ).domainMap { case (caseCount, alertCount, observableCount, caseTemplateCount) => Json.obj( From 2b081dac39368a3d84cda90c3b5c9ab2933f2c16 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 22:49:11 +0200 Subject: [PATCH 40/63] #1948 WIP: Use limited count in custom tags admin page --- .../organisation/OrgCustomTagsListCmp.js | 15 +++- .../components/org/custom-tags/tag-list.html | 75 ++++++++++++------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js b/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js index a61b8e5ca2..17c35a4418 100644 --- a/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js +++ b/frontend/app/scripts/components/organisation/OrgCustomTagsListCmp.js @@ -3,7 +3,7 @@ angular.module('theHiveComponents') .component('orgCustomTagsList', { - controller: function ($scope, PaginatedQuerySrv, FilteringSrv, TaxonomyCacheSrv, TagSrv, UserSrv, ModalUtilsSrv, NotificationSrv) { + controller: function ($scope, PaginatedQuerySrv, QuerySrv, FilteringSrv, TaxonomyCacheSrv, TagSrv, UserSrv, ModalUtilsSrv, NotificationSrv) { var self = this; self.tags = []; @@ -30,6 +30,18 @@ self.filtering.setPageSize(newValue); }); }); + + QuerySrv.query( + 'v1', + [{ '_name': 'countFreetags' }], + { + params: { + name: 'all-custom-tags.count' + } + }) + .then(function (response) { + self.freetagsCount = response.data; + }); }; this.load = function () { @@ -40,6 +52,7 @@ skipStream: true, sort: self.filtering.context.sort, loadAll: false, + limitedCount: true, pageSize: self.filtering.context.pageSize, filter: this.filtering.buildQuery(), operations: [ diff --git a/frontend/app/views/components/org/custom-tags/tag-list.html b/frontend/app/views/components/org/custom-tags/tag-list.html index e9f81371e1..481d50066b 100644 --- a/frontend/app/views/components/org/custom-tags/tag-list.html +++ b/frontend/app/views/components/org/custom-tags/tag-list.html @@ -1,18 +1,18 @@
    -
    +

    - Custom tags List ({{$vm.list.values.length || 0}} of {{$vm.list.total}}) + List of custom tags ({{$vm.list.total | limitedCount}} of {{$vm.freetagsCount}})

    -
    @@ -23,19 +23,22 @@

    No records
    -
    +
    @@ -69,24 +80,30 @@

    - - - - + + + +
    - - Name - - - - + + Name + + + + Colour @@ -48,17 +51,25 @@

    		Dates - + C. - - - + + + - + U. - - - + + + Actions
    - + - + + {{tag.extraData.usage.case}}{{tag.extraData.usage.alert}}{{tag.extraData.usage.observable}}{{tag.extraData.usage.caseTemplate}}{{tag.extraData.usage.case | limitedCount}}{{tag.extraData.usage.alert | limitedCount}}{{tag.extraData.usage.observable | limitedCount}}{{tag.extraData.usage.caseTemplate | limitedCount}} - From 9fe06fb667053a8a3a6804d96eb962c3ab7890a5 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sat, 10 Apr 2021 22:49:31 +0200 Subject: [PATCH 41/63] Update .gitignore and logback.xml --- .gitignore | 2 ++ conf/logback.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d78ccf369e..2c87b915d9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ tmp .bloop/ .metals/ metals.sbt + +.jvmopts diff --git a/conf/logback.xml b/conf/logback.xml index e72d600cf0..208d14f63e 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -38,7 +38,7 @@ - + From 3df647a923f0d1db4aa4c0daf2c58ea4f4066daa Mon Sep 17 00:00:00 2001 From: To-om Date: Sun, 11 Apr 2021 08:18:05 +0200 Subject: [PATCH 42/63] #1946 Use indexQuery for tag usage and observable count in case extraData --- .../thp/thehive/controllers/v1/CaseCtrl.scala | 2 +- .../thehive/controllers/v1/CaseRenderer.scala | 14 +++--- .../thehive/controllers/v1/TagRenderer.scala | 44 +++++++++++++------ 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index e88f443fdd..fc8c340add 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -65,7 +65,7 @@ class CaseCtrl @Inject() ( Query.init[Long]( "countCase", (graph, authContext) => - graph.indexCountQuery(s"""v."_label":Case AND v.organisationIds:${organisationSrv.currentId(graph, authContext).value} """) + graph.indexCountQuery(s"""v."_label":Case AND v.organisationIds:${organisationSrv.currentId(graph, authContext).value}""") ), Query[Traversal.V[Case], Traversal.V[Observable]]( "observables", diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala index 1fb00388e0..def02c9dc0 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala @@ -17,11 +17,15 @@ import java.util.{Collection => JCollection, List => JList, Map => JMap} trait CaseRenderer extends BaseRenderer[Case] { val limitedCountThreshold: Long - def observableStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsValue, JLong, Converter[JsValue, JLong]] = - _.share - .observables - .limitedCount(limitedCountThreshold) - .domainMap(count => Json.obj("total" -> count)) + def observableStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsObject, AnyRef, Converter[JsObject, AnyRef]] = + t => + t._id.domainMap { caseId => + Json.obj("total" -> t.graph.indexCountQuery(s"""v."_label":Case AND v.relatedId:${caseId.value}""")) + } +// _.share +// .observables +// .limitedCount(limitedCountThreshold) +// .domainMap(count => Json.obj("total" -> count)) def taskStats(implicit authContext: AuthContext diff --git a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala index 489887c737..31246eb65a 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala @@ -15,20 +15,36 @@ trait TagRenderer extends BaseRenderer[Tag] { def usageStats(implicit authContext: AuthContext ): Traversal.V[Tag] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = - _.project( - _.by(_.`case`.limitedCount(limitedCountThreshold)) - .by(_.alert.limitedCount(limitedCountThreshold)) - .by(_.observable.limitedCount(limitedCountThreshold)) - .by(_.caseTemplate.limitedCount(limitedCountThreshold)) - ).domainMap { - case (caseCount, alertCount, observableCount, caseTemplateCount) => - Json.obj( - "case" -> caseCount, - "alert" -> alertCount, - "observable" -> observableCount, - "caseTemplate" -> caseTemplateCount - ) - } + t => + t.project( + _.by(_.value(_.predicate)) + .by(_.caseTemplate.limitedCount(limitedCountThreshold)) + ).domainMap { + case (tag, caseTemplateCount) => + Json.obj( + "case" -> t.graph.indexCountQuery(s"""v."_label":Case AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}""""), + "alert" -> t.graph.indexCountQuery(s"""v."_label":Alert AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}""""), + "observable" -> t.graph.indexCountQuery(s"""v."_label":Observable AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}""""), + "caseTemplate" -> caseTemplateCount + ) + } +// def usageStats(implicit +// authContext: AuthContext +// ): Traversal.V[Tag] => Traversal[JsObject, JMap[String, Any], Converter[JsObject, JMap[String, Any]]] = +// _.project( +// _.by(_.`case`.limitedCount(limitedCountThreshold)) +// .by(_.alert.limitedCount(limitedCountThreshold)) +// .by(_.observable.limitedCount(limitedCountThreshold)) +// .by(_.caseTemplate.limitedCount(limitedCountThreshold)) +// ).domainMap { +// case (caseCount, alertCount, observableCount, caseTemplateCount) => +// Json.obj( +// "case" -> caseCount, +// "alert" -> alertCount, +// "observable" -> observableCount, +// "caseTemplate" -> caseTemplateCount +// ) +// } def tagStatsRenderer(extraData: Set[String])(implicit authContext: AuthContext From 6d88194b755f63bfde7097bb43f13cde17a8dfc4 Mon Sep 17 00:00:00 2001 From: To-om Date: Sun, 11 Apr 2021 09:43:12 +0200 Subject: [PATCH 43/63] #1946 Remove all obsolete index --- .../thp/thehive/models/TheHiveSchemaDefinition.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index ce482c23a6..0763d4397e 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -449,7 +449,6 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .removeIndex("Case", IndexType.basic, "status") .removeIndex("Task", IndexType.basic, "status") .removeIndex("Alert", IndexType.basic, "type", "source", "sourceRef") - //=====[release 4.1.3]===== .updateGraph("Set caseId in imported alerts", "Alert") { traversal => traversal .project( @@ -462,6 +461,15 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { } Success(()) } + .removeIndex("Alert", IndexType.standard) + .removeIndex("Attachment", IndexType.standard) + .removeIndex("Audit", IndexType.standard) + .removeIndex("Case", IndexType.standard) + .removeIndex("Log", IndexType.standard) + .removeIndex("Observable", IndexType.standard) + .removeIndex("Tag", IndexType.standard) + .removeIndex("Task", IndexType.standard) + //=====[release 4.1.3]===== val reflectionClasses = new Reflections( new ConfigurationBuilder() From 785a248ccc5da280dd75181c3cf170305f4a8b9f Mon Sep 17 00:00:00 2001 From: To-om Date: Sun, 11 Apr 2021 09:44:22 +0200 Subject: [PATCH 44/63] #1946 Filter tag usage by organisation --- .../thehive/controllers/v1/TagRenderer.scala | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala index 31246eb65a..4809deacbd 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala @@ -4,6 +4,7 @@ import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.traversal.TraversalOps._ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.models.Tag +import org.thp.thehive.services.OrganisationSrv import org.thp.thehive.services.TagOps._ import play.api.libs.json._ @@ -11,6 +12,7 @@ import java.util.{Map => JMap} trait TagRenderer extends BaseRenderer[Tag] { val limitedCountThreshold: Long + val organisationSrv: OrganisationSrv def usageStats(implicit authContext: AuthContext @@ -22,9 +24,24 @@ trait TagRenderer extends BaseRenderer[Tag] { ).domainMap { case (tag, caseTemplateCount) => Json.obj( - "case" -> t.graph.indexCountQuery(s"""v."_label":Case AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}""""), - "alert" -> t.graph.indexCountQuery(s"""v."_label":Alert AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}""""), - "observable" -> t.graph.indexCountQuery(s"""v."_label":Observable AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}""""), + "case" -> t + .graph + .indexCountQuery( + s"""v."_label":Case AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}" AND """ + + s"v.organisationIds:${organisationSrv.currentId(t.graph, authContext).value}" + ), + "alert" -> t + .graph + .indexCountQuery( + s"""v."_label":Alert AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}" AND """ + + s"v.organisationId:${organisationSrv.currentId(t.graph, authContext).value}" + ), + "observable" -> t + .graph + .indexCountQuery( + s"""v."_label":Observable AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}" AND """ + + s"v.organisationIds:${organisationSrv.currentId(t.graph, authContext).value}" + ), "caseTemplate" -> caseTemplateCount ) } From 8bb5b7c6052a80338e94acd4eabc99aebaca9110 Mon Sep 17 00:00:00 2001 From: To-om Date: Sun, 11 Apr 2021 18:07:19 +0200 Subject: [PATCH 45/63] #1946 Fix observable count in case extraData --- ScalliGraph | 2 +- conf/logback.xml | 1 + .../org/thp/thehive/controllers/v1/CaseCtrl.scala | 2 +- .../thp/thehive/controllers/v1/CaseRenderer.scala | 12 +++++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 6a1a6e77bf..b168949a6b 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 6a1a6e77bf75933aed4c48cdeaf426f984a43081 +Subproject commit b168949a6b3918306a206d554aa154023533ea14 diff --git a/conf/logback.xml b/conf/logback.xml index 208d14f63e..d8072baa3e 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -37,6 +37,7 @@ + diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala index fc8c340add..04064ecfaf 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseCtrl.scala @@ -33,7 +33,7 @@ class CaseCtrl @Inject() ( observableSrv: ObservableSrv, userSrv: UserSrv, taskSrv: TaskSrv, - organisationSrv: OrganisationSrv, + val organisationSrv: OrganisationSrv, alertSrv: AlertSrv, db: Database, appConfig: ApplicationConfig diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala index def02c9dc0..3b4325cccf 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala @@ -7,6 +7,7 @@ import org.thp.scalligraph.traversal.{Converter, Traversal} import org.thp.thehive.models.{Alert, AlertCase, Case} import org.thp.thehive.services.CaseOps._ import org.thp.thehive.services.OrganisationOps._ +import org.thp.thehive.services.OrganisationSrv import org.thp.thehive.services.ShareOps._ import org.thp.thehive.services.TaskOps._ import play.api.libs.json._ @@ -16,11 +17,20 @@ import java.util.{Collection => JCollection, List => JList, Map => JMap} trait CaseRenderer extends BaseRenderer[Case] { val limitedCountThreshold: Long + val organisationSrv: OrganisationSrv def observableStats(implicit authContext: AuthContext): Traversal.V[Case] => Traversal[JsObject, AnyRef, Converter[JsObject, AnyRef]] = t => t._id.domainMap { caseId => - Json.obj("total" -> t.graph.indexCountQuery(s"""v."_label":Case AND v.relatedId:${caseId.value}""")) + Json.obj( + "total" -> t + .graph + .indexCountQuery( + s"""v."_label":Observable AND """ + + s"v.relatedId:${caseId.value} AND " + + s"v.organisationIds:${organisationSrv.currentId(t.graph, authContext).value}" + ) + ) } // _.share // .observables From ba515db7dfe8b4d7b0d9032743f9401c5e62c1aa Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 11 Apr 2021 18:54:52 +0200 Subject: [PATCH 46/63] #1948 WIP: Add datalist-header component and use it in cases, alert, task and observables lists --- frontend/app/index.html | 1 + .../common/datalist-header.component.js | 15 +++++++ frontend/app/scripts/services/api/TagSrv.js | 2 +- .../services/common/data/PaginatedQuerySrv.js | 45 ++++++++++++------- .../common/datalist-header.component.html | 11 +++++ .../components/org/custom-tags/tag-list.html | 7 +-- frontend/app/views/directives/psearch.html | 16 +++++-- frontend/app/views/partials/alert/list.html | 15 ++++--- .../app/views/partials/case/case.list.html | 8 ++-- .../app/views/partials/case/case.tasks.html | 8 ++-- .../observables/list/observables.html | 7 ++- 11 files changed, 95 insertions(+), 40 deletions(-) create mode 100644 frontend/app/scripts/components/common/datalist-header.component.js create mode 100644 frontend/app/views/components/common/datalist-header.component.html diff --git a/frontend/app/index.html b/frontend/app/index.html index fd8df0c4c8..669cc171ca 100644 --- a/frontend/app/index.html +++ b/frontend/app/index.html @@ -140,6 +140,7 @@ + diff --git a/frontend/app/scripts/components/common/datalist-header.component.js b/frontend/app/scripts/components/common/datalist-header.component.js new file mode 100644 index 0000000000..3376537620 --- /dev/null +++ b/frontend/app/scripts/components/common/datalist-header.component.js @@ -0,0 +1,15 @@ +(function () { + 'use strict'; + + angular.module('theHiveComponents') + .component('datalistHeader', { + controller: function () { }, + controllerAs: '$ctrl', + templateUrl: 'views/components/common/datalist-header.component.html', + bindings: { + title: '@', + list: '<', + total: '<' + } + }); +})(); diff --git a/frontend/app/scripts/services/api/TagSrv.js b/frontend/app/scripts/services/api/TagSrv.js index 23918abc87..abf893ae77 100644 --- a/frontend/app/scripts/services/api/TagSrv.js +++ b/frontend/app/scripts/services/api/TagSrv.js @@ -21,7 +21,7 @@ { _name: 'filter', _ne: { '_field': 'colour', '_value': defaultColour } } ], { params: { - name: 'list-tags' + name: 'freetags-cache' } }) }) diff --git a/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js b/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js index 827c058de3..0a863d8fe5 100644 --- a/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js +++ b/frontend/app/scripts/services/common/data/PaginatedQuerySrv.js @@ -35,6 +35,8 @@ this.extraData = options.extraData || undefined; this.name = options.name || undefined; this.config = options.config || {}; + this.loading = false; + this.loadingCount = false; this.operations = options.operations; @@ -134,6 +136,7 @@ var filters = self.getFilter(); // Get the list + this.loading = true; QuerySrv.call(this.version, this.operations, { filter: filters, sort: self.getSort(), @@ -141,24 +144,29 @@ config: self.config, withParent: false, name: self.name - }).then(function (data) { - if (self.loadAll) { - self.allValues = data; - - self.total = data.length; - - self.changePage(); - } else { - self.values = data; - if (angular.isFunction(self.onUpdate)) { - self.onUpdate(updates); + }) + .then(function (data) { + if (self.loadAll) { + self.allValues = data; + + self.total = data.length; + + self.changePage(); + } else { + self.values = data; + if (angular.isFunction(self.onUpdate)) { + self.onUpdate(updates); + } } - } - }).catch(function (err) { - if (self.onFailure) { - self.onFailure(err); - } - }); + }) + .catch(function (err) { + if (self.onFailure) { + self.onFailure(err); + } + }) + .finally(function () { + self.loading = false; + }); // get the total if not cached var hash = $filter('md5')(JSON.stringify(this.filter)); @@ -166,6 +174,7 @@ this.filterHash = hash; // Compute the total again + self.loadingCount = true; QuerySrv.count('v1', this.operations, { filter: filters, name: self.name, @@ -173,6 +182,8 @@ limitedCount: self.limitedCount }).then(function (total) { self.total = total; + }).finally(function () { + self.loadingCount = false; }); } diff --git a/frontend/app/views/components/common/datalist-header.component.html b/frontend/app/views/components/common/datalist-header.component.html new file mode 100644 index 0000000000..dcd0e59398 --- /dev/null +++ b/frontend/app/views/components/common/datalist-header.component.html @@ -0,0 +1,11 @@ + + {{$ctrl.title}} + ({{$ctrl.list.total | limitedCount}} + ( + of {{$ctrl.total}}) + of ) + Fetching + data... + Counting records... + diff --git a/frontend/app/views/components/org/custom-tags/tag-list.html b/frontend/app/views/components/org/custom-tags/tag-list.html index 481d50066b..f94d46638d 100644 --- a/frontend/app/views/components/org/custom-tags/tag-list.html +++ b/frontend/app/views/components/org/custom-tags/tag-list.html @@ -8,7 +8,8 @@

    - List of custom tags ({{$vm.list.total | limitedCount}} of {{$vm.freetagsCount}}) + +

    @@ -19,11 +20,11 @@

    -
    +
    No records
    -
    +
    diff --git a/frontend/app/views/directives/psearch.html b/frontend/app/views/directives/psearch.html index 03c0512f66..dbcbeb9cd9 100644 --- a/frontend/app/views/directives/psearch.html +++ b/frontend/app/views/directives/psearch.html @@ -1,3 +1,13 @@ -
    -
      -
      +
      +
      +
        +
      +
      + +
      +
        +
        +
        diff --git a/frontend/app/views/partials/alert/list.html b/frontend/app/views/partials/alert/list.html index e8fda587f3..553dc6c736 100644 --- a/frontend/app/views/partials/alert/list.html +++ b/frontend/app/views/partials/alert/list.html @@ -2,7 +2,9 @@
        -

        List of alerts ({{$vm.list.total | limitedCount}} of {{$vm.alertCountAll}})

        +

        + +

        @@ -23,11 +25,12 @@

        List of alerts ({{$vm.list.total | limitedCount}} of {{$vm
        -
        + +
        No records
        -
        +

        @@ -156,12 +159,14 @@

        List of alerts ({{$vm.list.total | limitedCount}} of {{$vm

        {{event.caseId ? + ng-click="$vm.addFilterValue('imported', !!event.caseId)">{{event.caseId + ? 'Imported' : 'New'}} {{event.read ? 'Read' : + ng-click="$vm.addFilterValue('read', !!event.read)">{{event.read ? + 'Read' : 'Unread'}} diff --git a/frontend/app/views/partials/case/case.list.html b/frontend/app/views/partials/case/case.list.html index 46ecede4b0..2c67c8ed7f 100644 --- a/frontend/app/views/partials/case/case.list.html +++ b/frontend/app/views/partials/case/case.list.html @@ -2,7 +2,9 @@
        -

        List of alerts ({{$vm.list.total | limitedCount}} of {{$vm.caseCountAll}})

        +

        + +

        @@ -29,11 +31,11 @@

        List of alerts ({{$vm.list.total | limitedCount}} of {{$vm

        -
        +
        No records
        -
        +
        diff --git a/frontend/app/views/partials/case/case.tasks.html b/frontend/app/views/partials/case/case.tasks.html index fb998beb75..99dae58ae0 100755 --- a/frontend/app/views/partials/case/case.tasks.html +++ b/frontend/app/views/partials/case/case.tasks.html @@ -8,7 +8,7 @@

        - List of tasks ({{list.total | limitedCount}} of {{tasksCount}}) +

        @@ -17,7 +17,7 @@

        -
        +
        No task found for this case.
        @@ -53,7 +53,7 @@

        -
        +
        @@ -191,7 +191,7 @@

        -
        +
        diff --git a/frontend/app/views/partials/observables/list/observables.html b/frontend/app/views/partials/observables/list/observables.html index 898fb6af3a..3dafdf264b 100644 --- a/frontend/app/views/partials/observables/list/observables.html +++ b/frontend/app/views/partials/observables/list/observables.html @@ -1,8 +1,7 @@

        - List of observables ({{artifacts.total | limitedCount}} of {{observablesCount}}) - + ({{selection.artifacts.length}} selected) @@ -12,12 +11,12 @@

        on-clear-all="clearFilters()">

        -
        +
        No records.
        -
        +
        From 2e6b6105278cd11f45bdc63fdc12f4ebfa3d9722 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Sun, 11 Apr 2021 22:27:03 +0200 Subject: [PATCH 47/63] #1948 WIP: Fix pagination in task list --- .../scripts/controllers/case/CaseTasksCtrl.js | 178 +++--- frontend/app/views/directives/psearch.html | 4 +- .../app/views/partials/case/case.tasks.html | 540 +++++++++--------- .../views/partials/case/tasks/toolbar.html | 16 +- 4 files changed, 377 insertions(+), 361 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseTasksCtrl.js b/frontend/app/scripts/controllers/case/CaseTasksCtrl.js index 039b1aafaa..14a370d284 100755 --- a/frontend/app/scripts/controllers/case/CaseTasksCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseTasksCtrl.js @@ -1,9 +1,9 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveControllers') .controller('CaseTasksCtrl', CaseTasksCtrl); - function CaseTasksCtrl($scope, $state, $stateParams, $q, $uibModal, AuthenticationSrv, ModalUtilsSrv, FilteringSrv, CaseTabsSrv, PaginatedQuerySrv, CaseTaskSrv, UserSrv, NotificationSrv, CortexSrv, AppLayoutSrv) { + function CaseTasksCtrl($scope, $state, $stateParams, $q, AuthenticationSrv, ModalUtilsSrv, FilteringSrv, CaseTabsSrv, PaginatedQuerySrv, CaseTaskSrv, NotificationSrv, CortexSrv, AppLayoutSrv) { CaseTabsSrv.activateTab($state.current.data.tab); @@ -23,7 +23,7 @@ selectAll: false }; - this.$onInit = function() { + this.$onInit = function () { $scope.filtering = new FilteringSrv('task', 'task.list', { version: 'v1', defaults: { @@ -36,53 +36,55 @@ }); $scope.filtering.initContext($scope.caseId) - .then(function() { + .then(function () { $scope.load(); - $scope.$watchCollection('tasks.pageSize', function (newValue) { + $scope.$watch('list.pageSize', function (newValue) { $scope.filtering.setPageSize(newValue); }); }); }; - $scope.getAssignableUsers = function(taskId) { + $scope.getAssignableUsers = function (taskId) { return [ - {_name: 'getTask', idOrName: taskId}, - {_name: 'assignableUsers'} + { _name: 'getTask', idOrName: taskId }, + { _name: 'assignableUsers' } ]; }; - $scope.load = function() { + $scope.load = function () { $scope.list = new PaginatedQuerySrv({ name: 'case-tasks', root: $scope.caseId, - objectType: 'case_task', + objectType: 'task', + streamObjectType: 'case_task', version: 'v1', scope: $scope, sort: $scope.filtering.context.sort, loadAll: false, + limitedCount: true, pageSize: $scope.filtering.context.pageSize, filter: $scope.filtering.buildQuery(), baseFilter: { - _not: { + _ne: { _field: 'status', _value: 'Cancel' } }, operations: [ - {'_name': 'getCase', "idOrName": $scope.caseId}, - {'_name': 'tasks'} + { '_name': 'getCase', "idOrName": $scope.caseId }, + { '_name': 'tasks' } ], extraData: ['shareCount', 'actionRequired'], //extraData: ['isOwner', 'shareCount'], - onUpdate: function() { + onUpdate: function () { $scope.buildTaskGroups($scope.list.values); $scope.resetSelection(); } }); }; - $scope.resetSelection = function() { + $scope.resetSelection = function () { if ($scope.menu.selectAll) { $scope.selectAll(); } else { @@ -92,7 +94,7 @@ } }; - $scope.updateMenu = function() { + $scope.updateMenu = function () { // Handle flag/unflag menu items var temp = _.uniq(_.pluck($scope.selection, 'flag')); $scope.menu.unflag = temp.length === 1 && temp[0] === true; @@ -113,28 +115,28 @@ $scope.menu.delete = $scope.selection.length > 0; }; - $scope.select = function(task) { + $scope.select = function (task) { if (task.selected) { $scope.selection.push(task); } else { - $scope.selection = _.reject($scope.selection, function(item) { + $scope.selection = _.reject($scope.selection, function (item) { return item._id === task._id; }); } $scope.updateMenu(); }; - $scope.selectAll = function() { + $scope.selectAll = function () { var selected = $scope.menu.selectAll; - _.each($scope.list.values, function(item) { + _.each($scope.list.values, function (item) { // if(SecuritySrv.checkPermissions(['manageCase'], item.extraData.permissions)) { item.selected = selected; // } }); if (selected) { - $scope.selection = _.filter($scope.list.values, function(item) { + $scope.selection = _.filter($scope.list.values, function (item) { return !!item.selected; }); } else { @@ -144,8 +146,6 @@ $scope.updateMenu(); }; - // ######################@@@ - $scope.toggleStats = function () { $scope.filtering.toggleStats(); }; @@ -177,16 +177,16 @@ $scope.search(); }; - $scope.filterBy = function(field, value) { + $scope.filterBy = function (field, value) { $scope.filtering.clearFilters() - .then(function() { + .then(function () { $scope.addFilterValue(field, value); }); }; - $scope.filterMyTasks = function() { + $scope.filterMyTasks = function () { $scope.filtering.clearFilters() - .then(function() { + .then(function () { var currentUser = AuthenticationSrv.currentUser; $scope.filtering.addFilter({ field: 'assignee', @@ -202,31 +202,31 @@ }); }; - $scope.toggleGroupedView = function() { + $scope.toggleGroupedView = function () { $scope.state.showGrouped = !$scope.state.showGrouped; AppLayoutSrv.groupTasks($scope.state.showGrouped); }; - $scope.buildTaskGroups = function(tasks) { + $scope.buildTaskGroups = function (tasks) { // Sort tasks by order - var orderedTasks = _.sortBy(_.map(tasks, function(t) { + var orderedTasks = _.sortBy(_.map(tasks, function (t) { return _.pick(t, 'group', 'order'); }), 'order'); var groups = []; // Get group names by keeping the group orders - _.each(orderedTasks, function(task) { - if(groups.indexOf(task.group) === -1) { + _.each(orderedTasks, function (task) { + if (groups.indexOf(task.group) === -1) { groups.push(task.group); } }); var groupedTasks = []; - _.each(groups, function(group) { + _.each(groups, function (group) { groupedTasks.push({ group: group, - tasks: _.filter(tasks, function(t) { + tasks: _.filter(tasks, function (t) { return t.group === group; }) }); @@ -236,7 +236,7 @@ $scope.groupedTasks = groupedTasks; }; - $scope.showTask = function(taskId) { + $scope.showTask = function (taskId) { $state.go('app.case.tasks-item', { itemId: taskId }); @@ -247,98 +247,98 @@ field[fieldName] = newValue; return CaseTaskSrv.update({ taskId: task._id - }, field, function () {}, function (response) { + }, field, function () { }, function (response) { NotificationSrv.error('taskList', response.data, response.status); }); }; - $scope.addTask = function() { + $scope.addTask = function () { CaseTaskSrv.save({ 'caseId': $scope.caseId, 'flag': false - }, $scope.newTask, function() { + }, $scope.newTask, function () { $scope.isNewTask = false; $scope.newTask.title = ''; $scope.newTask.group = ''; NotificationSrv.success('Task has been successfully added'); - }, function(response) { + }, function (response) { NotificationSrv.error('taskList', response.data, response.status); }); }; - $scope.removeTask = function(task) { + $scope.removeTask = function (task) { ModalUtilsSrv.confirm('Delete task', 'Are you sure you want to delete the selected task?', { okText: 'Yes, remove it', flavor: 'danger' - }).then(function() { + }).then(function () { CaseTaskSrv.update({ 'taskId': task._id }, { status: 'Cancel' - }, function() { + }, function () { $scope.$emit('tasks:task-removed', task); NotificationSrv.success('Task has been successfully removed'); - }, function(response) { + }, function (response) { NotificationSrv.error('taskList', response.data, response.status); }); }); }; - $scope.bulkUpdate = function(ids, patch) { + $scope.bulkUpdate = function (ids, patch) { return CaseTaskSrv.bulkUpdate(ids, patch) - .then(function(/*responses*/) { + .then(function (/*responses*/) { NotificationSrv.log('Selected tasks have been updated successfully', 'success'); }) - .catch(function(err) { + .catch(function (err) { NotificationSrv.error('Bulk update tasks', err.data, err.status); }); } - $scope.bulkFlag = function(flag) { + $scope.bulkFlag = function (flag) { var ids = _.pluck($scope.selection, '_id'); - return $scope.bulkUpdate(ids, {flag: flag}); + return $scope.bulkUpdate(ids, { flag: flag }); } - $scope.bulkStatus = function(status) { + $scope.bulkStatus = function (status) { var ids = _.pluck($scope.selection, '_id'); - return $scope.bulkUpdate(ids, {status: status}); + return $scope.bulkUpdate(ids, { status: status }); } - $scope.bulkRemove = function() { + $scope.bulkRemove = function () { var ids = _.pluck($scope.selection, '_id'); ModalUtilsSrv.confirm('Delete selected tasks', 'Are you sure you want to delete the selected tasks?', { okText: 'Yes, proceed', flavor: 'danger' - }).then(function() { - return CaseTaskSrv.bulkUpdate(ids, {status: 'Cancel'}) - .then(function(/*responses*/) { + }).then(function () { + return CaseTaskSrv.bulkUpdate(ids, { status: 'Cancel' }) + .then(function (/*responses*/) { NotificationSrv.log('Selected tasks have been successfully removed', 'success'); - _.each($scope.selection, function(task) { + _.each($scope.selection, function (task) { $scope.$emit('tasks:task-removed', task); }); }) - .catch(function(err) { + .catch(function (err) { NotificationSrv.error('Bulk remove tasks', err.data, err.status); }); - }).catch(function(err) { - if(err && !_.isString(err)) { + }).catch(function (err) { + if (err && !_.isString(err)) { NotificationSrv.error('Bulk remove tasks', err.data, err.status); } }) } // open task tab with its details - $scope.startTask = function(task) { + $scope.startTask = function (task) { var taskId = task._id; if (task.status === 'Waiting') { $scope.updateTaskStatus(taskId, 'InProgress') - .then(function(/*response*/) { + .then(function (/*response*/) { $scope.showTask(taskId); }); } else { @@ -346,34 +346,34 @@ } }; - $scope.openTask = function(task) { + $scope.openTask = function (task) { if (task.status === 'Completed') { $scope.updateTaskStatus(task._id, 'InProgress') - .then(function(/*response*/) { + .then(function (/*response*/) { $scope.showTask(task._id); }); } }; - $scope.closeTask = function(task) { + $scope.closeTask = function (task) { if (task.status === 'InProgress') { $scope.updateTaskStatus(task._id, 'Completed') - .then(function() { + .then(function () { NotificationSrv.success('Task has been successfully closed'); }); } }; - $scope.updateTaskStatus = function(taskId, status) { + $scope.updateTaskStatus = function (taskId, status) { var defer = $q.defer(); CaseTaskSrv.update({ 'taskId': taskId }, { 'status': status - }, function(data) { + }, function (data) { defer.resolve(data); - }, function(response) { + }, function (response) { NotificationSrv.error('taskList', response.data, response.status); defer.reject(response); }); @@ -381,32 +381,32 @@ return defer.promise; }; - $scope.getTaskResponders = function(task, force) { - if(!force && $scope.taskResponders !== null) { - return; + $scope.getTaskResponders = function (task, force) { + if (!force && $scope.taskResponders !== null) { + return; } $scope.taskResponders = null; CortexSrv.getResponders('case_task', task._id) - .then(function(responders) { - $scope.taskResponders = responders; - return CortexSrv.promntForResponder(responders); - }) - .then(function(response) { - if(response && _.isString(response)) { - NotificationSrv.log(response, 'warning'); - } else { - return CortexSrv.runResponder(response.id, response.name, 'case_task', _.pick(task, '_id')); - } - }) - .then(function(response){ - NotificationSrv.success(['Responder', response.data.responderName, 'started successfully on task', task.title].join(' ')); - }) - .catch(function(err) { - if(err && !_.isString(err)) { - NotificationSrv.error('taskList', err.data, err.status); - } - }); + .then(function (responders) { + $scope.taskResponders = responders; + return CortexSrv.promntForResponder(responders); + }) + .then(function (response) { + if (response && _.isString(response)) { + NotificationSrv.log(response, 'warning'); + } else { + return CortexSrv.runResponder(response.id, response.name, 'case_task', _.pick(task, '_id')); + } + }) + .then(function (response) { + NotificationSrv.success(['Responder', response.data.responderName, 'started successfully on task', task.title].join(' ')); + }) + .catch(function (err) { + if (err && !_.isString(err)) { + NotificationSrv.error('taskList', err.data, err.status); + } + }); }; // $scope.runResponder = function(responderId, responderName, task) { diff --git a/frontend/app/views/directives/psearch.html b/frontend/app/views/directives/psearch.html index dbcbeb9cd9..72fe794a29 100644 --- a/frontend/app/views/directives/psearch.html +++ b/frontend/app/views/directives/psearch.html @@ -1,11 +1,11 @@
        -
        +
        -
        +
          diff --git a/frontend/app/views/partials/case/case.tasks.html b/frontend/app/views/partials/case/case.tasks.html index 99dae58ae0..ab13974183 100755 --- a/frontend/app/views/partials/case/case.tasks.html +++ b/frontend/app/views/partials/case/case.tasks.html @@ -53,290 +53,300 @@

          -
          +
          -
          - - - - - - - - - - - - - - - - - - - - + - - - + + + + +
          - - GroupTaskDateAssigneeActions
          - - - - - - - {{task.group}} - - -
          - Closed after {{(task.endDate - task.startDate) | amDurationFormat : - 'milliseconds'}} -
          -
          - Started -
          -           		- + +
          + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - -
          + + GroupTaskDateAssigneeActions
          + + + + + + + + + + {{task.group}} + + +
          + Closed after {{(task.endDate - task.startDate) | amDurationFormat : + 'milliseconds'}} +
          +
          + Started +
          +           		+
          + -
          		{{task.startDate | shortDate}} - - - - - - - - - - - - Not Assigned - - + + ({{task.extraData.shareCount || 0}}) + + - - - - - - - - - - - - - - - -
          {{task.startDate | shortDate}} + + + + + + + + + + + + Not Assigned + + -
          -
          -
          + + + + + + + + + + + + + + + +
          +
          +
          +
          -
          -
          + +
          +
          +
          + {{group.group || 'Not Specified'}} ({{group.tasks.length}} task(s)) +
          +
          + + + + + + + + + + + + + + + + + - + + + + + + + + + + + +
          + + GroupTaskDateAssigneeActions
          + + + + + + + + + {{task.group}} + + + + +
          + Closed after {{(task.endDate - task.startDate) | amDurationFormat : + 'milliseconds'}} +
          +
          + Started +
          +           		+
          +
          +
          +
          -
          diff --git a/frontend/app/views/partials/case/tasks/toolbar.html b/frontend/app/views/partials/case/tasks/toolbar.html index 8fe46997a2..d47a7eae3e 100644 --- a/frontend/app/views/partials/case/tasks/toolbar.html +++ b/frontend/app/views/partials/case/tasks/toolbar.html @@ -3,8 +3,11 @@
          - +
          From 29d65643894b5bd922cb2e425cd71fc42e4f257d Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 08:20:31 +0200 Subject: [PATCH 49/63] #1954 Add default file for service environment --- package.sbt | 1 + package/thehive.default | 4 ++++ package/thehive.service | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 package/thehive.default diff --git a/package.sbt b/package.sbt index bc6a06977a..a8ae03beb4 100644 --- a/package.sbt +++ b/package.sbt @@ -40,6 +40,7 @@ linuxPackageMappings ++= Seq( file("package/thehive.service") -> "/usr/lib/systemd/system/thehive.service" ).withPerms("644"), packageMapping( + file("package/thehive.default") -> "/etc/default/thehive", file("conf/application.sample.conf") -> "/etc/thehive/application.conf", file("package/logback.xml") -> "/etc/thehive/logback.xml", file("package/logback-migration.xml") -> "/etc/thehive/logback-migration.xml" diff --git a/package/thehive.default b/package/thehive.default new file mode 100644 index 0000000000..d72cf7a890 --- /dev/null +++ b/package/thehive.default @@ -0,0 +1,4 @@ +# Environment File for TheHive + +# JAVA_OPTS for TheHive service can be set here +#JAVA_OPTS="" diff --git a/package/thehive.service b/package/thehive.service index bb93d45540..77191ef248 100644 --- a/package/thehive.service +++ b/package/thehive.service @@ -5,6 +5,7 @@ Wants=network-online.target After=network-online.target [Service] +EnvironmentFile=-/etc/default/thehive WorkingDirectory=/opt/thehive User=thehive @@ -34,4 +35,4 @@ SendSIGKILL=no SuccessExitStatus=143 [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target From 4bdac381631b54d0ddbdca9e3f88b5116c947940 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 08:26:29 +0200 Subject: [PATCH 50/63] #1946 Use index to count alert observable --- thehive/app/org/thp/thehive/services/AlertSrv.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index a49baa3447..5eccade0b3 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -551,11 +551,11 @@ object AlertOps { .by(_.richCustomFields.fold) .by(_.`case`._id.option) .by(_.caseTemplate.value(_.name).option) - .by(_.observables.count) .by(entityRenderer) ) .domainMap { - case (alert, customFields, caseId, caseTemplate, observableCount, renderedEntity) => + case (alert, customFields, caseId, caseTemplate, renderedEntity) => + val observableCount = traversal.graph.indexCountQuery(s"""v."_label":Observable AND v.relatedId:${alert._id.value}""") RichAlert( alert, customFields, From 516d074c66e1078c8718feb2488045dd9987fb2c Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 09:16:25 +0200 Subject: [PATCH 51/63] #1946 Return all element count in platform status API --- .../thehive/controllers/v1/AdminCtrl.scala | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index ecc2fdb1c2..0e4641120f 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -3,8 +3,10 @@ package org.thp.thehive.controllers.v1 import akka.actor.ActorRef import akka.pattern.ask import akka.util.Timeout +import ch.qos.logback.classic.{Level, LoggerContext} +import org.slf4j.LoggerFactory import org.thp.scalligraph.controllers.Entrypoint -import org.thp.scalligraph.models.Database +import org.thp.scalligraph.models.{Database, UpdatableSchema, VertexModel} import org.thp.scalligraph.services.GenIntegrityCheckOps import org.thp.thehive.models.Permissions import org.thp.thehive.services.{CheckState, CheckStats, GetCheckStats, GlobalCheckRequest} @@ -17,8 +19,6 @@ import scala.collection.immutable import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} import scala.util.Success -import ch.qos.logback.classic.{Level, LoggerContext} -import org.slf4j.LoggerFactory @Singleton class AdminCtrl @Inject() ( @@ -26,6 +26,7 @@ class AdminCtrl @Inject() ( @Named("integrity-check-actor") integrityCheckActor: ActorRef, integrityCheckOps: immutable.Set[GenIntegrityCheckOps], db: Database, + schemas: immutable.Set[UpdatableSchema], implicit val ec: ExecutionContext ) { @@ -87,20 +88,14 @@ class AdminCtrl @Inject() ( } } - private val indexedModels = Seq("global") def indexStatus: Action[AnyContent] = entrypoint("Get index status") .authPermittedRoTransaction(db, Permissions.managePlatform) { _ => graph => - val status = indexedModels.map { label => -// val mixedCount = graph.V(label).getCount -// val compositeCount = graph.underlying.traversal().V().has("_label", label).count().next().toLong - label -> Json.obj( - "mixedCount" -> -1, - "compositeCount" -> -1, - "status" -> "OK" - ) + val indices = schemas.flatMap(_.modelList).collect { + case v: VertexModel => Json.obj("name" -> v.label, "count" -> graph.indexCountQuery(s"""v."_label":${v.label}""")) } - Success(Results.Ok(JsObject(status))) + val indexCount = Json.obj("name" -> "global", "indices" -> indices) + Success(Results.Ok(Json.obj("index" -> Seq(indexCount)))) } def reindex(label: String): Action[AnyContent] = From 4b305dc9eaf046ae406a95a53bd7e39dccfd6fec Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 09:16:48 +0200 Subject: [PATCH 52/63] #1946 Fix related alert count in case page --- frontend/app/scripts/controllers/case/CaseMainCtrl.js | 2 +- thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/app/scripts/controllers/case/CaseMainCtrl.js b/frontend/app/scripts/controllers/case/CaseMainCtrl.js index defd55aed9..b322fc7e9e 100644 --- a/frontend/app/scripts/controllers/case/CaseMainCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseMainCtrl.js @@ -128,7 +128,7 @@ // Stats for case alerts counter StreamQuerySrv('v1', [ - { _name: 'countImportedAlert', caseId: caseId }, + { _name: 'countRelatedAlert', caseId: caseId }, ], { scope: $scope, rootId: caseId, diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index eb1a15a69b..268b515b7e 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -73,6 +73,13 @@ class AlertCtrl @Inject() ( s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND NOT v.caseId:[* TO 'ZZZZZZZZ']""" ) ), + Query.initWithParam[InCase, Long]( + "countRelatedAlert", + (inCase, graph, authContext) => + graph.indexCountQuery( + s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND v.caseId:${inCase.caseId.value}""" + ) + ), Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables), Query[Traversal.V[Alert], Traversal.V[Case]]("case", (alertSteps, _) => alertSteps.`case`), Query.withParam[Option[InputQuery[Traversal.Unk, Traversal.Unk]], Traversal.V[Alert], Traversal[ From d4cae87470b64a9dbf58c51eb79e10ef4e61e598 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 09:40:22 +0200 Subject: [PATCH 53/63] #1946 Rename query countObservable=>countCaseObservable and add countAlertObservable query --- frontend/app/scripts/controllers/case/CaseMainCtrl.js | 2 +- .../thp/thehive/controllers/v1/ObservableCtrl.scala | 11 ++++++++++- .../thehive/controllers/v1/TheHiveQueryExecutor.scala | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/frontend/app/scripts/controllers/case/CaseMainCtrl.js b/frontend/app/scripts/controllers/case/CaseMainCtrl.js index b322fc7e9e..598f9dc691 100644 --- a/frontend/app/scripts/controllers/case/CaseMainCtrl.js +++ b/frontend/app/scripts/controllers/case/CaseMainCtrl.js @@ -110,7 +110,7 @@ // Stats for case observables counter StreamQuerySrv('v1', [ - { _name: 'countObservable', caseId: caseId }, + { _name: 'countCaseObservable', caseId: caseId }, ], { scope: $scope, rootId: caseId, diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index 53df4dcc8a..fb0d65a4c6 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -75,12 +75,21 @@ class ObservableCtrl @Inject() ( override val extraQueries: Seq[ParamQuery[_]] = Seq( Query.initWithParam[InCase, Long]( - "countObservable", + "countCaseObservable", (inCase, graph, authContext) => graph.indexCountQuery( s"""v."_label":Observable AND relatedId:${inCase.caseId.value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" ) ), + Query.initWithParam[InAlert, Long]( + "countAlertObservable", + (inAlert, graph, authContext) => + graph.indexCountQuery( + s"""v."_label":Observable AND relatedId:${inAlert + .alertId + .value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" + ) + ), Query[Traversal.V[Observable], Traversal.V[Organisation]]( "organisations", (observableSteps, authContext) => observableSteps.organisations.visible(authContext) diff --git a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala index 33833f4eb7..0909c32dbc 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TheHiveQueryExecutor.scala @@ -9,6 +9,7 @@ import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import javax.inject.{Inject, Singleton} case class InCase(caseId: EntityId) +case class InAlert(alertId: EntityId) case class OutputParam(from: Long, to: Long, extraData: Set[String]) From 75420fce2271ff11c1e0cb633769ac7d42213806 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 10:06:27 +0200 Subject: [PATCH 54/63] #1946 Escape parameters in indexCountQuery --- ScalliGraph | 2 +- .../app/org/thp/thehive/controllers/v1/AlertCtrl.scala | 4 +++- .../org/thp/thehive/controllers/v1/CaseRenderer.scala | 2 +- .../thp/thehive/controllers/v1/ObservableCtrl.scala | 10 ++++++---- .../org/thp/thehive/controllers/v1/TagRenderer.scala | 9 ++++++--- .../app/org/thp/thehive/controllers/v1/TaskCtrl.scala | 2 +- thehive/app/org/thp/thehive/services/AlertSrv.scala | 7 ++++++- 7 files changed, 24 insertions(+), 12 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index b168949a6b..ae7ceb013c 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit b168949a6b3918306a206d554aa154023533ea14 +Subproject commit ae7ceb013c923295f3ac95f7e2514b8e19218355 diff --git a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala index 268b515b7e..ecd6ed2077 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AlertCtrl.scala @@ -77,7 +77,9 @@ class AlertCtrl @Inject() ( "countRelatedAlert", (inCase, graph, authContext) => graph.indexCountQuery( - s"""v."_label":Alert AND v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND v.caseId:${inCase.caseId.value}""" + s"""v."_label":Alert AND """ + + s"v.organisationId:${organisationSrv.currentId(graph, authContext).value} AND " + + s"v.caseId:${graph.escapeQueryParameter(inCase.caseId.value)}" ) ), Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables), diff --git a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala index 3b4325cccf..f248a3ede7 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/CaseRenderer.scala @@ -27,7 +27,7 @@ trait CaseRenderer extends BaseRenderer[Case] { .graph .indexCountQuery( s"""v."_label":Observable AND """ + - s"v.relatedId:${caseId.value} AND " + + s"v.relatedId:${t.graph.escapeQueryParameter(caseId.value)} AND " + s"v.organisationIds:${organisationSrv.currentId(t.graph, authContext).value}" ) ) diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala index fb0d65a4c6..a74733e829 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala @@ -78,16 +78,18 @@ class ObservableCtrl @Inject() ( "countCaseObservable", (inCase, graph, authContext) => graph.indexCountQuery( - s"""v."_label":Observable AND relatedId:${inCase.caseId.value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" + s"""v."_label":Observable AND """ + + s"relatedId:${graph.escapeQueryParameter(inCase.caseId.value)} AND " + + s"organisationIds:${organisationSrv.currentId(graph, authContext).value}" ) ), Query.initWithParam[InAlert, Long]( "countAlertObservable", (inAlert, graph, authContext) => graph.indexCountQuery( - s"""v."_label":Observable AND relatedId:${inAlert - .alertId - .value} AND organisationIds:${organisationSrv.currentId(graph, authContext).value}""" + s"""v."_label":Observable AND """ + + s"relatedId:${graph.escapeQueryParameter(inAlert.alertId.value)} AND " + + s"organisationIds:${organisationSrv.currentId(graph, authContext).value}" ) ), Query[Traversal.V[Observable], Traversal.V[Organisation]]( diff --git a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala index 4809deacbd..c6f0d4f88c 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TagRenderer.scala @@ -27,19 +27,22 @@ trait TagRenderer extends BaseRenderer[Tag] { "case" -> t .graph .indexCountQuery( - s"""v."_label":Case AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}" AND """ + + s"""v."_label":Case AND """ + + s"v.tags:${t.graph.escapeQueryParameter(tag)} AND " + s"v.organisationIds:${organisationSrv.currentId(t.graph, authContext).value}" ), "alert" -> t .graph .indexCountQuery( - s"""v."_label":Alert AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}" AND """ + + s"""v."_label":Alert AND """ + + s"v.tags:${t.graph.escapeQueryParameter(tag)} AND " + s"v.organisationId:${organisationSrv.currentId(t.graph, authContext).value}" ), "observable" -> t .graph .indexCountQuery( - s"""v."_label":Observable AND v.tags:"${tag.replaceAllLiterally("\"", "\\\"")}" AND """ + + s"""v."_label":Observable AND """ + + s"v.tags:${t.graph.escapeQueryParameter(tag)} AND " + s"v.organisationIds:${organisationSrv.currentId(t.graph, authContext).value}" ), "caseTemplate" -> caseTemplateCount diff --git a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala index 7aeb37e41c..8acb8fcfb6 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala @@ -58,7 +58,7 @@ class TaskCtrl @Inject() ( (inCase, graph, authContext) => graph.indexCountQuery( s"""v."_label":Task AND """ + - s"v.relatedId:${inCase.caseId.value} AND " + + s"v.relatedId:${graph.escapeQueryParameter(inCase.caseId.value)} AND " + s"v.organisationIds:${organisationSrv.currentId(graph, authContext).value} AND " + "NOT v.status:Cancel" ) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 5eccade0b3..9783100087 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -555,7 +555,12 @@ object AlertOps { ) .domainMap { case (alert, customFields, caseId, caseTemplate, renderedEntity) => - val observableCount = traversal.graph.indexCountQuery(s"""v."_label":Observable AND v.relatedId:${alert._id.value}""") + val observableCount = traversal + .graph + .indexCountQuery( + s"""v."_label":Observable AND """ + + s"v.relatedId:${traversal.graph.escapeQueryParameter(alert._id.value)}" + ) RichAlert( alert, customFields, From 04effb70c70114a35f6963e732f593507cf47bd9 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 10:32:02 +0200 Subject: [PATCH 55/63] #1946 Filter labels return by platform status --- .../thehive/controllers/v1/AdminCtrl.scala | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index 0e4641120f..045e1c4467 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -26,7 +26,6 @@ class AdminCtrl @Inject() ( @Named("integrity-check-actor") integrityCheckActor: ActorRef, integrityCheckOps: immutable.Set[GenIntegrityCheckOps], db: Database, - schemas: immutable.Set[UpdatableSchema], implicit val ec: ExecutionContext ) { @@ -88,11 +87,37 @@ class AdminCtrl @Inject() ( } } + val labels = Seq( + "Config", + "ReportTag", + "KeyValue", + "Pattern", + "Case", + "Procedure", + "Alert", + "Dashboard", + "Observable", + "User", + "AnalyzerTemplate", + "Taxonomy", + "CustomField", + "Data", + "Organisation", + "Profile", + "Task", + "Action", + "Log", + "CaseTemplate", + "Audit", + "Tag", + "Job", + "Attachment" + ) def indexStatus: Action[AnyContent] = entrypoint("Get index status") .authPermittedRoTransaction(db, Permissions.managePlatform) { _ => graph => - val indices = schemas.flatMap(_.modelList).collect { - case v: VertexModel => Json.obj("name" -> v.label, "count" -> graph.indexCountQuery(s"""v."_label":${v.label}""")) + val indices = labels.map { label => + Json.obj("name" -> label, "count" -> graph.indexCountQuery(s"""v."_label":$label""")) } val indexCount = Json.obj("name" -> "global", "indices" -> indices) Success(Results.Ok(Json.obj("index" -> Seq(indexCount)))) From da6b3a0625ee8ae74aefb94223dd964e26fdeec0 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 10:32:53 +0200 Subject: [PATCH 56/63] #1946 Optimize imports --- thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index 045e1c4467..77ca67817b 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -6,7 +6,7 @@ import akka.util.Timeout import ch.qos.logback.classic.{Level, LoggerContext} import org.slf4j.LoggerFactory import org.thp.scalligraph.controllers.Entrypoint -import org.thp.scalligraph.models.{Database, UpdatableSchema, VertexModel} +import org.thp.scalligraph.models.Database import org.thp.scalligraph.services.GenIntegrityCheckOps import org.thp.thehive.models.Permissions import org.thp.thehive.services.{CheckState, CheckStats, GetCheckStats, GlobalCheckRequest} From ed5fc86034efa4c16e3be91d9f5799890ddbf301 Mon Sep 17 00:00:00 2001 From: To-om Date: Mon, 12 Apr 2021 10:43:57 +0200 Subject: [PATCH 57/63] #1946 Hide useless messages --- package/logback.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/package/logback.xml b/package/logback.xml index af4ead5d9d..ae905cd3d9 100644 --- a/package/logback.xml +++ b/package/logback.xml @@ -37,6 +37,7 @@ + From ce0fca735b4c43b188815cb7a17563e6b45af857 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 12 Apr 2021 10:52:54 +0200 Subject: [PATCH 58/63] #1946 Update the platform status page to use index backend for index counts --- .../views/partials/admin/platform/status.html | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/frontend/app/views/partials/admin/platform/status.html b/frontend/app/views/partials/admin/platform/status.html index e514771def..12b4c23851 100644 --- a/frontend/app/views/partials/admin/platform/status.html +++ b/frontend/app/views/partials/admin/platform/status.html @@ -31,7 +31,8 @@

          Database Schema status

          {{schema.error === null ? 'OK' : 'ERROR'}} + false: 'label-danger'}[schema.error === null]">{{schema.error === null ? 'OK' : + 'ERROR'}}

          @@ -49,37 +50,33 @@

          -

          Data index status Reload

          +

          Data index status Reload

          Loading index status...
          -
          +
          -
          Status
          -
          Index name
          +
          Index: {{index.name | uppercase}}
          # Entities
          -
          +
          + Reindex +
          -
          +
          -
          - {{data.status | uppercase}} -

          - {{indexName}} + {{item.name}}

          - {{data.mixedCount}} + {{item.count}}
          -
          - Reindex -
          +
          @@ -102,7 +99,7 @@

          Trigger + class="fa fa-cog mr-xxxs"> Trigger

          From 6c63d781d13ee88c1f5f31fad9d5b45162839b2a Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Mon, 12 Apr 2021 11:23:11 +0200 Subject: [PATCH 59/63] #1948 WIP: Use optimised counts in alert observable list --- .../alert/AlertObservableListCmp.js | 39 ++++++++------ .../alert/observable-list.component.html | 52 +++++++++++-------- .../components/alert/observables/toolbar.html | 13 ++++- 3 files changed, 64 insertions(+), 40 deletions(-) diff --git a/frontend/app/scripts/components/alert/AlertObservableListCmp.js b/frontend/app/scripts/components/alert/AlertObservableListCmp.js index 1e7b032827..d0d8dfcd23 100644 --- a/frontend/app/scripts/components/alert/AlertObservableListCmp.js +++ b/frontend/app/scripts/components/alert/AlertObservableListCmp.js @@ -1,12 +1,12 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveComponents') .component('alertObservableList', { - controller: function($scope, FilteringSrv, PaginatedQuerySrv) { + controller: function ($scope, FilteringSrv, QuerySrv, PaginatedQuerySrv) { var self = this; - self.$onInit = function() { + self.$onInit = function () { this.filtering = new FilteringSrv('observable', 'alert.dialog.observables', { version: 'v1', defaults: { @@ -19,37 +19,44 @@ }); self.filtering.initContext(this.alertId) - .then(function() { + .then(function () { self.load(); $scope.$watch('$cmp.list.pageSize', function (newValue) { self.filtering.setPageSize(newValue); }); + }); - $scope.$watch('$cmp.list.total', function (total) { - self.onListLoad({count: total}); - }); + QuerySrv.query( + 'v1', + [{ '_name': 'countAlertObservable', 'alertId': self.alertId }], + { + params: { + name: 'alert-all-observables.count' + } + }) + .then(function (response) { + self.observablesCount = response.data; + self.onListLoad({ count: self.observablesCount }); }); }; - this.load = function() { + this.load = function () { this.list = new PaginatedQuerySrv({ name: 'alert-observables', skipStream: true, version: 'v1', sort: self.filtering.context.sort, loadAll: false, + limitedCount: true, pageSize: self.filtering.context.pageSize, filter: this.filtering.buildQuery(), operations: [ - {'_name': 'getAlert', 'idOrName': this.alertId}, - {'_name': 'observables'} + { '_name': 'getAlert', 'idOrName': this.alertId }, + { '_name': 'observables' } ], extraData: ['seen'], - onUpdate: function() { - //self.resetSelection(); - - } + onUpdate: function () { } }); }; @@ -63,9 +70,9 @@ this.search(); }; - this.filterBy = function(field, value) { + this.filterBy = function (field, value) { this.filtering.clearFilters() - .then(function() { + .then(function () { self.addFilterValue(field, value); }); }; diff --git a/frontend/app/views/components/alert/observable-list.component.html b/frontend/app/views/components/alert/observable-list.component.html index 131bca24ed..e6bea41a0e 100644 --- a/frontend/app/views/components/alert/observable-list.component.html +++ b/frontend/app/views/components/alert/observable-list.component.html @@ -1,25 +1,25 @@ -
          +
          -
          +
          -
          -
          +
          No records
          -
          +
          @@ -34,28 +34,36 @@ diff --git a/frontend/app/views/components/alert/observables/toolbar.html b/frontend/app/views/components/alert/observables/toolbar.html index e83ac7f7a4..6b317e046a 100644 --- a/frontend/app/views/components/alert/observables/toolbar.html +++ b/frontend/app/views/components/alert/observables/toolbar.html @@ -1,13 +1,22 @@
          - + - {{observable.dataType}} + {{observable.dataType}} -
          - {{observable.data | fang | ellipsis:250}} -
          -
          - {{observable.attachment.name}} ({{observable.attachment.size}} bytes) -
          -
          - {{observable.remoteAttachment.filename}} -
          -
          - - -
          +
          + {{observable.data | fang | ellipsis:250}} +
          +
          + {{observable.attachment.name}} ({{observable.attachment.size}} bytes) +
          +
          + {{observable.remoteAttachment.filename}} +
          +
          + + +
          		- {{observable._createdAt | shortDate}} + {{observable._createdAt + | shortDate}}